From 35f16b2041cc205be56d6700e9fc933bfb1a8171 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Mar 2022 15:57:12 +0800 Subject: [PATCH 01/22] minor changes --- include/dnode/bnode/bnode.h | 2 -- include/dnode/qnode/qnode.h | 2 -- include/dnode/snode/snode.h | 2 -- source/dnode/mgmt/bnode/src/bmInt.c | 2 -- source/dnode/mgmt/qnode/src/qmInt.c | 2 -- source/dnode/mgmt/snode/src/smInt.c | 2 -- 6 files changed, 12 deletions(-) diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 84c2ea55f7..6f5888bdeb 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -28,8 +28,6 @@ typedef struct { } SBnodeLoad; typedef struct { - int32_t dnodeId; - int64_t clusterId; SMgmtWrapper *pWrapper; SendReqFp sendReqFp; SendMnodeReqFp sendMnodeReqFp; diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index df735cf1b3..354d8d2819 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -36,8 +36,6 @@ typedef struct { } SQnodeLoad; typedef struct { - int32_t dnodeId; - int64_t clusterId; SMgmtWrapper *pWrapper; SendReqFp sendReqFp; SendMnodeReqFp sendMnodeReqFp; diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 4202859359..4855f71a76 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -32,8 +32,6 @@ typedef struct { } SSnodeLoad; typedef struct { - int32_t dnodeId; - int64_t clusterId; SMgmtWrapper *pWrapper; SendReqFp sendReqFp; SendMnodeReqFp sendMnodeReqFp; diff --git a/source/dnode/mgmt/bnode/src/bmInt.c b/source/dnode/mgmt/bnode/src/bmInt.c index 9964b88090..85102cbffc 100644 --- a/source/dnode/mgmt/bnode/src/bmInt.c +++ b/source/dnode/mgmt/bnode/src/bmInt.c @@ -24,8 +24,6 @@ static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { pOption->sendReqFp = dndSendReqToDnode; pOption->sendMnodeReqFp = dndSendReqToMnode; pOption->sendRspFp = dndSendRsp; - pOption->dnodeId = pDnode->dnodeId; - pOption->clusterId = pDnode->clusterId; } static int32_t bmOpenImp(SBnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c index 78ef238ab8..f9a39a8774 100644 --- a/source/dnode/mgmt/qnode/src/qmInt.c +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -24,8 +24,6 @@ static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { pOption->sendReqFp = dndSendReqToDnode; pOption->sendMnodeReqFp = dndSendReqToMnode; pOption->sendRspFp = dndSendRsp; - pOption->dnodeId = pDnode->dnodeId; - pOption->clusterId = pDnode->clusterId; } static int32_t qmOpenImp(SQnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/snode/src/smInt.c b/source/dnode/mgmt/snode/src/smInt.c index 0742a43157..62f2edacb7 100644 --- a/source/dnode/mgmt/snode/src/smInt.c +++ b/source/dnode/mgmt/snode/src/smInt.c @@ -24,8 +24,6 @@ static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { pOption->sendReqFp = dndSendReqToDnode; pOption->sendMnodeReqFp = dndSendReqToMnode; pOption->sendRspFp = dndSendRsp; - pOption->dnodeId = pDnode->dnodeId; - pOption->clusterId = pDnode->clusterId; } static int32_t smOpenImp(SSnodeMgmt *pMgmt) { From 1751c0285a8f20caea417e87185790d750d9df40 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 21 Mar 2022 16:47:59 +0800 Subject: [PATCH 02/22] add parallel --- include/common/tmsg.h | 4 ++-- source/common/src/tmsg.c | 4 ++++ source/dnode/snode/src/snode.c | 5 ++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 852f77777a..3e5a53c288 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2274,8 +2274,8 @@ enum { }; typedef struct { - void* inputHandle; - void** executor; + void* inputHandle; + void* executor[4]; } SStreamTaskParRunner; typedef struct { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 1d2c9397c0..3517f7176c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2718,6 +2718,8 @@ int32_t tEncodeSStreamTask(SCoder *pEncoder, const SStreamTask *pTask) { if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->level) < 0) return -1; if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->pipeEnd) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->parallel) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->NextOpEp) < 0) return -1; if (tEncodeCStr(pEncoder, pTask->qmsg) < 0) return -1; tEndEncode(pEncoder); @@ -2730,6 +2732,8 @@ int32_t tDecodeSStreamTask(SCoder *pDecoder, SStreamTask *pTask) { if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->level) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->pipeEnd) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->parallel) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->NextOpEp) < 0) return -1; if (tDecodeCStrAlloc(pDecoder, &pTask->qmsg) < 0) return -1; tEndDecode(pDecoder); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index f4129e37ce..cbee8d2f90 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -57,7 +57,9 @@ void sndMetaDelete(SStreamMeta *pMeta) { } int32_t sndMetaDeployTask(SStreamMeta *pMeta, SStreamTask *pTask) { - pTask->runner.executor = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); + for (int i = 0; i < pTask->parallel; i++) { + pTask->runner.executor[i] = qCreateStreamExecTaskInfo(pTask->qmsg, NULL); + } return taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), pTask, sizeof(void *)); } @@ -95,6 +97,7 @@ void sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { SStreamTask *pTask = malloc(sizeof(SStreamTask)); if (pTask == NULL) { ASSERT(0); + return; } SCoder decoder; tCoderInit(&decoder, TD_LITTLE_ENDIAN, msg, pMsg->contLen - sizeof(SMsgHead), TD_DECODER); From 220fdfabe290cb512219eb9a49182180567a81ac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Mar 2022 19:08:25 +0800 Subject: [PATCH 03/22] refact dnode queue --- include/common/tmsg.h | 8 ---- include/common/tmsgcb.h | 54 +++++++++++++++++++++ include/dnode/bnode/bnode.h | 10 ++-- include/dnode/mnode/mnode.h | 21 ++++---- include/dnode/qnode/qnode.h | 10 ++-- include/dnode/snode/snode.h | 10 ++-- include/libs/qworker/qworker.h | 7 +-- source/common/src/tmsgcb.c | 31 ++++++++++++ source/dnode/bnode/inc/bndInt.h | 2 +- source/dnode/bnode/src/bnode.c | 1 + source/dnode/mgmt/bnode/src/bmInt.c | 11 +++-- source/dnode/mgmt/mnode/src/mmInt.c | 15 +++--- source/dnode/mgmt/qnode/src/qmInt.c | 11 +++-- source/dnode/mgmt/snode/src/smInt.c | 11 +++-- source/dnode/mgmt/vnode/src/vmInt.c | 17 ++++--- source/dnode/mgmt/vnode/src/vmMsg.c | 9 +++- source/dnode/mnode/impl/inc/mndInt.h | 13 ++--- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 2 +- source/dnode/mnode/impl/src/mnode.c | 33 ++----------- source/dnode/qnode/inc/qndInt.h | 4 +- source/dnode/qnode/src/qnode.c | 12 ++--- source/dnode/snode/inc/sndInt.h | 2 +- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/inc/vnode.h | 15 ++---- source/dnode/vnode/src/inc/vnd.h | 8 +--- source/dnode/vnode/src/vnd/vnodeMain.c | 4 +- source/dnode/vnode/src/vnd/vnodeMgr.c | 56 +++++----------------- source/dnode/vnode/src/vnd/vnodeQuery.c | 3 +- source/libs/qworker/inc/qworkerInt.h | 23 +++++---- source/libs/qworker/src/qworker.c | 9 ++-- source/libs/qworker/src/qworkerMsg.c | 14 +++--- source/libs/qworker/test/qworkerTests.cpp | 15 ++++-- 34 files changed, 223 insertions(+), 224 deletions(-) create mode 100644 include/common/tmsgcb.h create mode 100644 source/common/src/tmsgcb.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 595c03b752..b1c64e1b40 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2319,14 +2319,6 @@ typedef struct { #pragma pack(pop) -struct SRpcMsg; -struct SEpSet; -struct SMgmtWrapper; -typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); -typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* pReq); -typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); -typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pRsp); - #ifdef __cplusplus } #endif diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h new file mode 100644 index 0000000000..413cb957d2 --- /dev/null +++ b/include/common/tmsgcb.h @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_MSG_CB_H_ +#define _TD_COMMON_MSG_CB_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct SRpcMsg; +struct SEpSet; +struct SMgmtWrapper; +typedef struct SMgmtWrapper SMgmtWrapper; + +typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); +typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* pReq); +typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); +typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pRsp); + +typedef enum { QUERY_QUEUE, FETCH_QUEUE, WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, QUEUE_MAX } EMsgQueueType; + +typedef struct { + struct SMgmtWrapper* pWrapper; + PutToQueueFp queueFps[QUEUE_MAX]; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; +} SMsgCb; + +int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EMsgQueueType qtype, struct SRpcMsg* pReq); +int32_t tmsgSendReq(const SMsgCb* pMsgCb, struct SEpSet* epSet, struct SRpcMsg* pReq); +int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, struct SRpcMsg* pReq); +void tmsgSendRsp(const SMsgCb* pMsgCb, struct SRpcMsg* pRsp); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_MSG_CB_H_*/ diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 6f5888bdeb..528cea8828 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -16,22 +16,20 @@ #ifndef _TD_BNODE_H_ #define _TD_BNODE_H_ +#include "tmsgcb.h" + #ifdef __cplusplus extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SMgmtWrapper SMgmtWrapper; -typedef struct SBnode SBnode; +typedef struct SBnode SBnode; typedef struct { } SBnodeLoad; typedef struct { - SMgmtWrapper *pWrapper; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - SendRspFp sendRspFp; + SMsgCb msgCb; } SBnodeOpt; /* ------------------------ SBnode ------------------------ */ diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index d421b2e45d..eed13e0317 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -17,27 +17,22 @@ #define _TD_MND_H_ #include "monitor.h" +#include "tmsgcb.h" #ifdef __cplusplus extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SMgmtWrapper SMgmtWrapper; -typedef struct SMnode SMnode; +typedef struct SMnode SMnode; typedef struct { - int32_t dnodeId; - int64_t clusterId; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; - SMgmtWrapper *pWrapper; - PutToQueueFp putToWriteQFp; - PutToQueueFp putToReadQFp; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - SendRspFp sendRspFp; + int32_t dnodeId; + int64_t clusterId; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; + SMsgCb msgCb; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 354d8d2819..67ee509622 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -16,13 +16,14 @@ #ifndef _TD_QNODE_H_ #define _TD_QNODE_H_ +#include "tmsgcb.h" + #ifdef __cplusplus extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SMgmtWrapper SMgmtWrapper; -typedef struct SQnode SQnode; +typedef struct SQnode SQnode; typedef struct { int64_t numOfStartTask; @@ -36,10 +37,7 @@ typedef struct { } SQnodeLoad; typedef struct { - SMgmtWrapper *pWrapper; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - SendRspFp sendRspFp; + SMsgCb msgCb; } SQnodeOpt; /* ------------------------ SQnode ------------------------ */ diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 4855f71a76..037dca968a 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -16,7 +16,7 @@ #ifndef _TD_SNODE_H_ #define _TD_SNODE_H_ -#include "tcommon.h" +#include "tmsgcb.h" #include "tmsg.h" #include "trpc.h" @@ -25,17 +25,13 @@ extern "C" { #endif /* ------------------------ TYPES EXPOSED ------------------------ */ -typedef struct SMgmtWrapper SMgmtWrapper; -typedef struct SSnode SSnode; +typedef struct SSnode SSnode; typedef struct { } SSnodeLoad; typedef struct { - SMgmtWrapper *pWrapper; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - SendRspFp sendRspFp; + SMsgCb msgCb; } SSnodeOpt; /* ------------------------ SSnode ------------------------ */ diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 8954cbaa0e..944ac97ddb 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "tmsgcb.h" #include "trpc.h" @@ -48,11 +49,7 @@ typedef struct { uint64_t numOfErrors; } SQWorkerStat; -typedef int32_t (*putReqToQueryQFp)(void *, struct SRpcMsg *); -typedef int32_t (*sendReqFp)(void *, struct SEpSet *, struct SRpcMsg *); - -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqFp fp2); +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb); int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg); diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c new file mode 100644 index 0000000000..19973770e4 --- /dev/null +++ b/source/common/src/tmsgcb.c @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "tmsgcb.h" + +int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EMsgQueueType qtype, struct SRpcMsg* pReq) { + return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq); +} + +int32_t tmsgSendReq(const SMsgCb* pMsgCb, struct SEpSet* epSet, struct SRpcMsg* pReq) { + return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); +} + +int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, struct SRpcMsg* pReq) { + return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq); +} + +void tmsgSendRsp(const SMsgCb* pMsgCb, struct SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); } diff --git a/source/dnode/bnode/inc/bndInt.h b/source/dnode/bnode/inc/bndInt.h index 4a0d16a749..e611d230a3 100644 --- a/source/dnode/bnode/inc/bndInt.h +++ b/source/dnode/bnode/inc/bndInt.h @@ -30,7 +30,7 @@ extern "C" { #endif typedef struct SBnode { - SBnodeOpt opt; + SMsgCb msgCb; } SBnode; #ifdef __cplusplus diff --git a/source/dnode/bnode/src/bnode.c b/source/dnode/bnode/src/bnode.c index 2e5c96ecdd..4236e85a7f 100644 --- a/source/dnode/bnode/src/bnode.c +++ b/source/dnode/bnode/src/bnode.c @@ -17,6 +17,7 @@ SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) { SBnode *pBnode = calloc(1, sizeof(SBnode)); + pBnode->msgCb = pOption->msgCb; return pBnode; } diff --git a/source/dnode/mgmt/bnode/src/bmInt.c b/source/dnode/mgmt/bnode/src/bmInt.c index 85102cbffc..34c6dbeb57 100644 --- a/source/dnode/mgmt/bnode/src/bmInt.c +++ b/source/dnode/mgmt/bnode/src/bmInt.c @@ -19,11 +19,12 @@ static int32_t bmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static void bmInitOption(SBnodeMgmt *pMgmt, SBnodeOpt *pOption) { - SDnode *pDnode = pMgmt->pDnode; - pOption->pWrapper = pMgmt->pWrapper; - pOption->sendReqFp = dndSendReqToDnode; - pOption->sendMnodeReqFp = dndSendReqToMnode; - pOption->sendRspFp = dndSendRsp; + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; } static int32_t bmOpenImp(SBnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/mnode/src/mmInt.c b/source/dnode/mgmt/mnode/src/mmInt.c index a3c2d40510..aa348ae879 100644 --- a/source/dnode/mgmt/mnode/src/mmInt.c +++ b/source/dnode/mgmt/mnode/src/mmInt.c @@ -39,14 +39,17 @@ static int32_t mmRequire(SMgmtWrapper *pWrapper, bool *required) { static void mmInitOption(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { SDnode *pDnode = pMgmt->pDnode; - pOption->pWrapper = pMgmt->pWrapper; - pOption->putToWriteQFp = mmPutMsgToWriteQueue; - pOption->putToReadQFp = mmPutMsgToReadQueue; - pOption->sendReqFp = dndSendReqToDnode; - pOption->sendMnodeReqFp = dndSendReqToMnode; - pOption->sendRspFp = dndSendRsp; pOption->dnodeId = pDnode->dnodeId; pOption->clusterId = pDnode->clusterId; + + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = mmPutMsgToReadQueue; + msgCb.queueFps[WRITE_QUEUE] = mmPutMsgToWriteQueue; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; } static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c index f9a39a8774..38defd1dc8 100644 --- a/source/dnode/mgmt/qnode/src/qmInt.c +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -19,11 +19,12 @@ static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { - SDnode *pDnode = pMgmt->pDnode; - pOption->pWrapper = pMgmt->pWrapper; - pOption->sendReqFp = dndSendReqToDnode; - pOption->sendMnodeReqFp = dndSendReqToMnode; - pOption->sendRspFp = dndSendRsp; + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; } static int32_t qmOpenImp(SQnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/snode/src/smInt.c b/source/dnode/mgmt/snode/src/smInt.c index 62f2edacb7..b4d6576e57 100644 --- a/source/dnode/mgmt/snode/src/smInt.c +++ b/source/dnode/mgmt/snode/src/smInt.c @@ -19,11 +19,12 @@ static int32_t smRequire(SMgmtWrapper *pWrapper, bool *required) { return dndReadFile(pWrapper, required); } static void smInitOption(SSnodeMgmt *pMgmt, SSnodeOpt *pOption) { - SDnode *pDnode = pMgmt->pDnode; - pOption->pWrapper = pMgmt->pWrapper; - pOption->sendReqFp = dndSendReqToDnode; - pOption->sendMnodeReqFp = dndSendReqToMnode; - pOption->sendRspFp = dndSendRsp; + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + pOption->msgCb = msgCb; } static int32_t smOpenImp(SSnodeMgmt *pMgmt) { diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c index c5e79765c7..12f38d71ca 100644 --- a/source/dnode/mgmt/vnode/src/vmInt.c +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -128,7 +128,13 @@ static void *vmOpenVnodeFunc(void *param) { pMgmt->state.openVnodes, pMgmt->state.totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnodeCfg cfg = {.pWrapper = pMgmt->pWrapper, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + SVnodeCfg cfg = {.msgCb = msgCb, .pTfs = pMgmt->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); @@ -262,7 +268,6 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { SDnode *pDnode = pWrapper->pDnode; SVnodesMgmt *pMgmt = calloc(1, sizeof(SVnodesMgmt)); int32_t code = -1; - SVnodeOpt vnodeOpt = {0}; dInfo("vnodes-mgmt start to init"); if (pMgmt == NULL) goto _OVER; @@ -294,13 +299,7 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { goto _OVER; } - vnodeOpt.nthreads = tsNumOfCommitThreads; - vnodeOpt.putToQueryQFp = vmPutMsgToQueryQueue; - vnodeOpt.putToFetchQFp = vmPutMsgToQueryQueue; - vnodeOpt.sendReqFp = dndSendReqToDnode; - vnodeOpt.sendMnodeReqFp = dndSendReqToMnode; - vnodeOpt.sendRspFp = dndSendRsp; - if (vnodeInit(&vnodeOpt) != 0) { + if (vnodeInit() != 0) { dError("failed to init vnode since %s", terrstr()); goto _OVER; } diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c index 53423845d4..efae47b195 100644 --- a/source/dnode/mgmt/vnode/src/vmMsg.c +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -82,7 +82,14 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return -1; } - vnodeCfg.pWrapper = pMgmt->pWrapper; + SMsgCb msgCb = {0}; + msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; + msgCb.sendReqFp = dndSendReqToDnode; + msgCb.sendMnodeReqFp = dndSendReqToMnode; + msgCb.sendRspFp = dndSendRsp; + + vnodeCfg.msgCb = msgCb; vnodeCfg.pTfs = pMgmt->pTfs; vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 88dd3a5c22..d5bffada6a 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -119,19 +119,12 @@ typedef struct SMnode { SHashObj *infosMeta; SGrantInfo grant; MndMsgFp msgFp[TDMT_MAX]; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - PutToQueueFp putToWriteQFp; - PutToQueueFp putToReadQFp; + SMsgCb msgCb; } SMnode; -int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *rpcMsg); -int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg); -void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); - +void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp); uint64_t mndGenerateUid(char *name, int32_t len); - -void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); +void mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index bf79c89a1c..c44a5aaaa3 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -623,7 +623,7 @@ static int32_t mndProcessConfigDnodeReq(SNodeMsg *pReq) { .msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .ahandle = pReq->rpcMsg.ahandle}; mInfo("dnode:%d, app:%p config:%s req send to dnode", cfgReq.dnodeId, rpcMsg.ahandle, cfgReq.config); - mndSendReqToDnode(pMnode, &epSet, &rpcMsg); + tmsgSendReq(&pMnode->msgCb, &epSet, &rpcMsg); return 0; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 25ec5f7cd4..4fbf0352e5 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -420,7 +420,7 @@ static int32_t mndProcessMqTimerMsg(SNodeMsg *pMsg) { .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg), }; - (*pMnode->putToWriteQFp)(pMnode->pWrapper, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } else { taosHashCleanup(pRebMsg->rebSubHash); rpcFreeCont(pRebMsg); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d3b36aed31..e2a6e49b56 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -888,7 +888,7 @@ static int32_t mndTransSendActionMsg(SMnode *pMnode, STrans *pTrans, SArray *pAr } memcpy(rpcMsg.pCont, pAction->pCont, pAction->contLen); - if (mndSendReqToDnode(pMnode, &pAction->epSet, &rpcMsg) == 0) { + if (tmsgSendReq(&pMnode->msgCb, &pAction->epSet, &rpcMsg) == 0) { mDebug("trans:%d, action:%d is sent", pTrans->id, action); pAction->msgSent = 1; pAction->msgReceived = 0; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 39f2155109..6400bf69f1 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -43,24 +43,6 @@ #define TRNAS_TIMER_MS 6000 #define TELEM_TIMER_MS 86400000 -int32_t mndSendReqToDnode(SMnode *pMnode, SEpSet *pEpSet, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqFp == NULL) { - terrno = TSDB_CODE_MND_NOT_READY; - return -1; - } - - return (*pMnode->sendReqFp)(pMnode->pWrapper, pEpSet, pMsg); -} - -int32_t mndSendReqToMnode(SMnode *pMnode, SRpcMsg *pMsg) { - if (pMnode == NULL || pMnode->sendReqFp == NULL) { - terrno = TSDB_CODE_MND_NOT_READY; - return -1; - } - - return (*pMnode->sendMnodeReqFp)(pMnode->pWrapper, pMsg); -} - static void *mndBuildTimerMsg(int32_t *pContLen) { SMTimerReq timerReq = {0}; @@ -80,7 +62,7 @@ static void mndPullupTrans(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putToWriteQFp(pMnode->pWrapper, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } taosTmrReset(mndPullupTrans, TRNAS_TIMER_MS, pMnode, pMnode->timer, &pMnode->transTimer); @@ -96,7 +78,7 @@ static void mndCalMqRebalance(void *param, void *tmrId) { .pCont = pReq, .contLen = contLen, }; - pMnode->putToReadQFp(pMnode->pWrapper, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, QUERY_QUEUE, &rpcMsg); } taosTmrReset(mndCalMqRebalance, MQ_TIMER_MS, pMnode, pMnode->timer, &pMnode->mqTimer); @@ -108,7 +90,7 @@ static void mndPullupTelem(void *param, void *tmrId) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; - pMnode->putToReadQFp(pMnode->pWrapper, &rpcMsg); + tmsgPutToQueue(&pMnode->msgCb, QUERY_QUEUE, &rpcMsg); } taosTmrReset(mndPullupTelem, TELEM_TIMER_MS, pMnode, pMnode->timer, &pMnode->telemTimer); @@ -286,14 +268,9 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->replica = pOption->replica; pMnode->selfIndex = pOption->selfIndex; memcpy(&pMnode->replicas, pOption->replicas, sizeof(SReplica) * TSDB_MAX_REPLICA); - pMnode->pWrapper = pOption->pWrapper; - pMnode->putToWriteQFp = pOption->putToWriteQFp; - pMnode->putToReadQFp = pOption->putToReadQFp; - pMnode->sendReqFp = pOption->sendReqFp; - pMnode->sendMnodeReqFp = pOption->sendMnodeReqFp; + pMnode->msgCb = pOption->msgCb; - if (pMnode->sendReqFp == NULL || pMnode->sendMnodeReqFp == NULL || - pMnode->putToWriteQFp == NULL || pMnode->dnodeId < 0 || pMnode->clusterId < 0) { + if (pMnode->dnodeId < 0 || pMnode->clusterId < 0) { terrno = TSDB_CODE_MND_INVALID_OPTIONS; return -1; } diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index 357a62052a..168e538ac0 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -32,8 +32,8 @@ typedef struct SQWorkerMgmt SQHandle; typedef struct SQnode { int32_t qndId; - SQnodeOpt opt; - SQHandle* pQuery; + SMsgCb msgCb; + SQHandle* pQuery; } SQnode; #ifdef __cplusplus diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index a257b343c2..37fd4cbed5 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -13,14 +13,10 @@ * along with this program. If not, see . */ +#include "executor.h" #include "qndInt.h" #include "query.h" #include "qworker.h" -#include "executor.h" - -int32_t qnodePutReqToVQueryQ(SQnode* pQnode, struct SRpcMsg* pReq) {} -void qnodeSendReqToDnode(SQnode* pQnode, struct SEpSet* epSet, struct SRpcMsg* pReq) {} - SQnode *qndOpen(const SQnodeOpt *pOption) { SQnode *pQnode = calloc(1, sizeof(SQnode)); @@ -29,12 +25,12 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { return NULL; } - if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, pQnode, - (putReqToQueryQFp)qnodePutReqToVQueryQ, (sendReqFp)qnodeSendReqToDnode)) { + if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { tfree(pQnode); return NULL; } - + + pQnode->msgCb = pOption->msgCb; return pQnode; } diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index d1122fc4ec..519b94cf46 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -44,7 +44,7 @@ typedef struct { typedef struct SSnode { SStreamMeta* pMeta; - SSnodeOpt cfg; + SMsgCb msgCb; } SSnode; SStreamMeta* sndMetaNew(); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 1e9e48e206..b0ce7ad8c5 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -22,7 +22,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { if (pSnode == NULL) { return NULL; } - memcpy(&pSnode->cfg, pOption, sizeof(SSnodeOpt)); + pSnode->msgCb = pOption->msgCb; pSnode->pMeta = sndMetaNew(); if (pSnode->pMeta == NULL) { free(pSnode); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 1ffd4e0d78..67446d15ab 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -18,6 +18,7 @@ #include "os.h" #include "trpc.h" +#include "tmsgcb.h" #include "meta.h" #include "tarray.h" @@ -40,7 +41,6 @@ typedef struct { typedef struct { int32_t vgId; uint64_t dbId; - void *pWrapper; STfs *pTfs; uint64_t wsize; uint64_t ssize; @@ -54,20 +54,12 @@ typedef struct { SMetaCfg metaCfg; STqCfg tqCfg; SWalCfg walCfg; + SMsgCb msgCb; uint32_t hashBegin; uint32_t hashEnd; int8_t hashMethod; } SVnodeCfg; -typedef struct { - uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) - PutToQueueFp putToQueryQFp; - PutToQueueFp putToFetchQFp; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - SendRspFp sendRspFp; -} SVnodeOpt; - typedef struct { int64_t ver; int64_t tbUid; @@ -87,10 +79,9 @@ typedef struct { /** * @brief Initialize the vnode module * - * @param pOption Option of the vnode mnodule * @return int 0 for success and -1 for failure */ -int vnodeInit(const SVnodeOpt *pOption); +int vnodeInit(); /** * @brief Cleanup the vnode module diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index de9b7bac83..c911f95e4a 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -52,12 +52,6 @@ typedef struct SVnodeMgr { TdThreadMutex mutex; TdThreadCond hasTask; TD_DLIST(SVnodeTask) queue; - // For vnode Mgmt - PutToQueueFp putToQueryQFp; - PutToQueueFp putToFetchQFp; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - SendRspFp sendRspFp; } SVnodeMgr; extern SVnodeMgr vnodeMgr; @@ -81,7 +75,7 @@ struct SVnode { SWal* pWal; tsem_t canCommit; SQHandle* pQuery; - void* pWrapper; + SMsgCb msgCb; STfs* pTfs; }; diff --git a/source/dnode/vnode/src/vnd/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c index c7405fdcea..86e670d533 100644 --- a/source/dnode/vnode/src/vnd/vnodeMain.c +++ b/source/dnode/vnode/src/vnd/vnodeMain.c @@ -27,7 +27,7 @@ SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnodeCfg cfg = defaultVnodeOptions; if (pVnodeCfg != NULL) { cfg.vgId = pVnodeCfg->vgId; - cfg.pWrapper = pVnodeCfg->pWrapper; + cfg.msgCb = pVnodeCfg->msgCb; cfg.pTfs = pVnodeCfg->pTfs; cfg.dbId = pVnodeCfg->dbId; cfg.hashBegin = pVnodeCfg->hashBegin; @@ -79,7 +79,7 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { } pVnode->vgId = pVnodeCfg->vgId; - pVnode->pWrapper = pVnodeCfg->pWrapper; + pVnode->msgCb = pVnodeCfg->msgCb; pVnode->pTfs = pVnodeCfg->pTfs; pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index 51aaf9e68f..920b5b0947 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -14,43 +14,33 @@ */ #include "vnd.h" +#include "tglobal.h" SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED}; static void* loop(void* arg); -int vnodeInit(const SVnodeOpt *pOption) { +int vnodeInit() { if (TD_CHECK_AND_SET_MODE_INIT(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_INITIALIZED) { return 0; } vnodeMgr.stop = false; - vnodeMgr.putToQueryQFp = pOption->putToQueryQFp; - vnodeMgr.putToFetchQFp = pOption->putToFetchQFp; - vnodeMgr.sendReqFp = pOption->sendReqFp; - vnodeMgr.sendMnodeReqFp = pOption->sendMnodeReqFp; - vnodeMgr.sendRspFp = pOption->sendRspFp; // Start commit handers - if (pOption->nthreads > 0) { - vnodeMgr.nthreads = pOption->nthreads; - vnodeMgr.threads = (TdThread*)calloc(pOption->nthreads, sizeof(TdThread)); - if (vnodeMgr.threads == NULL) { - return -1; - } + vnodeMgr.nthreads = tsNumOfCommitThreads; + vnodeMgr.threads = calloc(vnodeMgr.nthreads, sizeof(TdThread)); + if (vnodeMgr.threads == NULL) { + return -1; + } - taosThreadMutexInit(&(vnodeMgr.mutex), NULL); - taosThreadCondInit(&(vnodeMgr.hasTask), NULL); - TD_DLIST_INIT(&(vnodeMgr.queue)); + taosThreadMutexInit(&(vnodeMgr.mutex), NULL); + taosThreadCondInit(&(vnodeMgr.hasTask), NULL); + TD_DLIST_INIT(&(vnodeMgr.queue)); - for (uint16_t i = 0; i < pOption->nthreads; i++) { - taosThreadCreate(&(vnodeMgr.threads[i]), NULL, loop, NULL); - // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); - } - } else { - // TODO: if no commit thread is set, then another mechanism should be - // given. Otherwise, it is a false. - ASSERT(0); + for (uint16_t i = 0; i < vnodeMgr.nthreads; i++) { + taosThreadCreate(&(vnodeMgr.threads[i]), NULL, loop, NULL); + // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } if (walInit() < 0) { @@ -92,26 +82,6 @@ int vnodeScheduleTask(SVnodeTask* pTask) { return 0; } -int32_t vnodePutToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { - return (*vnodeMgr.putToQueryQFp)(pVnode->pWrapper, pReq); -} - -int32_t vnodePutToVFetchQ(SVnode* pVnode, struct SRpcMsg* pReq) { - return (*vnodeMgr.putToFetchQFp)(pVnode->pWrapper, pReq); -} - -int32_t vnodeSendReq(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pReq) { - return (*vnodeMgr.sendReqFp)(pVnode->pWrapper, epSet, pReq); -} - -int32_t vnodeSendMnodeReq(SVnode* pVnode, struct SRpcMsg* pReq) { - return (*vnodeMgr.sendMnodeReqFp)(pVnode->pWrapper, pReq); -} - -void vnodeSendRsp(SVnode* pVnode, struct SEpSet* epSet, struct SRpcMsg* pRsp) { - (*vnodeMgr.sendRspFp)(pVnode->pWrapper, pRsp); -} - /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { setThreadName("vnode-commit"); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 3132c675e9..d7af04c462 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -20,8 +20,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg); static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg); int vnodeQueryOpen(SVnode *pVnode) { - return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, pVnode, - (putReqToQueryQFp)vnodePutToVQueryQ, (sendReqFp)vnodeSendReq); + return qWorkerInit(NODE_TYPE_VNODE, pVnode->vgId, NULL, (void **)&pVnode->pQuery, &pVnode->msgCb); } void vnodeQueryClose(SVnode *pVnode) { qWorkerDestroy((void **)&pVnode->pQuery); } diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index f806a7b212..84c7f9194f 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "qworker.h" #include "tlockfree.h" #include "ttimer.h" @@ -139,18 +140,16 @@ typedef struct SQWSchStatus { // Qnode/Vnode level task management typedef struct SQWorkerMgmt { - SQWorkerCfg cfg; - int8_t nodeType; - int32_t nodeId; - void *timer; - tmr_h hbTimer; - SRWLatch schLock; - //SRWLatch ctxLock; - SHashObj *schHash; //key: schedulerId, value: SQWSchStatus - SHashObj *ctxHash; //key: queryId+taskId, value: SQWTaskCtx - void *nodeObj; - putReqToQueryQFp putToQueueFp; - sendReqFp sendReqFp; + SQWorkerCfg cfg; + int8_t nodeType; + int32_t nodeId; + void *timer; + tmr_h hbTimer; + SRWLatch schLock; + // SRWLatch ctxLock; + SHashObj *schHash; // key: schedulerId, value: SQWSchStatus + SHashObj *ctxHash; // key: queryId+taskId, value: SQWTaskCtx + SMsgCb msgCb; } SQWorkerMgmt; #define QW_FPARAMS_DEF SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int64_t rId diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 73611a5ed9..55dbb893d6 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1442,9 +1442,8 @@ _return: taosTmrReset(qwProcessHbTimerEvent, QW_DEFAULT_HEARTBEAT_MSEC, param, mgmt->timer, &mgmt->hbTimer); } -int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, void *nodeObj, - putReqToQueryQFp fp1, sendReqFp fp2) { - if (NULL == qWorkerMgmt || NULL == nodeObj || NULL == fp1 || NULL == fp2) { +int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qWorkerMgmt, const SMsgCb *pMsgCb) { + if (NULL == qWorkerMgmt || pMsgCb->pWrapper == NULL) { qError("invalid param to init qworker"); QW_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1500,9 +1499,7 @@ int32_t qWorkerInit(int8_t nodeType, int32_t nodeId, SQWorkerCfg *cfg, void **qW mgmt->nodeType = nodeType; mgmt->nodeId = nodeId; - mgmt->nodeObj = nodeObj; - mgmt->putToQueueFp = fp1; - mgmt->sendReqFp = fp2; + mgmt->msgCb = *pMsgCb; *qWorkerMgmt = mgmt; diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 7d633d1c73..25107430a7 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -245,15 +245,15 @@ int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, void *connection) { req->taskId = tId; SRpcMsg pNewMsg = { - .handle = pMsg->handle, - .ahandle = pMsg->ahandle, - .msgType = TDMT_VND_QUERY_CONTINUE, - .pCont = req, - .contLen = sizeof(SQueryContinueReq), - .code = 0, + .handle = pMsg->handle, + .ahandle = pMsg->ahandle, + .msgType = TDMT_VND_QUERY_CONTINUE, + .pCont = req, + .contLen = sizeof(SQueryContinueReq), + .code = 0, }; - int32_t code = (*mgmt->putToQueueFp)(mgmt->nodeObj, &pNewMsg); + int32_t code = tmsgPutToQueue(&mgmt->msgCb, QUERY_QUEUE, &pNewMsg); if (TSDB_CODE_SUCCESS != code) { QW_SCH_TASK_ELOG("put query continue msg to queue failed, vgId:%d, code:%s", mgmt->nodeId, tstrerror(code)); rpcFreeCont(req); diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index a45ec41e6c..30c0e2fca2 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -1080,7 +1080,10 @@ TEST(rcTest, shortExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; @@ -1161,7 +1164,10 @@ TEST(rcTest, longExecshortDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 1000000; @@ -1244,7 +1250,10 @@ TEST(rcTest, shortExeclongDelay) { qwtTestStop = false; qwtTestQuitThreadNum = 0; - code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, mockPointer, qwtPutReqToQueue, NULL); + SMsgCb msgCb = {0}; + msgCb.pWrapper = (struct SMgmtWrapper *)mockPointer; + msgCb.queueFps[QUERY_QUEUE] = (PutToQueueFp)qwtPutReqToQueue; + code = qWorkerInit(NODE_TYPE_VNODE, 1, NULL, &mgmt, &msgCb); ASSERT_EQ(code, 0); qwtTestMaxExecTaskUsec = 0; From 48aace92c5ca7949da98fe6c6cb5459faeff9c0e Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 21 Mar 2022 19:26:48 +0800 Subject: [PATCH 04/22] sync refactor --- source/libs/sync/inc/syncInt.h | 10 +- source/libs/sync/src/syncMain.c | 11 ++ source/libs/sync/test/CMakeLists.txt | 11 +- source/libs/sync/test/syncElectTest2.cpp | 135 +++++++++++++++++++++++ 4 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 source/libs/sync/test/syncElectTest2.cpp diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index ee009d6428..eaf4d56f67 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -217,10 +217,12 @@ int32_t syncNodeStartHeartbeatTimer(SSyncNode* pSyncNode); int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode); // utils -------------- -int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); -int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); -cJSON* syncNode2Json(const SSyncNode* pSyncNode); -char* syncNode2Str(const SSyncNode* pSyncNode); +int32_t syncNodeSendMsgById(const SRaftId* destRaftId, SSyncNode* pSyncNode, SRpcMsg* pMsg); +int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, SRpcMsg* pMsg); +cJSON* syncNode2Json(const SSyncNode* pSyncNode); +char* syncNode2Str(const SSyncNode* pSyncNode); +SSyncNode* syncNodeAcquire(int64_t rid); +void syncNodeRelease(SSyncNode* pNode); // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c4adb4762b..facb0180f9 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -575,6 +575,17 @@ char* syncNode2Str(const SSyncNode* pSyncNode) { return serialized; } +SSyncNode* syncNodeAcquire(int64_t rid) { + SSyncNode* pNode = taosAcquireRef(tsNodeRefId, rid); + if (pNode == NULL) { + sTrace("failed to acquire node from refId:%" PRId64, rid); + } + + return pNode; +} + +void syncNodeRelease(SSyncNode* pNode) { taosReleaseRef(tsNodeRefId, pNode->rid); } + // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 3fcb9087ab..f02ba29218 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -28,6 +28,7 @@ add_executable(syncRpcMsgTest "") add_executable(syncPingTimerTest2 "") add_executable(syncPingSelfTest "") add_executable(syncElectTest "") +add_executable(syncElectTest2 "") add_executable(syncEncodeTest "") add_executable(syncWriteTest "") add_executable(syncReplicateTest "") @@ -154,6 +155,10 @@ target_sources(syncElectTest PRIVATE "syncElectTest.cpp" ) +target_sources(syncElectTest2 + PRIVATE + "syncElectTest2.cpp" +) target_sources(syncEncodeTest PRIVATE "syncEncodeTest.cpp" @@ -322,7 +327,7 @@ target_include_directories(syncElectTest "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) -target_include_directories(syncElectTest +target_include_directories(syncElectTest2 PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" @@ -469,6 +474,10 @@ target_link_libraries(syncElectTest sync gtest_main ) +target_link_libraries(syncElectTest2 + sync + gtest_main +) target_link_libraries(syncEncodeTest sync gtest_main diff --git a/source/libs/sync/test/syncElectTest2.cpp b/source/libs/sync/test/syncElectTest2.cpp new file mode 100644 index 0000000000..6113c37fec --- /dev/null +++ b/source/libs/sync/test/syncElectTest2.cpp @@ -0,0 +1,135 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "tref.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; +int32_t replicaNum = 3; +int32_t myIndex = 0; + +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; + +int64_t syncNodeInit() { + syncInfo.vgId = 1234; + syncInfo.rpcClient = gSyncIO->clientRpc; + syncInfo.FpSendMsg = syncIOSendMsg; + syncInfo.queue = gSyncIO->pMsgQ; + syncInfo.FpEqMsg = syncIOEqMsg; + syncInfo.pFsm = pFsm; + snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect_test_%d", myIndex); + + int code = walInit(); + assert(code == 0); + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = syncInfo.vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + + char tmpdir[128]; + snprintf(tmpdir, sizeof(tmpdir), "./elect_test_wal_%d", myIndex); + pWal = walOpen(tmpdir, &walCfg); + assert(pWal != NULL); + + syncInfo.pWal = pWal; + + SSyncCfg* pCfg = &syncInfo.syncCfg; + pCfg->myIndex = myIndex; + pCfg->replicaNum = replicaNum; + + for (int i = 0; i < replicaNum; ++i) { + pCfg->nodeInfo[i].nodePort = ports[i]; + snprintf(pCfg->nodeInfo[i].nodeFqdn, sizeof(pCfg->nodeInfo[i].nodeFqdn), "%s", "127.0.0.1"); + // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); + } + + + int64_t rid = syncStart(&syncInfo); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncRequestVote = pSyncNode->FpOnRequestVote; + gSyncIO->FpOnSyncRequestVoteReply = pSyncNode->FpOnRequestVoteReply; + gSyncIO->FpOnSyncAppendEntries = pSyncNode->FpOnAppendEntries; + gSyncIO->FpOnSyncAppendEntriesReply = pSyncNode->FpOnAppendEntriesReply; + gSyncIO->FpOnSyncPing = pSyncNode->FpOnPing; + gSyncIO->FpOnSyncPingReply = pSyncNode->FpOnPingReply; + gSyncIO->FpOnSyncTimeout = pSyncNode->FpOnTimeout; + gSyncIO->pSyncNode = pSyncNode; + + syncNodeRelease(pSyncNode); + + return rid; +} + +void initRaftId(SSyncNode* pSyncNode) { + for (int i = 0; i < replicaNum; ++i) { + ids[i] = pSyncNode->replicasId[i]; + char* s = syncUtilRaftId2Str(&ids[i]); + printf("raftId[%d] : %s\n", i, s); + free(s); + } +} + +int main(int argc, char** argv) { + // taosInitLog((char *)"syncTest.log", 100000, 10); + tsAsyncLog = 0; + sDebugFlag = 143 + 64; + + myIndex = 0; + if (argc >= 2) { + myIndex = atoi(argv[1]); + } + + int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); + assert(ret == 0); + + ret = syncInit(); + assert(ret == 0); + + int64_t rid = syncNodeInit(); + assert(rid > 0); + + SSyncNode* pSyncNode = (SSyncNode*)syncNodeAcquire(rid); + assert(pSyncNode != NULL); + + syncNodePrint2((char*)"", pSyncNode); + initRaftId(pSyncNode); + + //--------------------------- + while (1) { + sTrace( + "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", + pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, + pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + } + + syncNodeRelease(pSyncNode); + + return 0; +} From 0432a8ec7559a61f4e26ce09c4f9d20462608a59 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Mar 2022 19:49:20 +0800 Subject: [PATCH 05/22] refact query queue --- include/common/tmsgcb.h | 33 +++++---- include/dnode/qnode/qnode.h | 5 +- include/libs/qcom/query.h | 8 +-- source/common/src/tmsgcb.c | 8 +-- source/dnode/mgmt/mnode/src/mmWorker.c | 92 +++++++++++++------------- source/dnode/mgmt/qnode/inc/qmInt.h | 3 + source/dnode/mgmt/qnode/src/qmInt.c | 2 + source/dnode/mgmt/qnode/src/qmWorker.c | 87 +++++++++++++++++------- source/dnode/mgmt/vnode/inc/vmInt.h | 2 - source/dnode/mgmt/vnode/src/vmInt.c | 2 + source/dnode/mgmt/vnode/src/vmMsg.c | 2 + source/dnode/mgmt/vnode/src/vmWorker.c | 36 +++++----- source/dnode/qnode/inc/qndInt.h | 1 + source/dnode/qnode/src/qnode.c | 25 +++---- 14 files changed, 170 insertions(+), 136 deletions(-) diff --git a/include/common/tmsgcb.h b/include/common/tmsgcb.h index 413cb957d2..5ce4b16300 100644 --- a/include/common/tmsgcb.h +++ b/include/common/tmsgcb.h @@ -22,30 +22,29 @@ extern "C" { #endif -struct SRpcMsg; -struct SEpSet; -struct SMgmtWrapper; +typedef struct SRpcMsg SRpcMsg; +typedef struct SEpSet SEpSet; typedef struct SMgmtWrapper SMgmtWrapper; -typedef int32_t (*PutToQueueFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); -typedef int32_t (*SendReqFp)(struct SMgmtWrapper* pWrapper, struct SEpSet* epSet, struct SRpcMsg* pReq); -typedef int32_t (*SendMnodeReqFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pReq); -typedef void (*SendRspFp)(struct SMgmtWrapper* pWrapper, struct SRpcMsg* pRsp); +typedef int32_t (*PutToQueueFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); +typedef int32_t (*SendReqFp)(SMgmtWrapper* pWrapper, SEpSet* epSet, SRpcMsg* pReq); +typedef int32_t (*SendMnodeReqFp)(SMgmtWrapper* pWrapper, SRpcMsg* pReq); +typedef void (*SendRspFp)(SMgmtWrapper* pWrapper, SRpcMsg* pRsp); -typedef enum { QUERY_QUEUE, FETCH_QUEUE, WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, QUEUE_MAX } EMsgQueueType; +typedef enum { QUERY_QUEUE, FETCH_QUEUE, WRITE_QUEUE, APPLY_QUEUE, SYNC_QUEUE, QUEUE_MAX } EQueueType; typedef struct { - struct SMgmtWrapper* pWrapper; - PutToQueueFp queueFps[QUEUE_MAX]; - SendReqFp sendReqFp; - SendMnodeReqFp sendMnodeReqFp; - SendRspFp sendRspFp; + SMgmtWrapper* pWrapper; + PutToQueueFp queueFps[QUEUE_MAX]; + SendReqFp sendReqFp; + SendMnodeReqFp sendMnodeReqFp; + SendRspFp sendRspFp; } SMsgCb; -int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EMsgQueueType qtype, struct SRpcMsg* pReq); -int32_t tmsgSendReq(const SMsgCb* pMsgCb, struct SEpSet* epSet, struct SRpcMsg* pReq); -int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, struct SRpcMsg* pReq); -void tmsgSendRsp(const SMsgCb* pMsgCb, struct SRpcMsg* pRsp); +int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq); +int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq); +int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq); +void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp); #ifdef __cplusplus } diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 67ee509622..89553f978b 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -70,10 +70,9 @@ int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad); * * @param pQnode The qnode object. * @param pMsg The request message - * @param pRsp The response message - * @return int32_t 0 for success, -1 for failure */ -int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp); +int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg); +int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b6f42eeee0..07a5e261a0 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -25,7 +25,7 @@ extern "C" { #include "tlog.h" #include "tmsg.h" -enum { +typedef enum { JOB_TASK_STATUS_NULL = 0, JOB_TASK_STATUS_NOT_START = 1, JOB_TASK_STATUS_EXECUTING, @@ -35,12 +35,12 @@ enum { JOB_TASK_STATUS_CANCELLING, JOB_TASK_STATUS_CANCELLED, JOB_TASK_STATUS_DROPPING, -}; +} EJobTaskType; -enum { +typedef enum { TASK_TYPE_PERSISTENT = 1, TASK_TYPE_TEMP, -}; +} ETaskType; typedef struct STableComInfo { uint8_t numOfTags; // the number of tags in schema diff --git a/source/common/src/tmsgcb.c b/source/common/src/tmsgcb.c index 19973770e4..b13f727db6 100644 --- a/source/common/src/tmsgcb.c +++ b/source/common/src/tmsgcb.c @@ -16,16 +16,16 @@ #define _DEFAULT_SOURCE #include "tmsgcb.h" -int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EMsgQueueType qtype, struct SRpcMsg* pReq) { +int32_t tmsgPutToQueue(const SMsgCb* pMsgCb, EQueueType qtype, SRpcMsg* pReq) { return (*pMsgCb->queueFps[qtype])(pMsgCb->pWrapper, pReq); } -int32_t tmsgSendReq(const SMsgCb* pMsgCb, struct SEpSet* epSet, struct SRpcMsg* pReq) { +int32_t tmsgSendReq(const SMsgCb* pMsgCb, SEpSet* epSet, SRpcMsg* pReq) { return (*pMsgCb->sendReqFp)(pMsgCb->pWrapper, epSet, pReq); } -int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, struct SRpcMsg* pReq) { +int32_t tmsgSendMnodeReq(const SMsgCb* pMsgCb, SRpcMsg* pReq) { return (*pMsgCb->sendMnodeReqFp)(pMsgCb->pWrapper, pReq); } -void tmsgSendRsp(const SMsgCb* pMsgCb, struct SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); } +void tmsgSendRsp(const SMsgCb* pMsgCb, SRpcMsg* pRsp) { return (*pMsgCb->sendRspFp)(pMsgCb->pWrapper, pRsp); } diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c index 4fa2f60b09..ab9f46e323 100644 --- a/source/dnode/mgmt/mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -42,6 +42,52 @@ static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { taosFreeQitem(pMsg); } +static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) { + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); + return dndWriteMsgToWorker(pWorker, pMsg); +} + +int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); +} + +int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); +} + +int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { + return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); +} + +static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) { + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg == NULL) { + return -1; + } + + dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + + int32_t code = dndWriteMsgToWorker(pWorker, pMsg); + if (code != 0) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + return code; +} + +int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc); +} + +int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SMnodeMgmt *pMgmt = pWrapper->pMgmt; + return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc); +} + int32_t mmStartWorker(SMnodeMgmt *pMgmt) { if (dndInitWorker(pMgmt, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, mmProcessQueue) != 0) { dError("failed to start mnode read worker since %s", terrstr()); @@ -66,49 +112,3 @@ void mmStopWorker(SMnodeMgmt *pMgmt) { dndCleanupWorker(&pMgmt->writeWorker); dndCleanupWorker(&pMgmt->syncWorker); } - -static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) { - dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); - return dndWriteMsgToWorker(pWorker, pMsg); -} - -int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { - return mmPutMsgToWorker(pMgmt, &pMgmt->writeWorker, pMsg); -} - -int32_t mmProcessSyncMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { - return mmPutMsgToWorker(pMgmt, &pMgmt->syncWorker, pMsg); -} - -int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { - return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); -} - -static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) { - SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); - if (pMsg == NULL) { - return -1; - } - - dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); - pMsg->rpcMsg = *pRpc; - - int32_t code = mmPutMsgToWorker(pMgmt, pWorker, pMsg); - if (code != 0) { - dTrace("msg:%p, is freed", pMsg); - taosFreeQitem(pMsg); - rpcFreeCont(pRpc->pCont); - } - - return code; -} - -int32_t mmPutMsgToWriteQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - return mmPutRpcMsgToWorker(pMgmt, &pMgmt->writeWorker, pRpc); -} - -int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - SMnodeMgmt *pMgmt = pWrapper->pMgmt; - return mmPutRpcMsgToWorker(pMgmt, &pMgmt->readWorker, pRpc); -} diff --git a/source/dnode/mgmt/qnode/inc/qmInt.h b/source/dnode/mgmt/qnode/inc/qmInt.h index 2629486d60..dc7f10b939 100644 --- a/source/dnode/mgmt/qnode/inc/qmInt.h +++ b/source/dnode/mgmt/qnode/inc/qmInt.h @@ -42,6 +42,9 @@ int32_t qmProcessCreateReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg); // qmWorker.c +int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); +int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); + int32_t qmStartWorker(SQnodeMgmt *pMgmt); void qmStopWorker(SQnodeMgmt *pMgmt); int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg); diff --git a/source/dnode/mgmt/qnode/src/qmInt.c b/source/dnode/mgmt/qnode/src/qmInt.c index 38defd1dc8..40959192c3 100644 --- a/source/dnode/mgmt/qnode/src/qmInt.c +++ b/source/dnode/mgmt/qnode/src/qmInt.c @@ -21,6 +21,8 @@ static int32_t qmRequire(SMgmtWrapper *pWrapper, bool *required) { return dndRea static void qmInitOption(SQnodeMgmt *pMgmt, SQnodeOpt *pOption) { SMsgCb msgCb = {0}; msgCb.pWrapper = pMgmt->pWrapper; + msgCb.queueFps[QUERY_QUEUE] = qmPutMsgToQueryQueue; + msgCb.queueFps[FETCH_QUEUE] = qmPutMsgToFetchQueue; msgCb.sendReqFp = dndSendReqToDnode; msgCb.sendMnodeReqFp = dndSendReqToMnode; msgCb.sendRspFp = dndSendRsp; diff --git a/source/dnode/mgmt/qnode/src/qmWorker.c b/source/dnode/mgmt/qnode/src/qmWorker.c index 6285a4bb74..1a140c69e7 100644 --- a/source/dnode/mgmt/qnode/src/qmWorker.c +++ b/source/dnode/mgmt/qnode/src/qmWorker.c @@ -16,50 +16,87 @@ #define _DEFAULT_SOURCE #include "qmInt.h" -static void qmProcessQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { - dTrace("msg:%p, will be processed in qnode queue", pMsg); - SRpcMsg *pRsp = NULL; - SRpcMsg *pRpc = &pMsg->rpcMsg; - int32_t code = qndProcessMsg(pMgmt->pQnode, pRpc, &pRsp); +static void qmSendRsp(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int32_t code) { + SRpcMsg rsp = {.handle = pMsg->rpcMsg.handle, .ahandle = pMsg->rpcMsg.ahandle, .code = code}; + dndSendRsp(pWrapper, &rsp); +} - if (pRpc->msgType & 1u) { - if (pRsp != NULL) { - pRsp->ahandle = pRpc->ahandle; - dndSendRsp(pMgmt->pWrapper, pRsp); - free(pRsp); - } else { - if (code != 0) code = terrno; - SRpcMsg rpcRsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = code}; - dndSendRsp(pMgmt->pWrapper, &rpcRsp); - } +static void qmProcessQueryQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in qnode-query queue", pMsg); + int32_t code = qndProcessQueryMsg(pMgmt->pQnode, &pMsg->rpcMsg); + if (code != 0) { + qmSendRsp(pMgmt->pWrapper, pMsg, code); } dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - rpcFreeCont(pRpc->pCont); + rpcFreeCont(pMsg->rpcMsg.pCont); taosFreeQitem(pMsg); } -int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { - SDnodeWorker *pWorker = &pMgmt->queryWorker; +static void qmProcessFetchQueue(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { + dTrace("msg:%p, will be processed in qnode-fetch queue", pMsg); + int32_t code = qndProcessFetchMsg(pMgmt->pQnode, &pMsg->rpcMsg); + if (code != 0) { + qmSendRsp(pMgmt->pWrapper, pMsg, code); + } - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); +} + +static int32_t qmPutMsgToWorker(SDnodeWorker *pWorker, SNodeMsg *pMsg) { + dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); return dndWriteMsgToWorker(pWorker, pMsg); } -int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { - SDnodeWorker *pWorker = &pMgmt->fetchWorker; +int32_t qmProcessQueryMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->queryWorker, pMsg); } - dTrace("msg:%p, put into worker:%s", pMsg, pWorker->name); - return dndWriteMsgToWorker(pWorker, pMsg); +int32_t qmProcessFetchMsg(SQnodeMgmt *pMgmt, SNodeMsg *pMsg) { return qmPutMsgToWorker(&pMgmt->fetchWorker, pMsg); } + +static int32_t qmPutRpcMsgToWorker(SQnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) { + SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); + if (pMsg == NULL) { + return -1; + } + + dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); + pMsg->rpcMsg = *pRpc; + + int32_t code = dndWriteMsgToWorker(pWorker, pMsg); + if (code != 0) { + dTrace("msg:%p, is freed", pMsg); + taosFreeQitem(pMsg); + rpcFreeCont(pRpc->pCont); + } + + return code; +} + +int32_t qmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + return qmPutRpcMsgToWorker(pMgmt, &pMgmt->queryWorker, pRpc); +} + +int32_t qmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { + SQnodeMgmt *pMgmt = pWrapper->pMgmt; + return qmPutRpcMsgToWorker(pMgmt, &pMgmt->fetchWorker, pRpc); } int32_t qmStartWorker(SQnodeMgmt *pMgmt) { - if (dndInitWorker(pMgmt, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, qmProcessQueue) != 0) { + int32_t maxFetchThreads = 4; + int32_t minFetchThreads = TMIN(maxFetchThreads, tsNumOfCores); + int32_t minQueryThreads = TMAX((int32_t)(tsNumOfCores * tsRatioOfQueryCores), 1); + int32_t maxQueryThreads = minQueryThreads; + + if (dndInitWorker(pMgmt, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", minQueryThreads, maxQueryThreads, + qmProcessQueryQueue) != 0) { dError("failed to start qnode query worker since %s", terrstr()); return -1; } - if (dndInitWorker(pMgmt, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, qmProcessQueue) != 0) { + if (dndInitWorker(pMgmt, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", minFetchThreads, maxFetchThreads, + qmProcessFetchQueue) != 0) { dError("failed to start qnode fetch worker since %s", terrstr()); return -1; } diff --git a/source/dnode/mgmt/vnode/inc/vmInt.h b/source/dnode/mgmt/vnode/inc/vmInt.h index 2020a3d219..d670e073da 100644 --- a/source/dnode/mgmt/vnode/inc/vmInt.h +++ b/source/dnode/mgmt/vnode/inc/vmInt.h @@ -24,8 +24,6 @@ extern "C" { #endif -typedef enum { VND_WRITE_QUEUE, VND_QUERY_QUEUE, VND_FETCH_QUEUE, VND_APPLY_QUEUE, VND_SYNC_QUEUE } EVndQueueType; - typedef struct SVnodesMgmt { SHashObj *hash; SRWLatch latch; diff --git a/source/dnode/mgmt/vnode/src/vmInt.c b/source/dnode/mgmt/vnode/src/vmInt.c index 12f38d71ca..1c6c7d089e 100644 --- a/source/dnode/mgmt/vnode/src/vmInt.c +++ b/source/dnode/mgmt/vnode/src/vmInt.c @@ -131,6 +131,8 @@ static void *vmOpenVnodeFunc(void *param) { SMsgCb msgCb = {0}; msgCb.pWrapper = pMgmt->pWrapper; msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; + msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; + msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; msgCb.sendReqFp = dndSendReqToDnode; msgCb.sendMnodeReqFp = dndSendReqToMnode; msgCb.sendRspFp = dndSendRsp; diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c index efae47b195..375a83421a 100644 --- a/source/dnode/mgmt/vnode/src/vmMsg.c +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -85,6 +85,8 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SMsgCb msgCb = {0}; msgCb.pWrapper = pMgmt->pWrapper; msgCb.queueFps[QUERY_QUEUE] = vmPutMsgToQueryQueue; + msgCb.queueFps[FETCH_QUEUE] = vmPutMsgToFetchQueue; + msgCb.queueFps[APPLY_QUEUE] = vmPutMsgToApplyQueue; msgCb.sendReqFp = dndSendReqToDnode; msgCb.sendMnodeReqFp = dndSendReqToMnode; msgCb.sendRspFp = dndSendRsp; diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index fe01b19d2d..b0fe8d55e1 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -152,7 +152,7 @@ static void vmProcessSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOf } } -static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EVndQueueType qtype) { +static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueType qtype) { SRpcMsg *pRpc = &pMsg->rpcMsg; int32_t code = -1; @@ -167,20 +167,22 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EVndQueue } switch (qtype) { - case VND_QUERY_QUEUE: + case QUERY_QUEUE: dTrace("msg:%p, will be written into vnode-query queue", pMsg); code = taosWriteQitem(pVnode->pQueryQ, pMsg); break; - case VND_FETCH_QUEUE: + case FETCH_QUEUE: dTrace("msg:%p, will be written into vnode-fetch queue", pMsg); code = taosWriteQitem(pVnode->pFetchQ, pMsg); break; - case VND_WRITE_QUEUE: + case WRITE_QUEUE: dTrace("msg:%p, will be written into vnode-write queue", pMsg); code = taosWriteQitem(pVnode->pWriteQ, pMsg); - case VND_SYNC_QUEUE: + break; + case SYNC_QUEUE: dTrace("msg:%p, will be written into vnode-sync queue", pMsg); code = taosWriteQitem(pVnode->pSyncQ, pMsg); + break; default: terrno = TSDB_CODE_INVALID_PARA; break; @@ -191,19 +193,19 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EVndQueue } int32_t vmProcessSyncMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_SYNC_QUEUE); + return vmPutNodeMsgToQueue(pMgmt, pMsg, SYNC_QUEUE); } int32_t vmProcessWriteMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_WRITE_QUEUE); + return vmPutNodeMsgToQueue(pMgmt, pMsg, WRITE_QUEUE); } int32_t vmProcessQueryMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_QUERY_QUEUE); + return vmPutNodeMsgToQueue(pMgmt, pMsg, QUERY_QUEUE); } int32_t vmProcessFetchMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { - return vmPutNodeMsgToQueue(pMgmt, pMsg, VND_FETCH_QUEUE); + return vmPutNodeMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { @@ -212,7 +214,7 @@ int32_t vmProcessMgmtMsg(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { return dndWriteMsgToWorker(pWorker, pMsg); } -static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EVndQueueType qtype) { +static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EQueueType qtype) { SVnodesMgmt *pMgmt = pWrapper->pMgmt; int32_t code = -1; SMsgHead *pHead = pRpc->pCont; @@ -225,20 +227,18 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EVndQue dTrace("msg:%p, is created, type:%s", pMsg, TMSG_INFO(pRpc->msgType)); pMsg->rpcMsg = *pRpc; switch (qtype) { - case VND_QUERY_QUEUE: + case QUERY_QUEUE: dTrace("msg:%p, will be put into vnode-query queue", pMsg); code = taosWriteQitem(pVnode->pQueryQ, pMsg); break; - case VND_FETCH_QUEUE: + case FETCH_QUEUE: dTrace("msg:%p, will be put into vnode-fetch queue", pMsg); code = taosWriteQitem(pVnode->pFetchQ, pMsg); break; - case VND_APPLY_QUEUE: + case APPLY_QUEUE: dTrace("msg:%p, will be put into vnode-apply queue", pMsg); code = taosWriteQitem(pVnode->pApplyQ, pMsg); break; - case VND_WRITE_QUEUE: - case VND_SYNC_QUEUE: default: terrno = TSDB_CODE_INVALID_PARA; break; @@ -249,15 +249,15 @@ static int32_t vmPutRpcMsgToQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, EVndQue } int32_t vmPutMsgToQueryQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_QUERY_QUEUE); + return vmPutRpcMsgToQueue(pWrapper, pRpc, QUERY_QUEUE); } int32_t vmPutMsgToFetchQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_FETCH_QUEUE); + return vmPutRpcMsgToQueue(pWrapper, pRpc, FETCH_QUEUE); } int32_t vmPutMsgToApplyQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { - return vmPutRpcMsgToQueue(pWrapper, pRpc, VND_APPLY_QUEUE); + return vmPutRpcMsgToQueue(pWrapper, pRpc, APPLY_QUEUE); } int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index 168e538ac0..307bf3efb8 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -21,6 +21,7 @@ #include "tlog.h" #include "tmsg.h" #include "trpc.h" +#include "tmsgcb.h" #include "qnode.h" diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 37fd4cbed5..bc1a602b52 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -34,25 +34,20 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { return pQnode; } -void qndClose(SQnode *pQnode) { +void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); - free(pQnode); + free(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } -int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - *pRsp = NULL; - return 0; -} - -int qnodeProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { +int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { qTrace("message in query queue is processing"); SReadHandle handle = {0}; switch (pMsg->msgType) { - case TDMT_VND_QUERY:{ + case TDMT_VND_QUERY: { return qWorkerProcessQueryMsg(&handle, pQnode->pQuery, pMsg); } case TDMT_VND_QUERY_CONTINUE: @@ -63,7 +58,7 @@ int qnodeProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { } } -int qnodeProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { +int32_t qndProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { qTrace("message in fetch queue is processing"); switch (pMsg->msgType) { case TDMT_VND_FETCH: @@ -81,17 +76,13 @@ int qnodeProcessFetchMsg(SQnode *pQnode, SRpcMsg *pMsg) { case TDMT_VND_SHOW_TABLES: return qWorkerProcessShowMsg(pQnode, pQnode->pQuery, pMsg); case TDMT_VND_SHOW_TABLES_FETCH: - //return vnodeGetTableList(pQnode, pMsg); + // return vnodeGetTableList(pQnode, pMsg); case TDMT_VND_TABLE_META: - //return vnodeGetTableMeta(pQnode, pMsg); + // return vnodeGetTableMeta(pQnode, pMsg); case TDMT_VND_CONSUME: - //return tqProcessConsumeReq(pQnode->pTq, pMsg); + // return tqProcessConsumeReq(pQnode->pTq, pMsg); default: qError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; } } - - - - From 3139aaef0502033d98a2e4266059bd20a700b3d5 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Mon, 21 Mar 2022 20:06:57 +0800 Subject: [PATCH 06/22] sync refactor --- source/libs/sync/inc/syncIO.h | 2 +- source/libs/sync/test/syncElectTest.cpp | 1 + source/libs/sync/test/syncElectTest2.cpp | 14 +++++++------- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/sync/inc/syncIO.h b/source/libs/sync/inc/syncIO.h index 0185d5f211..99f9deb99e 100644 --- a/source/libs/sync/inc/syncIO.h +++ b/source/libs/sync/inc/syncIO.h @@ -35,7 +35,7 @@ extern "C" { typedef struct SSyncIO { STaosQueue *pMsgQ; STaosQset * pQset; - TdThread consumerTid; + TdThread consumerTid; void * serverRpc; void * clientRpc; diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 47f55aa660..99cfe75867 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -120,6 +120,7 @@ int main(int argc, char** argv) { "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", gSyncNode->state, syncUtilState2String(gSyncNode->state), gSyncNode->pRaftStore->currentTerm, gSyncNode->electTimerLogicClock, gSyncNode->electTimerLogicClockUser, gSyncNode->electTimerMS); + taosMsleep(1000); } return 0; diff --git a/source/libs/sync/test/syncElectTest2.cpp b/source/libs/sync/test/syncElectTest2.cpp index 6113c37fec..8463ffd448 100644 --- a/source/libs/sync/test/syncElectTest2.cpp +++ b/source/libs/sync/test/syncElectTest2.cpp @@ -21,10 +21,10 @@ uint16_t ports[] = {7010, 7110, 7210, 7310, 7410}; int32_t replicaNum = 3; int32_t myIndex = 0; -SRaftId ids[TSDB_MAX_REPLICA]; -SSyncInfo syncInfo; -SSyncFSM* pFsm; -SWal* pWal; +SRaftId ids[TSDB_MAX_REPLICA]; +SSyncInfo syncInfo; +SSyncFSM* pFsm; +SWal* pWal; int64_t syncNodeInit() { syncInfo.vgId = 1234; @@ -33,7 +33,7 @@ int64_t syncNodeInit() { syncInfo.queue = gSyncIO->pMsgQ; syncInfo.FpEqMsg = syncIOEqMsg; syncInfo.pFsm = pFsm; - snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect_test_%d", myIndex); + snprintf(syncInfo.path, sizeof(syncInfo.path), "./elect2_test_%d", myIndex); int code = walInit(); assert(code == 0); @@ -48,7 +48,7 @@ int64_t syncNodeInit() { walCfg.level = TAOS_WAL_FSYNC; char tmpdir[128]; - snprintf(tmpdir, sizeof(tmpdir), "./elect_test_wal_%d", myIndex); + snprintf(tmpdir, sizeof(tmpdir), "./elect2_test_wal_%d", myIndex); pWal = walOpen(tmpdir, &walCfg); assert(pWal != NULL); @@ -64,7 +64,6 @@ int64_t syncNodeInit() { // taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn); } - int64_t rid = syncStart(&syncInfo); assert(rid > 0); @@ -127,6 +126,7 @@ int main(int argc, char** argv) { "elect sleep, state: %d, %s, term:%lu electTimerLogicClock:%lu, electTimerLogicClockUser:%lu, electTimerMS:%d", pSyncNode->state, syncUtilState2String(pSyncNode->state), pSyncNode->pRaftStore->currentTerm, pSyncNode->electTimerLogicClock, pSyncNode->electTimerLogicClockUser, pSyncNode->electTimerMS); + taosMsleep(1000); } syncNodeRelease(pSyncNode); From 4c999f74ae663a70b9b1ad568d0e684ada1289fd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Mar 2022 20:52:17 +0800 Subject: [PATCH 07/22] Enabling qnodes to process messages --- source/dnode/mgmt/container/inc/dnd.h | 6 +- source/dnode/mgmt/container/src/dndInt.c | 3 +- .../dnode/mgmt/container/src/dndTransport.c | 54 +++++-- source/dnode/mgmt/dnode/src/dmMsg.c | 26 ++-- source/dnode/mgmt/mnode/src/mmMsg.c | 140 +++++++++--------- source/dnode/mgmt/qnode/src/qmMsg.c | 14 +- source/dnode/mgmt/snode/src/smMsg.c | 4 +- source/dnode/mgmt/vnode/src/vmMsg.c | 74 ++++----- source/dnode/mgmt/vnode/src/vmWorker.c | 2 +- 9 files changed, 184 insertions(+), 139 deletions(-) diff --git a/source/dnode/mgmt/container/inc/dnd.h b/source/dnode/mgmt/container/inc/dnd.h index 29c05d6d01..f9fa896f42 100644 --- a/source/dnode/mgmt/container/inc/dnd.h +++ b/source/dnode/mgmt/container/inc/dnd.h @@ -89,6 +89,9 @@ typedef struct { } SDnodeWorker; typedef struct SMsgHandle { + int32_t vgId; + NodeMsgFp vgIdMsgFp; + SMgmtWrapper *pVgIdWrapper; // Handle the case where the same message type is distributed to qnode or vnode NodeMsgFp msgFp; SMgmtWrapper *pWrapper; } SMsgHandle; @@ -114,6 +117,7 @@ typedef struct SMgmtWrapper { void *pMgmt; SDnode *pDnode; NodeMsgFp msgFps[TDMT_MAX]; + int32_t msgVgIds[TDMT_MAX]; // Handle the case where the same message type is distributed to qnode or vnode SMgmtFp fp; } SMgmtWrapper; @@ -149,7 +153,7 @@ typedef struct SDnode { EDndStatus dndGetStatus(SDnode *pDnode); void dndSetStatus(SDnode *pDnode, EDndStatus stat); SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType nodeType); -void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp); +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId); void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); void dndSendMonitorReport(SDnode *pDnode); diff --git a/source/dnode/mgmt/container/src/dndInt.c b/source/dnode/mgmt/container/src/dndInt.c index 4decc6ba87..8ad4351a88 100644 --- a/source/dnode/mgmt/container/src/dndInt.c +++ b/source/dnode/mgmt/container/src/dndInt.c @@ -65,8 +65,9 @@ void dndCleanup() { dInfo("dnode env is cleaned up"); } -void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp) { +void dndSetMsgHandle(SMgmtWrapper *pWrapper, int32_t msgType, NodeMsgFp nodeMsgFp, int32_t vgId) { pWrapper->msgFps[TMSG_INDEX(msgType)] = nodeMsgFp; + pWrapper->msgVgIds[TMSG_INDEX(msgType)] = vgId; } EDndStatus dndGetStatus(SDnode *pDnode) { return pDnode->status; } diff --git a/source/dnode/mgmt/container/src/dndTransport.c b/source/dnode/mgmt/container/src/dndTransport.c index 2593a762e8..20069d89d6 100644 --- a/source/dnode/mgmt/container/src/dndTransport.c +++ b/source/dnode/mgmt/container/src/dndTransport.c @@ -20,13 +20,26 @@ #define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" +static inline void dndProcessQVnodeRpcMsg(SMsgHandle *pHandle, SRpcMsg *pMsg, SEpSet *pEpSet) { + SMsgHead *pHead = pMsg->pCont; + int32_t vgId = htonl(pHead->vgId); + + SMgmtWrapper *pWrapper = pHandle->pWrapper; + if (vgId == pHandle->vgId && pHandle->pVgIdWrapper != NULL) { + pWrapper = pHandle->pVgIdWrapper; + } + + dTrace("msg:%s will be processed by %s, handle:%p app:%p vgId:%d", TMSG_INFO(pMsg->msgType), pWrapper->name, + pMsg->handle, pMsg->ahandle, vgId); + dndProcessRpcMsg(pWrapper, pMsg, pEpSet); +} + static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { SDnode *pDnode = parent; STransMgmt *pMgmt = &pDnode->trans; tmsg_t msgType = pRsp->msgType; if (dndGetStatus(pDnode) != DND_STAT_RUNNING) { - // if (pRsp == NULL || pRsp->pCont == NULL) return; dTrace("rsp:%s ignored since dnode not running, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); rpcFreeCont(pRsp->pCont); return; @@ -34,9 +47,13 @@ static void dndProcessResponse(void *parent, SRpcMsg *pRsp, SEpSet *pEpSet) { SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; if (pHandle->msgFp != NULL) { - dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), - pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); - dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); + if (pHandle->vgId == 0) { + dTrace("rsp:%s will be processed by %s, handle:%p app:%p code:0x%04x:%s", TMSG_INFO(msgType), + pHandle->pWrapper->name, pRsp->handle, pRsp->ahandle, pRsp->code & 0XFFFF, tstrerror(pRsp->code)); + dndProcessRpcMsg(pHandle->pWrapper, pRsp, pEpSet); + } else { + dndProcessQVnodeRpcMsg(pHandle, pRsp, pEpSet); + } } else { dError("rsp:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pRsp->handle, pRsp->ahandle); rpcFreeCont(pRsp->pCont); @@ -110,9 +127,13 @@ static void dndProcessRequest(void *param, SRpcMsg *pReq, SEpSet *pEpSet) { SMsgHandle *pHandle = &pMgmt->msgHandles[TMSG_INDEX(msgType)]; if (pHandle->msgFp != NULL) { - dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, - pReq->handle, pReq->ahandle); - dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); + if (pHandle->vgId == 0) { + dTrace("req:%s will be processed by %s, handle:%p app:%p", TMSG_INFO(msgType), pHandle->pWrapper->name, + pReq->handle, pReq->ahandle); + dndProcessRpcMsg(pHandle->pWrapper, pReq, pEpSet); + } else { + dndProcessQVnodeRpcMsg(pHandle, pReq, pEpSet); + } } else { dError("req:%s not processed since no handle, handle:%p app:%p", TMSG_INFO(msgType), pReq->handle, pReq->ahandle); SRpcMsg rspMsg = {.handle = pReq->handle, .code = TSDB_CODE_MSG_NOT_PROCESSED, .ahandle = pReq->ahandle}; @@ -245,17 +266,24 @@ int32_t dndInitMsgHandle(SDnode *pDnode) { for (int32_t msgIndex = 0; msgIndex < TDMT_MAX; ++msgIndex) { NodeMsgFp msgFp = pWrapper->msgFps[msgIndex]; + int32_t vgId = pWrapper->msgVgIds[msgIndex]; if (msgFp == NULL) continue; SMsgHandle *pHandle = &pMgmt->msgHandles[msgIndex]; - if (pHandle->msgFp != NULL) { - dError("msg:%s has multiple process nodes, prev node:%s, curr node:%s", tMsgInfo[msgIndex], - pHandle->pWrapper->name, pWrapper->name); + if (pHandle->msgFp != NULL && pHandle->vgId == vgId) { + dError("msg:%s has multiple process nodes, prev node:%s:%d, curr node:%s:%d", tMsgInfo[msgIndex], + pHandle->pWrapper->name, pHandle->pWrapper->msgVgIds[msgIndex], pWrapper->name, vgId); return -1; } else { - dTrace("msg:%s will be processed by %s", tMsgInfo[msgIndex], pWrapper->name); - pHandle->msgFp = msgFp; - pHandle->pWrapper = pWrapper; + dTrace("msg:%s will be processed by %s, vgId:%d", tMsgInfo[msgIndex], pWrapper->name, vgId); + if (vgId == 0) { + pHandle->msgFp = msgFp; + pHandle->pWrapper = pWrapper; + } else { + pHandle->vgId = vgId; + pHandle->vgIdMsgFp = msgFp; + pHandle->pVgIdWrapper = pWrapper; + } } } } diff --git a/source/dnode/mgmt/dnode/src/dmMsg.c b/source/dnode/mgmt/dnode/src/dmMsg.c index f451c2ab05..7876d854ea 100644 --- a/source/dnode/mgmt/dnode/src/dmMsg.c +++ b/source/dnode/mgmt/dnode/src/dmMsg.c @@ -114,19 +114,19 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SNodeMsg *pMsg) { void dmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_NETWORK_TEST, (NodeMsgFp)dmProcessMgmtMsg, 0); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH_RSP, (NodeMsgFp)dmProcessMgmtMsg, 0); } diff --git a/source/dnode/mgmt/mnode/src/mmMsg.c b/source/dnode/mgmt/mnode/src/mmMsg.c index 8dbea36c22..333c4954ba 100644 --- a/source/dnode/mgmt/mnode/src/mmMsg.c +++ b/source/dnode/mgmt/mnode/src/mmMsg.c @@ -75,78 +75,78 @@ int32_t mmProcessAlterReq(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { void mmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by DNODE - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_MNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_QNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_SNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_BNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_CONFIG_DNODE_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); // Requests handled by MNODE - dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg); + dndSetMsgHandle(pWrapper, TDMT_MND_CONNECT, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_ACCT, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_USER, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_USER, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_USER, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_USER_AUTH, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CONFIG_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_BNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_DB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_DB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_USE_DB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_DB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_SYNC_DB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_COMPACT_DB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_RETRIEVE_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_FUNC, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_STB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_STB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_STB, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_TABLE_META, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_VGROUP_LIST, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_QUERY, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_CONN, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_HEARTBEAT, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_SHOW_RETRIEVE, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_STATUS, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_KILL_TRANS, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_GRANT, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_AUTH, (NodeMsgFp)mmProcessReadMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_MNODE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_CREATE_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_ALTER_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_DROP_TOPIC, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_SUBSCRIBE, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_MQ_COMMIT_OFFSET, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_MND_GET_SUB_EP, (NodeMsgFp)mmProcessReadMsg, 0); // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)mmProcessWriteMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB_RSP, (NodeMsgFp)mmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_DEPLOY, (NodeMsgFp)mmProcessWriteMsg, 0); } diff --git a/source/dnode/mgmt/qnode/src/qmMsg.c b/source/dnode/mgmt/qnode/src/qmMsg.c index fa7d42b3e3..ddcd791b7b 100644 --- a/source/dnode/mgmt/qnode/src/qmMsg.c +++ b/source/dnode/mgmt/qnode/src/qmMsg.c @@ -54,4 +54,16 @@ int32_t qmProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { } } -void qmInitMsgHandles(SMgmtWrapper *pWrapper) {} +void qmInitMsgHandles(SMgmtWrapper *pWrapper) { + // Requests handled by VNODE + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)qmProcessQueryMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)qmProcessQueryMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)qmProcessFetchMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)qmProcessFetchMsg, 1); + + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)qmProcessFetchMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)qmProcessFetchMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)qmProcessFetchMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)qmProcessFetchMsg, 1); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)qmProcessFetchMsg, 1); +} diff --git a/source/dnode/mgmt/snode/src/smMsg.c b/source/dnode/mgmt/snode/src/smMsg.c index 1bff5597bf..0986556207 100644 --- a/source/dnode/mgmt/snode/src/smMsg.c +++ b/source/dnode/mgmt/snode/src/smMsg.c @@ -56,6 +56,6 @@ int32_t smProcessDropReq(SMgmtWrapper *pWrapper, SNodeMsg *pMsg) { void smInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by SNODE - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_DEPLOY, (NodeMsgFp)smProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_SND_TASK_EXEC, (NodeMsgFp)smProcessExecMsg, 0); } diff --git a/source/dnode/mgmt/vnode/src/vmMsg.c b/source/dnode/mgmt/vnode/src/vmMsg.c index 375a83421a..9f86351985 100644 --- a/source/dnode/mgmt/vnode/src/vmMsg.c +++ b/source/dnode/mgmt/vnode/src/vmMsg.c @@ -243,42 +243,42 @@ int32_t vmProcessCompactVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { void vmInitMsgHandles(SMgmtWrapper *pWrapper) { // Requests handled by VNODE - dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg); - dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg); + dndSetMsgHandle(pWrapper, TDMT_VND_SUBMIT, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY, (NodeMsgFp)vmProcessQueryMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_CONTINUE, (NodeMsgFp)vmProcessQueryMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_FETCH_RSP, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_UPDATE_TAG_VAL, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLE_META, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_TABLES_META, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONSUME, (NodeMsgFp)vmProcessQueryMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_QUERY, (NodeMsgFp)vmProcessQueryMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_CONNECT, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_DISCONNECT, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_RES_READY, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_TASKS_STATUS, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CANCEL_TASK, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TASK, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_STB, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_STB, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_STB, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CREATE_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_ALTER_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_DROP_TABLE, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_SHOW_TABLES_FETCH, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CONN, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_REB, (NodeMsgFp)vmProcessWriteMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_MQ_SET_CUR, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_CONSUME, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_QUERY_HEARTBEAT, (NodeMsgFp)vmProcessFetchMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_VND_TASK_EXEC, (NodeMsgFp)vmProcessFetchMsg, 0); - dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg); - dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg); + dndSetMsgHandle(pWrapper, TDMT_DND_CREATE_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_ALTER_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_DROP_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_SYNC_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); + dndSetMsgHandle(pWrapper, TDMT_DND_COMPACT_VNODE, (NodeMsgFp)vmProcessMgmtMsg, 0); } diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index b0fe8d55e1..52ee624103 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -162,7 +162,7 @@ static int32_t vmPutNodeMsgToQueue(SVnodesMgmt *pMgmt, SNodeMsg *pMsg, EQueueTyp SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dError("vgId:%d, failed to write msg:%p to queue since %s", pHead->vgId, pMsg, terrstr()); + dError("vgId:%d, failed to write msg:%p to vnode-queue since %s", pHead->vgId, pMsg, terrstr()); return -1; } From 8a20b2fe500dd12b79d0218afa85c8eb532f3b9f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 21 Mar 2022 21:39:16 +0800 Subject: [PATCH 08/22] fix memory error --- include/common/tcommon.h | 59 +++++++-------- include/common/tmsg.h | 50 ++++++------- source/client/src/tmq.c | 84 ++++++++++++---------- source/dnode/mgmt/vnode/src/vmWorker.c | 5 +- source/dnode/mnode/impl/src/mndSubscribe.c | 17 +++-- source/dnode/vnode/src/tq/tq.c | 5 +- source/dnode/vnode/src/vnd/vnodeInt.c | 4 +- 7 files changed, 116 insertions(+), 108 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 385c123fec..9c36856f1d 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -54,25 +54,28 @@ typedef struct SColumnDataAgg { } SColumnDataAgg; typedef struct SDataBlockInfo { - STimeWindow window; - int32_t rows; - int32_t rowSize; - int16_t numOfCols; - int16_t hasVarCol; - union {int64_t uid; int64_t blockId;}; + STimeWindow window; + int32_t rows; + int32_t rowSize; + int16_t numOfCols; + int16_t hasVarCol; + union { + int64_t uid; + int64_t blockId; + }; } SDataBlockInfo; -//typedef struct SConstantItem { -// SColumnInfo info; -// int32_t startRow; // run-length-encoding to save the space for multiple rows -// int32_t endRow; -// SVariant value; -//} SConstantItem; +// typedef struct SConstantItem { +// SColumnInfo info; +// int32_t startRow; // run-length-encoding to save the space for multiple rows +// int32_t endRow; +// SVariant value; +// } SConstantItem; // info.numOfCols = taosArrayGetSize(pDataBlock) + taosArrayGetSize(pConstantList); typedef struct SSDataBlock { - SColumnDataAgg *pBlockAgg; - SArray *pDataBlock; // SArray + SColumnDataAgg* pBlockAgg; + SArray* pDataBlock; // SArray SDataBlockInfo info; } SSDataBlock; @@ -108,13 +111,13 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { int32_t tlen = 0; int32_t sz = 0; - tlen += taosEncodeFixedI64(buf, pRsp->consumerId); + // tlen += taosEncodeFixedI64(buf, pRsp->consumerId); tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); tlen += taosEncodeFixedI32(buf, pRsp->skipLogNum); tlen += taosEncodeFixedI32(buf, pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return tlen; - tlen += tEncodeSSchemaWrapper(buf, pRsp->schemas); + tlen += tEncodeSSchemaWrapper(buf, pRsp->schema); if (pRsp->pBlockData) { sz = taosArrayGetSize(pRsp->pBlockData); } @@ -128,15 +131,15 @@ static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { int32_t sz; - buf = taosDecodeFixedI64(buf, &pRsp->consumerId); + // buf = taosDecodeFixedI64(buf, &pRsp->consumerId); buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); buf = taosDecodeFixedI32(buf, &pRsp->skipLogNum); buf = taosDecodeFixedI32(buf, &pRsp->numOfTopics); if (pRsp->numOfTopics == 0) return buf; - pRsp->schemas = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); - if (pRsp->schemas == NULL) return NULL; - buf = tDecodeSSchemaWrapper(buf, pRsp->schemas); + pRsp->schema = (SSchemaWrapper*)calloc(1, sizeof(SSchemaWrapper)); + if (pRsp->schema == NULL) return NULL; + buf = tDecodeSSchemaWrapper(buf, pRsp->schema); buf = taosDecodeFixedI32(buf, &sz); pRsp->pBlockData = taosArrayInit(sz, sizeof(SSDataBlock)); for (int32_t i = 0; i < sz; i++) { @@ -148,11 +151,11 @@ static FORCE_INLINE void* tDecodeSMqPollRsp(void* buf, SMqPollRsp* pRsp) { } static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { - if (pRsp->schemas) { - if (pRsp->schemas->nCols) { - tfree(pRsp->schemas->pSchema); + if (pRsp->schema) { + if (pRsp->schema->nCols) { + tfree(pRsp->schema->pSchema); } - free(pRsp->schemas); + free(pRsp->schema); } taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); pRsp->pBlockData = NULL; @@ -196,7 +199,7 @@ typedef struct SGroupbyExpr { typedef struct SFunctParam { int32_t type; - SColumn *pCol; + SColumn* pCol; SVariant param; } SFunctParam; @@ -214,12 +217,12 @@ typedef struct SResSchame { typedef struct SExprBasicInfo { SResSchema resSchema; int16_t numOfParams; // argument value of each function - SFunctParam *pParam; + SFunctParam* pParam; } SExprBasicInfo; typedef struct SExprInfo { - struct SExprBasicInfo base; - struct tExprNode *pExpr; + struct SExprBasicInfo base; + struct tExprNode* pExpr; } SExprInfo; typedef struct SStateWindow { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 54081cab67..58e991c294 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1282,7 +1282,7 @@ static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { if (pRebSub == NULL) { goto _err; } - pRebSub->key = key; + pRebSub->key = strdup(key); pRebSub->lostConsumers = taosArrayInit(0, sizeof(int64_t)); if (pRebSub->lostConsumers == NULL) { goto _err; @@ -2116,25 +2116,16 @@ typedef struct { int8_t mqMsgType; int32_t code; int32_t epoch; + int64_t consumerId; } SMqRspHead; -typedef struct { - int64_t consumerId; - SSchemaWrapper* schemas; - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t numOfTopics; - SArray* pBlockData; // SArray -} SMqPollRsp; - -// one req for one vg+topic typedef struct { SMsgHead head; int64_t consumerId; int64_t blockingTime; int32_t epoch; + int8_t withSchema; char cgroup[TSDB_CGROUP_LEN]; int64_t currentOffset; @@ -2153,19 +2144,21 @@ typedef struct { } SMqSubTopicEp; typedef struct { - int64_t consumerId; - char cgroup[TSDB_CGROUP_LEN]; - SArray* topics; // SArray -} SMqCMGetSubEpRsp; + SMqRspHead head; + // TODO: remove from msg + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + int32_t numOfTopics; + SSchemaWrapper* schema; + SArray* pBlockData; // SArray +} SMqPollRsp; typedef struct { SMqRspHead head; - union { - SMqPollRsp consumeRsp; - SMqCMGetSubEpRsp getEpRsp; - }; - void* extra; -} SMqMsgWrapper; + char cgroup[TSDB_CGROUP_LEN]; + SArray* topics; // SArray +} SMqCMGetSubEpRsp; typedef struct { int32_t curBlock; @@ -2173,11 +2166,13 @@ typedef struct { void** uData; } SMqRowIter; -struct tmq_message_t_v1 { - SMqPollRsp rsp; +struct tmq_message_t { + SMqPollRsp msg; + void* vg; SMqRowIter iter; }; +#if 0 struct tmq_message_t { SMqRspHead head; union { @@ -2189,6 +2184,7 @@ struct tmq_message_t { int32_t curRow; void** uData; }; +#endif static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } @@ -2241,8 +2237,7 @@ static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicE static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSubEpRsp* pRsp) { int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->consumerId); - tlen += taosEncodeString(buf, pRsp->cgroup); + // tlen += taosEncodeString(buf, pRsp->cgroup); int32_t sz = taosArrayGetSize(pRsp->topics); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { @@ -2253,8 +2248,7 @@ static FORCE_INLINE int32_t tEncodeSMqCMGetSubEpRsp(void** buf, const SMqCMGetSu } static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->consumerId); - buf = taosDecodeStringTo(buf, pRsp->cgroup); + // buf = taosDecodeStringTo(buf, pRsp->cgroup); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); pRsp->topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index a8fc394720..4cb39ef4f8 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -681,7 +681,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { if (tmq_message == NULL) return 0; - SMqPollRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->msg; return pRsp->skipLogNum; } @@ -690,15 +690,15 @@ void tmqShowMsg(tmq_message_t* tmq_message) { static bool noPrintSchema; char pBuf[128]; - SMqPollRsp* pRsp = &tmq_message->consumeRsp; - int32_t colNum = pRsp->schemas->nCols; + SMqPollRsp* pRsp = &tmq_message->msg; + int32_t colNum = pRsp->schema->nCols; if (!noPrintSchema) { printf("|"); for (int32_t i = 0; i < colNum; i++) { if (i == 0) - printf(" %25s |", pRsp->schemas->pSchema[i].name); + printf(" %25s |", pRsp->schema->pSchema[i].name); else - printf(" %15s |", pRsp->schemas->pSchema[i].name); + printf(" %15s |", pRsp->schema->pSchema[i].name); } printf("\n"); printf("===============================================\n"); @@ -778,19 +778,19 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { goto WRITE_QUEUE_FAIL; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->consumeRsp); - pRsp->curBlock = 0; - pRsp->curRow = 0; + tDecodeSMqPollRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->msg); + pRsp->iter.curBlock = 0; + pRsp->iter.curRow = 0; // TODO: alloc mem /*pRsp->*/ /*printf("rsp commit off:%ld rsp off:%ld has data:%d\n", pRsp->committedOffset, pRsp->rspOffset, pRsp->numOfTopics);*/ - if (pRsp->consumeRsp.numOfTopics == 0) { + if (pRsp->msg.numOfTopics == 0) { /*printf("no data\n");*/ taosFreeQitem(pRsp); goto WRITE_QUEUE_FAIL; } - pRsp->extra = pParam->pVg; + pRsp->vg = pParam->pVg; taosWriteQitem(tmq->mqueue, pRsp); atomic_add_fetch_32(&tmq->readyRequest, 1); tsem_post(&tmq->rspSem); @@ -860,14 +860,14 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { } tDeleteSMqCMGetSubEpRsp(&rsp); } else { - tmq_message_t* pRsp = taosAllocateQitem(sizeof(tmq_message_t)); + SMqCMGetSubEpRsp* pRsp = taosAllocateQitem(sizeof(SMqCMGetSubEpRsp)); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; goto END; } memcpy(pRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRsp->getEpRsp); + tDecodeSMqCMGetSubEpRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pRsp); taosWriteQitem(tmq->mqueue, pRsp); tsem_post(&tmq->rspSem); @@ -983,6 +983,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t blockingTime, SMqClientTo return pReq; } +#if 0 tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { tmq_message_t* msg = NULL; for (int i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { @@ -1050,6 +1051,7 @@ tmq_message_t* tmqSyncPollImpl(tmq_t* tmq, int64_t blockingTime) { } return NULL; } +#endif int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { /*printf("call poll\n");*/ @@ -1111,11 +1113,12 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t blockingTime) { } // return -int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { - if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__EP_RSP) { +int32_t tmqHandleRes(tmq_t* tmq, SMqRspHead* rspHead, bool* pReset) { + if (rspHead->mqMsgType == TMQ_MSG_TYPE__EP_RSP) { /*printf("ep %d %d\n", rspMsg->head.epoch, tmq->epoch);*/ - if (rspMsg->head.epoch > atomic_load_32(&tmq->epoch)) { - tmqUpdateEp(tmq, rspMsg->head.epoch, &rspMsg->getEpRsp); + if (rspHead->epoch > atomic_load_32(&tmq->epoch)) { + SMqCMGetSubEpRsp* rspMsg = (SMqCMGetSubEpRsp*)rspHead; + tmqUpdateEp(tmq, rspHead->epoch, rspMsg); tmqClearUnhandleMsg(tmq); *pReset = true; } else { @@ -1129,21 +1132,22 @@ int32_t tmqHandleRes(tmq_t* tmq, tmq_message_t* rspMsg, bool* pReset) { tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfReset) { while (1) { - tmq_message_t* rspMsg = NULL; - taosGetQitem(tmq->qall, (void**)&rspMsg); - if (rspMsg == NULL) { + SMqRspHead* rspHead = NULL; + taosGetQitem(tmq->qall, (void**)&rspHead); + if (rspHead == NULL) { taosReadAllQitems(tmq->mqueue, tmq->qall); - taosGetQitem(tmq->qall, (void**)&rspMsg); - if (rspMsg == NULL) return NULL; + taosGetQitem(tmq->qall, (void**)&rspHead); + if (rspHead == NULL) return NULL; } - if (rspMsg->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { + if (rspHead->mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { + tmq_message_t* rspMsg = (tmq_message_t*)rspHead; atomic_sub_fetch_32(&tmq->readyRequest, 1); /*printf("handle poll rsp %d\n", rspMsg->head.mqMsgType);*/ - if (rspMsg->head.epoch == atomic_load_32(&tmq->epoch)) { + if (rspMsg->msg.head.epoch == atomic_load_32(&tmq->epoch)) { /*printf("epoch match\n");*/ - SMqClientVg* pVg = rspMsg->extra; - pVg->currentOffset = rspMsg->consumeRsp.rspOffset; + SMqClientVg* pVg = rspMsg->vg; + pVg->currentOffset = rspMsg->msg.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); return rspMsg; } else { @@ -1153,8 +1157,8 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } else { /*printf("handle ep rsp %d\n", rspMsg->head.mqMsgType);*/ bool reset = false; - tmqHandleRes(tmq, rspMsg, &reset); - taosFreeQitem(rspMsg); + tmqHandleRes(tmq, rspHead, &reset); + taosFreeQitem(rspHead); if (pollIfReset && reset) { printf("reset and repoll\n"); tmqPollImpl(tmq, blockingTime); @@ -1163,6 +1167,7 @@ tmq_message_t* tmqHandleAllRsp(tmq_t* tmq, int64_t blockingTime, bool pollIfRese } } +#if 0 tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* rspMsg = NULL; int64_t startTime = taosGetTimestampMs(); @@ -1185,6 +1190,7 @@ tmq_message_t* tmq_consumer_poll_v1(tmq_t* tmq, int64_t blocking_time) { return NULL; } } +#endif tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { tmq_message_t* rspMsg; @@ -1350,7 +1356,7 @@ tmq_resp_err_t tmq_commit(tmq_t* tmq, const tmq_topic_vgroup_list_t* tmq_topic_v void tmq_message_destroy(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; - SMqPollRsp* pRsp = &tmq_message->consumeRsp; + SMqPollRsp* pRsp = &tmq_message->msg; tDeleteSMqConsumeRsp(pRsp); /*free(tmq_message);*/ taosFreeQitem(tmq_message); @@ -1366,24 +1372,24 @@ const char* tmq_err2str(tmq_resp_err_t err) { } TAOS_ROW tmq_get_row(tmq_message_t* message) { - SMqPollRsp* rsp = &message->consumeRsp; + SMqPollRsp* rsp = &message->msg; while (1) { - if (message->curBlock < taosArrayGetSize(rsp->pBlockData)) { - SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->curBlock); - if (message->curRow < pBlock->info.rows) { + if (message->iter.curBlock < taosArrayGetSize(rsp->pBlockData)) { + SSDataBlock* pBlock = taosArrayGet(rsp->pBlockData, message->iter.curBlock); + if (message->iter.curRow < pBlock->info.rows) { for (int i = 0; i < pBlock->info.numOfCols; i++) { SColumnInfoData* pData = taosArrayGet(pBlock->pDataBlock, i); - if (colDataIsNull_s(pData, message->curRow)) - message->uData[i] = NULL; + if (colDataIsNull_s(pData, message->iter.curRow)) + message->iter.uData[i] = NULL; else { - message->uData[i] = colDataGetData(pData, message->curRow); + message->iter.uData[i] = colDataGetData(pData, message->iter.curRow); } } - message->curRow++; - return message->uData; + message->iter.curRow++; + return message->iter.uData; } else { - message->curBlock++; - message->curRow = 0; + message->iter.curBlock++; + message->iter.curRow = 0; continue; } } diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index fe01b19d2d..61aafab8a2 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -77,8 +77,9 @@ static void vmProcessFetchQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { } dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - rpcFreeCont(pMsg->rpcMsg.pCont); - taosFreeQitem(pMsg); + // TODO: handle invalid write + /*rpcFreeCont(pMsg->rpcMsg.pCont);*/ + /*taosFreeQitem(pMsg);*/ } static void vmProcessWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 25ec5f7cd4..ead8d6805b 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -272,7 +272,6 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { /*sdbWrite(pMnode->pSdb, pConsumerRaw);*/ strcpy(rsp.cgroup, pReq->cgroup); - rsp.consumerId = consumerId; if (epoch != pConsumer->epoch) { mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, pConsumer->epoch); SArray *pTopics = pConsumer->currentTopics; @@ -322,6 +321,7 @@ static int32_t mndProcessGetSubEpReq(SNodeMsg *pMsg) { } ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; ((SMqRspHead *)buf)->epoch = pConsumer->epoch; + ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqCMGetSubEpRsp(&abuf, &rsp); @@ -344,14 +344,14 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) } static SMqRebSubscribe *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { - SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key)); + SMqRebSubscribe *pRebSub = taosHashGet(pHash, key, strlen(key) + 1); if (pRebSub == NULL) { pRebSub = tNewSMqRebSubscribe(key); if (pRebSub == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - taosHashPut(pHash, key, strlen(key), pRebSub, sizeof(SMqRebSubscribe)); + taosHashPut(pHash, key, strlen(key) + 1, pRebSub, sizeof(SMqRebSubscribe)); } return pRebSub; } @@ -441,6 +441,7 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { if (pIter == NULL) break; SMqRebSubscribe *pRebSub = (SMqRebSubscribe *)pIter; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebSub->key); + tfree(pRebSub->key); mInfo("mq rebalance subscription: %s", pSub->key); @@ -503,7 +504,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__IDLE); } - mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status, pRebConsumer->status); + mInfo("mq consumer:%" PRId64 ", status change from %d to %d", pRebConsumer->consumerId, status, + pRebConsumer->status); SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pRebConsumer); sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); @@ -543,8 +545,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); mndReleaseTopic(pMnode, pTopic); } else { - mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", pConsumerEp->vgId, - pConsumerEp->oldConsumerId, pConsumerEp->consumerId); + mInfo("mq rebalance: assign vgroup %d, from consumer %" PRId64 " to consumer %" PRId64 "", + pConsumerEp->vgId, pConsumerEp->oldConsumerId, pConsumerEp->consumerId); mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); } @@ -1099,7 +1101,8 @@ static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, newTopicName); bool createSub = false; if (pSub == NULL) { - mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup, newTopicName); + mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup, + newTopicName); pSub = mndCreateSubscription(pMnode, pTopic, cgroup); createSub = true; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 02fecb49b7..d9337d8edc 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -245,7 +245,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } SMqPollRsp rsp = { - .consumerId = consumerId, + /*.consumerId = consumerId,*/ .numOfTopics = 0, .pBlockData = NULL, }; @@ -298,7 +298,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } taosArrayPush(pRes, pDataBlock); - rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; + rsp.schema = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; rsp.rspOffset = fetchOffset; rsp.numOfTopics = 1; @@ -312,6 +312,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = consumerId; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqPollRsp(&abuf, &rsp); diff --git a/source/dnode/vnode/src/vnd/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c index 7d0b594e95..a64e834ff8 100644 --- a/source/dnode/vnode/src/vnd/vnodeInt.c +++ b/source/dnode/vnode/src/vnd/vnodeInt.c @@ -14,8 +14,8 @@ */ #define _DEFAULT_SOURCE -#include "vnd.h" #include "sync.h" +#include "vnd.h" // #include "vnodeInt.h" int32_t vnodeAlter(SVnode *pVnode, const SVnodeCfg *pCfg) { return 0; } @@ -41,6 +41,6 @@ int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { } int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - vInfo("sync message is processed"); + /*vInfo("sync message is processed");*/ return 0; } From b08c3120bfbfab9d5a3867d046030d072dddafb3 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 21 Mar 2022 23:36:21 +0800 Subject: [PATCH 09/22] fix memory error --- include/common/tcommon.h | 24 +- include/common/tmsg.h | 9 +- source/client/src/tmq.c | 2 +- source/common/src/tdatablock.c | 373 +++++++++++---------- source/dnode/mnode/impl/src/mndSubscribe.c | 10 +- source/dnode/mnode/impl/src/mndTopic.c | 8 +- source/dnode/vnode/src/tq/tq.c | 2 +- tests/test/c/tmqDemo.c | 2 +- 8 files changed, 232 insertions(+), 198 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9c36856f1d..5308be72bb 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -101,13 +101,27 @@ void* blockDataDestroy(SSDataBlock* pBlock); int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); -static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { - if (pBlock == NULL) { - return; +static FORCE_INLINE void blockDestroyInner(SSDataBlock* pBlock) { + // WARNING: do not use info.numOfCols, + // sometimes info.numOfCols != array size + int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + tfree(pColInfoData->varmeta.offset); + } else { + tfree(pColInfoData->nullbitmap); + } + + tfree(pColInfoData->pData); } - blockDataDestroy(pBlock); + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); } +static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { blockDestroyInner(pBlock); } + static FORCE_INLINE int32_t tEncodeSMqPollRsp(void** buf, const SMqPollRsp* pRsp) { int32_t tlen = 0; int32_t sz = 0; @@ -157,7 +171,7 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqPollRsp* pRsp) { } free(pRsp->schema); } - taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); + taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))blockDestroyInner); pRsp->pBlockData = NULL; } diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 58e991c294..c7e37b1613 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2145,11 +2145,12 @@ typedef struct { typedef struct { SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int32_t skipLogNum; + // TODO: replace with topic name + int32_t numOfTopics; // TODO: remove from msg - int64_t reqOffset; - int64_t rspOffset; - int32_t skipLogNum; - int32_t numOfTopics; SSchemaWrapper* schema; SArray* pBlockData; // SArray } SMqPollRsp; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 4cb39ef4f8..7c35b7c149 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -735,7 +735,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqClientVg* pVg = pParam->pVg; tmq_t* tmq = pParam->tmq; if (code != 0) { - printf("msg discard\n"); + printf("msg discard %x\n", code); goto WRITE_QUEUE_FAIL; } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 31328cc4b2..0d48d7cc14 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -18,14 +18,14 @@ #include "tcompare.h" #include "tglobal.h" -int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { +int32_t taosGetFqdnPortFromEp(const char* ep, SEp* pEp) { pEp->port = 0; strcpy(pEp->fqdn, ep); - char *temp = strchr(pEp->fqdn, ':'); + char* temp = strchr(pEp->fqdn, ':'); if (temp) { *temp = 0; - pEp->port = atoi(temp+1); + pEp->port = atoi(temp + 1); } if (pEp->port == 0) { @@ -36,7 +36,7 @@ int32_t taosGetFqdnPortFromEp(const char *ep, SEp* pEp) { return 0; } -void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) { +void addEpIntoEpSet(SEpSet* pEpSet, const char* fqdn, uint16_t port) { if (pEpSet == NULL || fqdn == NULL || strlen(fqdn) == 0) { return; } @@ -47,26 +47,25 @@ void addEpIntoEpSet(SEpSet *pEpSet, const char* fqdn, uint16_t port) { pEpSet->numOfEps += 1; } -bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2) { +bool isEpsetEqual(const SEpSet* s1, const SEpSet* s2) { if (s1->numOfEps != s2->numOfEps || s1->inUse != s2->inUse) { return false; } for (int32_t i = 0; i < s1->numOfEps; i++) { - if (s1->eps[i].port != s2->eps[i].port - || strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0) + if (s1->eps[i].port != s2->eps[i].port || strncmp(s1->eps[i].fqdn, s2->eps[i].fqdn, TSDB_FQDN_LEN) != 0) return false; } return true; } -void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) { +void updateEpSet_s(SCorEpSet* pEpSet, SEpSet* pNewEpSet) { taosCorBeginWrite(&pEpSet->version); pEpSet->epSet = *pNewEpSet; taosCorEndWrite(&pEpSet->version); } -SEpSet getEpSet_s(SCorEpSet *pEpSet) { +SEpSet getEpSet_s(SCorEpSet* pEpSet) { SEpSet ep = {0}; taosCorBeginRead(&pEpSet->version); ep = pEpSet->epSet; @@ -75,7 +74,6 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { return ep; } - int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -95,7 +93,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con if (isNull) { // There is a placehold for each NULL value of binary or nchar type. if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. + pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. } else { colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); } @@ -113,7 +111,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con newSize = 8; } - while(newSize < pAttr->length + varDataTLen(pData)) { + while (newSize < pAttr->length + varDataTLen(pData)) { newSize = newSize * 1.5; } @@ -133,19 +131,40 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con pColumnInfoData->varmeta.length += varDataTLen(pData); } else { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; - switch(type) { - case TSDB_DATA_TYPE_BOOL: {*(bool*) p = *(bool*) pData;break;} + switch (type) { + case TSDB_DATA_TYPE_BOOL: { + *(bool*)p = *(bool*)pData; + break; + } case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} + case TSDB_DATA_TYPE_UTINYINT: { + *(int8_t*)p = *(int8_t*)pData; + break; + } case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: {*(int16_t*) p = *(int16_t*) pData;break;} + case TSDB_DATA_TYPE_USMALLINT: { + *(int16_t*)p = *(int16_t*)pData; + break; + } case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: {*(int32_t*) p = *(int32_t*) pData;break;} + case TSDB_DATA_TYPE_UINT: { + *(int32_t*)p = *(int32_t*)pData; + break; + } case TSDB_DATA_TYPE_TIMESTAMP: case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: {*(int64_t*) p = *(int64_t*) pData;break;} - case TSDB_DATA_TYPE_FLOAT: {*(float*) p = *(float*) pData;break;} - case TSDB_DATA_TYPE_DOUBLE: {*(double*) p = *(double*) pData;break;} + case TSDB_DATA_TYPE_UBIGINT: { + *(int64_t*)p = *(int64_t*)pData; + break; + } + case TSDB_DATA_TYPE_FLOAT: { + *(float*)p = *(float*)pData; + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + *(double*)p = *(double*)pData; + break; + } default: assert(0); } @@ -154,7 +173,8 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con return 0; } -static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, int32_t numOfRow2) { +static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, + int32_t numOfRow2) { uint32_t total = numOfRow1 + numOfRow2; if (BitmapLen(numOfRow1) < BitmapLen(total)) { @@ -188,7 +208,8 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2) { +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, + uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { @@ -202,8 +223,8 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, co // TODO } - pColumnInfoData->varmeta.offset = (int32_t*) p; - for(int32_t i = 0; i < numOfRow2; ++i) { + pColumnInfoData->varmeta.offset = (int32_t*)p; + for (int32_t i = 0; i < numOfRow2; ++i) { pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length; } @@ -244,9 +265,7 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return pBlock->info.numOfCols; } -size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { - return pBlock->info.rows; -} +size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { @@ -263,8 +282,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { } ASSERT(pColInfoData->nullbitmap == NULL); - pDataBlock->info.window.skey = *(TSKEY*) colDataGetData(pColInfoData, 0); - pDataBlock->info.window.ekey = *(TSKEY*) colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); + pDataBlock->info.window.skey = *(TSKEY*)colDataGetData(pColInfoData, 0); + pDataBlock->info.window.ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); return 0; } @@ -272,7 +291,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); int32_t numOfCols = pSrc->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); @@ -280,7 +299,7 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { uint32_t newLen = colDataGetLength(pCol1, pSrc->info.rows); int32_t newSize = oldLen + newLen; - char* tmp = realloc(pCol2->pData, newSize); + char* tmp = realloc(pCol2->pData, newSize); if (tmp != NULL) { pCol2->pData = tmp; colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); @@ -296,10 +315,10 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { size_t blockDataGetSize(const SSDataBlock* pBlock) { assert(pBlock != NULL); - size_t total = 0; + size_t total = 0; int32_t numOfCols = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); total += colDataGetLength(pColInfoData, pBlock->info.rows); @@ -315,7 +334,8 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) { // the number of tuples can be fit in one page. // Actual data rows pluses the corresponding meta data must fit in one memory buffer of the given page size. -int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize) { +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, + int32_t pageSize) { ASSERT(pBlock != NULL && stopIndex != NULL); int32_t numOfCols = pBlock->info.numOfCols; @@ -323,13 +343,13 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t bitmapChar = 1; - size_t headerSize = sizeof(int32_t); + size_t headerSize = sizeof(int32_t); size_t colHeaderSize = sizeof(int32_t) * numOfCols; - size_t payloadSize = pageSize - (headerSize + colHeaderSize); + size_t payloadSize = pageSize - (headerSize + colHeaderSize); // TODO speedup by checking if the whole page can fit in firstly. if (!hasVarCol) { - size_t rowSize = blockDataGetRowSize(pBlock); + size_t rowSize = blockDataGetRowSize(pBlock); int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8; *stopIndex = startIndex + capacity; @@ -342,7 +362,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd // iterate the rows that can be fit in this buffer page int32_t size = (headerSize + colHeaderSize); - for(int32_t j = startIndex; j < numOfRows; ++j) { + for (int32_t j = startIndex; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { @@ -359,7 +379,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd size += pColInfoData->info.bytes; if (((j - startIndex) & 0x07) == 0) { - size += 1; // the space for null bitmap + size += 1; // the space for null bitmap } } } @@ -393,8 +413,8 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 pDst->info.rows = 0; pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); colInfo.info = pSrcCol->info; @@ -414,7 +434,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { - bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); + bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); char* p = colDataGetData(pColData, j); colDataAppend(pDstCol, j - startIndex, p, isNull); @@ -440,14 +460,14 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); // write the number of rows - *(uint32_t*) buf = pBlock->info.rows; + *(uint32_t*)buf = pBlock->info.rows; int32_t numOfCols = pBlock->info.numOfCols; int32_t numOfRows = pBlock->info.rows; char* pStart = buf + sizeof(uint32_t); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pCol->info.type)) { memcpy(pStart, pCol->varmeta.offset, numOfRows * sizeof(int32_t)); @@ -459,7 +479,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { uint32_t dataSize = colDataGetLength(pCol, numOfRows); - *(int32_t*) pStart = dataSize; + *(int32_t*)pStart = dataSize; pStart += sizeof(int32_t); memcpy(pStart, pCol->pData, dataSize); @@ -470,12 +490,12 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { } int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { - pBlock->info.rows = *(int32_t*) buf; + pBlock->info.rows = *(int32_t*)buf; - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = pBlock->info.numOfCols; const char* pStart = buf + sizeof(uint32_t); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); size_t metaSize = pBlock->info.rows * sizeof(int32_t); @@ -487,7 +507,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { pStart += BitmapLen(pBlock->info.rows); } - int32_t colLength = *(int32_t*) pStart; + int32_t colLength = *(int32_t*)pStart; pStart += sizeof(int32_t); if (IS_VAR_DATA_TYPE(pCol->info.type)) { @@ -517,7 +537,7 @@ size_t blockDataGetRowSize(const SSDataBlock* pBlock) { size_t rowSize = 0; size_t numOfCols = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; } @@ -537,10 +557,10 @@ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { SSchema* blockDataExtractSchema(const SSDataBlock* pBlock, int32_t* numOfCols) { SSchema* pSchema = calloc(pBlock->info.numOfCols, sizeof(SSchema)); - for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pSchema[i].bytes = pColInfoData->info.bytes; - pSchema[i].type = pColInfoData->info.type; + pSchema[i].type = pColInfoData->info.type; pSchema[i].colId = pColInfoData->info.colId; } @@ -556,14 +576,14 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { double rowSize = 0; size_t numOfCols = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { rowSize += sizeof(int32_t); } else { - rowSize += 1/8.0; + rowSize += 1 / 8.0; } } @@ -571,56 +591,56 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { } typedef struct SSDataBlockSortHelper { - SArray *orderInfo; // SArray - SSDataBlock *pDataBlock; + SArray* orderInfo; // SArray + SSDataBlock* pDataBlock; bool nullFirst; } SSDataBlockSortHelper; int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { - const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; + const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; SSDataBlock* pDataBlock = pHelper->pDataBlock; - int32_t left = *(int32_t*) p1; - int32_t right = *(int32_t*) p2; + int32_t left = *(int32_t*)p1; + int32_t right = *(int32_t*)p2; SArray* pInfo = pHelper->orderInfo; - for(int32_t i = 0; i < pInfo->size; ++i) { + for (int32_t i = 0; i < pInfo->size; ++i) { SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); - SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); + SColumnInfoData* pColInfoData = pOrder->pColData; // TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); if (pColInfoData->hasNull) { - bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); + bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); if (leftNull && rightNull) { - continue; // continue to next slot + continue; // continue to next slot } if (rightNull) { - return pHelper->nullFirst? 1:-1; + return pHelper->nullFirst ? 1 : -1; } if (leftNull) { - return pHelper->nullFirst? -1:1; + return pHelper->nullFirst ? -1 : 1; } } - void* left1 = colDataGetData(pColInfoData, left); + void* left1 = colDataGetData(pColInfoData, left); void* right1 = colDataGetData(pColInfoData, right); - switch(pColInfoData->info.type) { + switch (pColInfoData->info.type) { case TSDB_DATA_TYPE_INT: { - int32_t leftx = *(int32_t*) left1; - int32_t rightx = *(int32_t*) right1; + int32_t leftx = *(int32_t*)left1; + int32_t rightx = *(int32_t*)right1; if (leftx == rightx) { break; } else { if (pOrder->order == TSDB_ORDER_ASC) { - return (leftx < rightx)? -1:1; + return (leftx < rightx) ? -1 : 1; } else { - return (leftx < rightx)? 1:-1; + return (leftx < rightx) ? 1 : -1; } } } @@ -632,7 +652,8 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { return 0; } -static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { +static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, + int32_t tupleIndex) { int32_t code = 0; int32_t numOfCols = pSrcBlock->info.numOfCols; @@ -666,17 +687,17 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } #else - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pSrc->info.type)) { - memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); - pDst->varmeta.length = pSrc->varmeta.length; + memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); + pDst->varmeta.length = pSrc->varmeta.length; - for(int32_t j = 0; j < pDataBlock->info.rows; ++j) { - pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; - } + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; + } } else { switch (pSrc->info.type) { case TSDB_DATA_TYPE_UINT: @@ -749,7 +770,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { return NULL; } - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); pCols[i].info = pColInfoData->info; @@ -771,7 +792,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { int32_t numOfCols = pDataBlock->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); pColInfoData->info = pCols[i].info; @@ -796,31 +817,39 @@ static int32_t* createTupleIndex(size_t rows) { return NULL; } - for(int32_t i = 0; i < rows; ++i) { + for (int32_t i = 0; i < rows; ++i) { index[i] = i; } return index; } -static void destroyTupleIndex(int32_t* index) { - tfree(index); -} +static void destroyTupleIndex(int32_t* index) { tfree(index); } static __compar_fn_t getComparFn(int32_t type, int32_t order) { - switch(type) { - case TSDB_DATA_TYPE_TINYINT: return order == TSDB_ORDER_ASC? compareInt8Val:compareInt8ValDesc; - case TSDB_DATA_TYPE_SMALLINT: return order == TSDB_ORDER_ASC? compareInt16Val:compareInt16ValDesc; - case TSDB_DATA_TYPE_INT: return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; - case TSDB_DATA_TYPE_BIGINT: return order == TSDB_ORDER_ASC? compareInt64Val:compareInt64ValDesc; - case TSDB_DATA_TYPE_FLOAT: return order == TSDB_ORDER_ASC? compareFloatVal:compareFloatValDesc; - case TSDB_DATA_TYPE_DOUBLE: return order == TSDB_ORDER_ASC? compareDoubleVal:compareDoubleValDesc; - case TSDB_DATA_TYPE_UTINYINT: return order == TSDB_ORDER_ASC? compareUint8Val:compareUint8ValDesc; - case TSDB_DATA_TYPE_USMALLINT:return order == TSDB_ORDER_ASC? compareUint16Val:compareUint16ValDesc; - case TSDB_DATA_TYPE_UINT: return order == TSDB_ORDER_ASC? compareUint32Val:compareUint32ValDesc; - case TSDB_DATA_TYPE_UBIGINT: return order == TSDB_ORDER_ASC? compareUint64Val:compareUint64ValDesc; + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + return order == TSDB_ORDER_ASC ? compareInt8Val : compareInt8ValDesc; + case TSDB_DATA_TYPE_SMALLINT: + return order == TSDB_ORDER_ASC ? compareInt16Val : compareInt16ValDesc; + case TSDB_DATA_TYPE_INT: + return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc; + case TSDB_DATA_TYPE_BIGINT: + return order == TSDB_ORDER_ASC ? compareInt64Val : compareInt64ValDesc; + case TSDB_DATA_TYPE_FLOAT: + return order == TSDB_ORDER_ASC ? compareFloatVal : compareFloatValDesc; + case TSDB_DATA_TYPE_DOUBLE: + return order == TSDB_ORDER_ASC ? compareDoubleVal : compareDoubleValDesc; + case TSDB_DATA_TYPE_UTINYINT: + return order == TSDB_ORDER_ASC ? compareUint8Val : compareUint8ValDesc; + case TSDB_DATA_TYPE_USMALLINT: + return order == TSDB_ORDER_ASC ? compareUint16Val : compareUint16ValDesc; + case TSDB_DATA_TYPE_UINT: + return order == TSDB_ORDER_ASC ? compareUint32Val : compareUint32ValDesc; + case TSDB_DATA_TYPE_UBIGINT: + return order == TSDB_ORDER_ASC ? compareUint64Val : compareUint64ValDesc; default: - return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; + return order == TSDB_ORDER_ASC ? compareInt32Val : compareInt32ValDesc; } } @@ -865,10 +894,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs return TSDB_CODE_SUCCESS; } else { // var data type - } } else if (pDataBlock->info.numOfCols == 2) { - } } @@ -881,7 +908,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs int64_t p0 = taosGetTimestampUs(); SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; - for(int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i); pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex); } @@ -909,7 +936,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1-p0, p2 - p1, p3 - p2, p4-p3, rows); + printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, + p3 - p2, p4 - p3, rows); destroyTupleIndex(index); return TSDB_CODE_SUCCESS; @@ -917,14 +945,18 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirs typedef struct SHelper { int32_t index; - union {char *pData; int64_t i64; double d64;}; + union { + char* pData; + int64_t i64; + double d64; + }; } SHelper; SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* pBlock) { int32_t sortValLengthPerRow = 0; int32_t numOfCols = taosArrayGetSize(pOrderInfo); - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); pInfo->pColData = pColInfo; @@ -933,19 +965,20 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* size_t len = sortValLengthPerRow * pBlock->info.rows; - char* buf = calloc(1, len); + char* buf = calloc(1, len); SHelper* phelper = calloc(numOfRows, sizeof(SHelper)); - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { phelper[i].index = i; phelper[i].pData = buf + sortValLengthPerRow * i; } int32_t offset = 0; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); - for(int32_t j = 0; j < numOfRows; ++j) { - phelper[j].i64 = *(int32_t*) pInfo->pColData->pData + pInfo->pColData->info.bytes * j; -// memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, pInfo->pColData->info.bytes); + for (int32_t j = 0; j < numOfRows; ++j) { + phelper[j].i64 = *(int32_t*)pInfo->pColData->pData + pInfo->pColData->info.bytes * j; + // memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, + // pInfo->pColData->info.bytes); } offset += pInfo->pColData->info.bytes; @@ -955,70 +988,68 @@ SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* } int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { - const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; + const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*)param; -// SSDataBlock* pDataBlock = pHelper->pDataBlock; + // SSDataBlock* pDataBlock = pHelper->pDataBlock; - SHelper* left = (SHelper*) p1; - SHelper* right = (SHelper*) p2; + SHelper* left = (SHelper*)p1; + SHelper* right = (SHelper*)p2; SArray* pInfo = pHelper->orderInfo; int32_t offset = 0; -// for(int32_t i = 0; i < pInfo->size; ++i) { -// SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); -// SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); + // for(int32_t i = 0; i < pInfo->size; ++i) { + // SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); + // SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); -// if (pColInfoData->hasNull) { -// bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); -// bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); -// if (leftNull && rightNull) { -// continue; // continue to next slot -// } -// -// if (rightNull) { -// return pHelper->nullFirst? 1:-1; -// } -// -// if (leftNull) { -// return pHelper->nullFirst? -1:1; -// } -// } + // if (pColInfoData->hasNull) { + // bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); + // bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); + // if (leftNull && rightNull) { + // continue; // continue to next slot + // } + // + // if (rightNull) { + // return pHelper->nullFirst? 1:-1; + // } + // + // if (leftNull) { + // return pHelper->nullFirst? -1:1; + // } + // } -// void* left1 = colDataGetData(pColInfoData, left); -// void* right1 = colDataGetData(pColInfoData, right); + // void* left1 = colDataGetData(pColInfoData, left); + // void* right1 = colDataGetData(pColInfoData, right); -// switch(pColInfoData->info.type) { -// case TSDB_DATA_TYPE_INT: { - int32_t leftx = *(int32_t*)left->pData;//*(int32_t*)(left->pData + offset); - int32_t rightx = *(int32_t*)right->pData;//*(int32_t*)(right->pData + offset); + // switch(pColInfoData->info.type) { + // case TSDB_DATA_TYPE_INT: { + int32_t leftx = *(int32_t*)left->pData; //*(int32_t*)(left->pData + offset); + int32_t rightx = *(int32_t*)right->pData; //*(int32_t*)(right->pData + offset); -// offset += pColInfoData->info.bytes; - if (leftx == rightx) { -// break; - return 0; - } else { -// if (pOrder->order == TSDB_ORDER_ASC) { - return (leftx < rightx)? -1:1; -// } else { -// return (leftx < rightx)? 1:-1; -// } - } -// } -// default: -// assert(0); -// } -// } + // offset += pColInfoData->info.bytes; + if (leftx == rightx) { + // break; + return 0; + } else { + // if (pOrder->order == TSDB_ORDER_ASC) { + return (leftx < rightx) ? -1 : 1; + // } else { + // return (leftx < rightx)? 1:-1; + // } + } + // } + // default: + // assert(0); + // } + // } return 0; } -int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { - return 0; -} +int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { return 0; } int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { -// Allocate the additional buffer. + // Allocate the additional buffer. int64_t p0 = taosGetTimestampUs(); SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; @@ -1032,7 +1063,7 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF taosqsort(index, rows, sizeof(SHelper), &helper, dataBlockCompar_rv); - int64_t p1 = taosGetTimestampUs(); + int64_t p1 = taosGetTimestampUs(); SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); if (pCols == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1052,7 +1083,8 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF copyBackToBlock(pDataBlock, pCols); int64_t p4 = taosGetTimestampUs(); - printf("sort:%" PRId64 ", create:%" PRId64", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); + printf("sort:%" PRId64 ", create:%" PRId64 ", assign:%" PRId64 ", copyback:%" PRId64 ", rows:%d\n", p1 - p0, p2 - p1, + p3 - p2, p4 - p3, rows); // destroyTupleIndex(index); return 0; } @@ -1110,8 +1142,8 @@ int32_t blockDataEnsureColumnCapacity(SColumnInfoData* pColumn, uint32_t numOfRo int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; - - for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); code = blockDataEnsureColumnCapacity(p, numOfRows); if (code) { @@ -1127,21 +1159,8 @@ void* blockDataDestroy(SSDataBlock* pBlock) { return NULL; } - int32_t numOfOutput = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - tfree(pColInfoData->varmeta.offset); - } else { - tfree(pColInfoData->nullbitmap); - } - - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); - // tfree(pBlock); + blockDestroyInner(pBlock); + tfree(pBlock); return NULL; } @@ -1154,7 +1173,7 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock) { pBlock->info.numOfCols = numOfCols; pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfo.info = p->info; @@ -1230,4 +1249,4 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { taosArrayPush(pBlock->pDataBlock, &data); } return (void*)buf; -} \ No newline at end of file +} diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index ead8d6805b..8b52f30474 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -337,9 +337,9 @@ static int32_t mndSplitSubscribeKey(const char *key, char *topic, char *cgroup) while (key[i] != TMQ_SEPARATOR) { i++; } - memcpy(topic, key, i - 1); - topic[i] = 0; - strcpy(cgroup, &key[i + 1]); + memcpy(cgroup, key, i); + cgroup[i] = 0; + strcpy(topic, &key[i + 1]); return 0; } @@ -539,8 +539,8 @@ static int32_t mndProcessDoRebalanceMsg(SNodeMsg *pMsg) { mndSplitSubscribeKey(pSub->key, topic, cgroup); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, topic, - pConsumerEp->consumerId); + mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 " cgroup: %s", pConsumerEp->vgId, + topic, pConsumerEp->consumerId, cgroup); mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); mndReleaseTopic(pMnode, pTopic); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e94ae4f8ec..de3a686d15 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -241,12 +241,12 @@ static int32_t mndGetPlanString(SCMCreateTopicReq *pCreate, char **pStr) { return TSDB_CODE_SUCCESS; } - SNode* pAst = NULL; + SNode *pAst = NULL; int32_t code = nodesStringToNode(pCreate->ast, &pAst); - SQueryPlan* pPlan = NULL; + SQueryPlan *pPlan = NULL; if (TSDB_CODE_SUCCESS == code) { - SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = true }; + SPlanContext cxt = {.pAstRoot = pAst, .topicQuery = true}; code = qCreateQueryPlan(&cxt, &pPlan, NULL); } @@ -274,7 +274,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq topicObj.logicalPlan = ""; topicObj.sqlLen = strlen(pCreate->sql); - char* pPlanStr = NULL; + char *pPlanStr = NULL; if (TSDB_CODE_SUCCESS != mndGetPlanString(pCreate, &pPlanStr)) { mError("topic:%s, failed to get plan since %s", pCreate->name, terrstr()); return -1; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d9337d8edc..83b3649aa6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -316,7 +316,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqPollRsp(&abuf, &rsp); - taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); + /*taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock);*/ pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index 08e49a7efe..cfbbf52057 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -306,7 +306,7 @@ int32_t init_env() { } //const char* sql = "select * from tu1"; - sprintf(sqlStr, "create topic test_stb_topic_1 as select * from %s%d", g_stConfInfo.stbName, 0); + sprintf(sqlStr, "create topic test_stb_topic_1 as select * from %s", g_stConfInfo.stbName); /*pRes = tmq_create_topic(pConn, "test_stb_topic_1", sqlStr, strlen(sqlStr));*/ pRes = taos_query(pConn, sqlStr); if (taos_errno(pRes) != 0) { From a263f72d5820528d96b2bc43d33a8d4edbdcd7e6 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Tue, 22 Mar 2022 00:54:21 +0800 Subject: [PATCH 10/22] [TD-13762]: redefine atomic api. --- include/client/taos.h | 4 - include/os/osAtomic.h | 417 +++------- include/os/osDir.h | 1 + include/os/osFile.h | 1 + include/os/osLocale.h | 1 + include/os/osRand.h | 1 + include/os/osSleep.h | 1 + include/os/osSocket.h | 4 +- include/os/osString.h | 1 + include/os/osSystem.h | 1 + include/os/osThread.h | 1 + include/os/osTime.h | 7 +- include/os/osTimer.h | 1 + include/os/osTimezone.h | 1 + source/client/src/clientEnv.c | 6 +- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndTopic.c | 2 +- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 8 +- source/libs/executor/src/tsimplehash.c | 4 +- source/libs/function/src/functionMgt.c | 2 +- .../libs/parser/test/mockCatalogService.cpp | 2 +- source/libs/qworker/test/qworkerTests.cpp | 2 +- source/os/src/osAtomic.c | 719 +++++++++++++++++- source/os/src/osSocket.c | 6 - source/util/src/tcache.c | 8 +- source/util/src/thash.c | 10 +- source/util/src/ttimer.c | 2 +- 28 files changed, 833 insertions(+), 384 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index a102629efa..da24136d80 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -85,11 +85,7 @@ typedef struct taosField { int32_t bytes; } TAOS_FIELD; -#ifdef _TD_GO_DLL_ -#define DLL_EXPORT __declspec(dllexport) -#else #define DLL_EXPORT -#endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); diff --git a/include/os/osAtomic.h b/include/os/osAtomic.h index bd3d372fc0..5d45ced3ed 100644 --- a/include/os/osAtomic.h +++ b/include/os/osAtomic.h @@ -20,337 +20,96 @@ extern "C" { #endif -#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) - #define atomic_load_8(ptr) (*(char volatile*)(ptr)) - #define atomic_load_16(ptr) (*(short volatile*)(ptr)) - #define atomic_load_32(ptr) (*(long volatile*)(ptr)) - #define atomic_load_64(ptr) (*(__int64 volatile*)(ptr)) - #define atomic_load_ptr(ptr) (*(void* volatile*)(ptr)) - - #define atomic_store_8(ptr, val) ((*(char volatile*)(ptr)) = (char)(val)) - #define atomic_store_16(ptr, val) ((*(short volatile*)(ptr)) = (short)(val)) - #define atomic_store_32(ptr, val) ((*(long volatile*)(ptr)) = (long)(val)) - #define atomic_store_64(ptr, val) ((*(__int64 volatile*)(ptr)) = (__int64)(val)) - #define atomic_store_ptr(ptr, val) ((*(void* volatile*)(ptr)) = (void*)(val)) - - #define atomic_exchange_8(ptr, val) _InterlockedExchange8((char volatile*)(ptr), (char)(val)) - #define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) - #define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) - #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _WIN64 - #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - #else - #define atomic_exchange_ptr(ptr, val) _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - #endif - - #ifdef _TD_GO_DLL_ - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #else - #define atomic_val_compare_exchange_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval)) - #endif - #define atomic_val_compare_exchange_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval)) - #define atomic_val_compare_exchange_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval)) - #define atomic_val_compare_exchange_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval)) - #define atomic_val_compare_exchange_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)) - - char interlocked_add_fetch_8(char volatile *ptr, char val); - short interlocked_add_fetch_16(short volatile *ptr, short val); - long interlocked_add_fetch_32(long volatile *ptr, long val); - __int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val); - - char interlocked_and_fetch_8(char volatile* ptr, char val); - short interlocked_and_fetch_16(short volatile* ptr, short val); - long interlocked_and_fetch_32(long volatile* ptr, long val); - __int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val); - - __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val); - - char interlocked_or_fetch_8(char volatile* ptr, char val); - short interlocked_or_fetch_16(short volatile* ptr, short val); - long interlocked_or_fetch_32(long volatile* ptr, long val); - __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val); - - char interlocked_xor_fetch_8(char volatile* ptr, char val); - short interlocked_xor_fetch_16(short volatile* ptr, short val); - long interlocked_xor_fetch_32(long volatile* ptr, long val); - __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val); - - __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val); - - #define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - #ifdef _TD_GO_DLL_ - #define atomic_fetch_add_8 __sync_fetch_and_ad - #define atomic_fetch_add_16 __sync_fetch_and_add - #else - #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) - #endif - #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val)) - #define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val)) - #define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val)) - #define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val)) - - #define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val)) - #define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val)) - #define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val)) - #define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val)) - - #define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val)) - #define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val)) - #define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) - - #define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val)) - #define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val)) - #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val)) - - #ifdef _WIN64 - #define atomic_add_fetch_ptr atomic_add_fetch_64 - #define atomic_fetch_add_ptr atomic_fetch_add_64 - #define atomic_sub_fetch_ptr atomic_sub_fetch_64 - #define atomic_fetch_sub_ptr atomic_fetch_sub_64 - #define atomic_and_fetch_ptr atomic_and_fetch_64 - #define atomic_fetch_and_ptr atomic_fetch_and_64 - #define atomic_or_fetch_ptr atomic_or_fetch_64 - #define atomic_fetch_or_ptr atomic_fetch_or_64 - #define atomic_xor_fetch_ptr atomic_xor_fetch_64 - #define atomic_fetch_xor_ptr atomic_fetch_xor_64 - #else - #define atomic_add_fetch_ptr atomic_add_fetch_32 - #define atomic_fetch_add_ptr atomic_fetch_add_32 - #define atomic_sub_fetch_ptr atomic_sub_fetch_32 - #define atomic_fetch_sub_ptr atomic_fetch_sub_32 - #define atomic_and_fetch_ptr atomic_and_fetch_32 - #define atomic_fetch_and_ptr atomic_fetch_and_32 - #define atomic_or_fetch_ptr atomic_or_fetch_32 - #define atomic_fetch_or_ptr atomic_fetch_or_32 - #define atomic_xor_fetch_ptr atomic_xor_fetch_32 - #define atomic_fetch_xor_ptr atomic_fetch_xor_32 - #endif -#elif defined(_TD_NINGSI_60) - /* - * type __sync_fetch_and_add (type *ptr, type value); - * type __sync_fetch_and_sub (type *ptr, type value); - * type __sync_fetch_and_or (type *ptr, type value); - * type __sync_fetch_and_and (type *ptr, type value); - * type __sync_fetch_and_xor (type *ptr, type value); - * type __sync_fetch_and_nand (type *ptr, type value); - * type __sync_add_and_fetch (type *ptr, type value); - * type __sync_sub_and_fetch (type *ptr, type value); - * type __sync_or_and_fetch (type *ptr, type value); - * type __sync_and_and_fetch (type *ptr, type value); - * type __sync_xor_and_fetch (type *ptr, type value); - * type __sync_nand_and_fetch (type *ptr, type value); - * - * bool __sync_bool_compare_and_swap (type*ptr, type oldval, type newval, ...) - * type __sync_val_compare_and_swap (type *ptr, type oldval, ?type newval, ...) - * */ - - #define atomic_load_8(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_16(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_32(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_64(ptr) __sync_fetch_and_add((ptr), 0) - #define atomic_load_ptr(ptr) __sync_fetch_and_add((ptr), 0) - - #define atomic_store_8(ptr, val) (*(ptr)=(val)) - #define atomic_store_16(ptr, val) (*(ptr)=(val)) - #define atomic_store_32(ptr, val) (*(ptr)=(val)) - #define atomic_store_64(ptr, val) (*(ptr)=(val)) - #define atomic_store_ptr(ptr, val) (*(ptr)=(val)) - - int8_t atomic_exchange_8_impl(int8_t* ptr, int8_t val ); - int16_t atomic_exchange_16_impl(int16_t* ptr, int16_t val ); - int32_t atomic_exchange_32_impl(int32_t* ptr, int32_t val ); - int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ); - void* atomic_exchange_ptr_impl( void **ptr, void *val ); - - #define atomic_exchange_8(ptr, val) atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val) - #define atomic_exchange_16(ptr, val) atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val) - #define atomic_exchange_32(ptr, val) atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val) - #define atomic_exchange_64(ptr, val) atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val) - #define atomic_exchange_ptr(ptr, val) atomic_exchange_ptr_impl((void **)ptr, (void*)val) - - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap - - #define atomic_add_fetch_8(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_16(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_32(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_64(ptr, val) __sync_add_and_fetch((ptr), (val)) - #define atomic_add_fetch_ptr(ptr, val) __sync_add_and_fetch((ptr), (val)) - - #define atomic_fetch_add_8(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_16(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_32(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_64(ptr, val) __sync_fetch_and_add((ptr), (val)) - #define atomic_fetch_add_ptr(ptr, val) __sync_fetch_and_add((ptr), (val)) - - #define atomic_sub_fetch_8(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_16(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_32(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_64(ptr, val) __sync_sub_and_fetch((ptr), (val)) - #define atomic_sub_fetch_ptr(ptr, val) __sync_sub_and_fetch((ptr), (val)) - - #define atomic_fetch_sub_8(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_16(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_32(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_64(ptr, val) __sync_fetch_and_sub((ptr), (val)) - #define atomic_fetch_sub_ptr(ptr, val) __sync_fetch_and_sub((ptr), (val)) - - #define atomic_and_fetch_8(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_16(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_32(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_64(ptr, val) __sync_and_and_fetch((ptr), (val)) - #define atomic_and_fetch_ptr(ptr, val) __sync_and_and_fetch((ptr), (val)) - - #define atomic_fetch_and_8(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_16(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_32(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_64(ptr, val) __sync_fetch_and_and((ptr), (val)) - #define atomic_fetch_and_ptr(ptr, val) __sync_fetch_and_and((ptr), (val)) - - #define atomic_or_fetch_8(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_16(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_32(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_64(ptr, val) __sync_or_and_fetch((ptr), (val)) - #define atomic_or_fetch_ptr(ptr, val) __sync_or_and_fetch((ptr), (val)) - - #define atomic_fetch_or_8(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_16(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_32(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_64(ptr, val) __sync_fetch_and_or((ptr), (val)) - #define atomic_fetch_or_ptr(ptr, val) __sync_fetch_and_or((ptr), (val)) - - #define atomic_xor_fetch_8(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_16(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_32(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_64(ptr, val) __sync_xor_and_fetch((ptr), (val)) - #define atomic_xor_fetch_ptr(ptr, val) __sync_xor_and_fetch((ptr), (val)) - - #define atomic_fetch_xor_8(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_16(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_32(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_64(ptr, val) __sync_fetch_and_xor((ptr), (val)) - #define atomic_fetch_xor_ptr(ptr, val) __sync_fetch_and_xor((ptr), (val)) - -#else - #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - - #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap - - #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +// If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. +#ifndef ALLOW_FORBID_FUNC + #define __atomic_load_n __ATOMIC_LOAD_N_FUNC_TAOS_FORBID + #define __atomic_store_n __ATOMIC_STORE_N_FUNC_TAOS_FORBID + #define __atomic_exchange_n __ATOMIC_EXCHANGE_N_FUNC_TAOS_FORBID + #define __sync_val_compare_and_swap __SYNC_VAL_COMPARE_AND_SWAP_FUNC_TAOS_FORBID + #define __atomic_add_fetch __ATOMIC_ADD_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_add __ATOMIC_FETCH_ADD_FUNC_TAOS_FORBID + #define __atomic_sub_fetch __ATOMIC_SUB_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_sub __ATOMIC_FETCH_SUB_FUNC_TAOS_FORBID + #define __atomic_and_fetch __ATOMIC_AND_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_and __ATOMIC_FETCH_AND_FUNC_TAOS_FORBID + #define __atomic_or_fetch __ATOMIC_OR_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_or __ATOMIC_FETCH_OR_FUNC_TAOS_FORBID + #define __atomic_xor_fetch __ATOMIC_XOR_FETCH_FUNC_TAOS_FORBID + #define __atomic_fetch_xor __ATOMIC_FETCH_XOR_FUNC_TAOS_FORBID #endif +int8_t atomic_load_8(int8_t volatile *ptr); +int16_t atomic_load_16(int16_t volatile *ptr); +int32_t atomic_load_32(int32_t volatile *ptr); +int64_t atomic_load_64(int64_t volatile *ptr); +void* atomic_load_ptr(void *ptr); +void atomic_store_8(int8_t volatile *ptr, int8_t val); +void atomic_store_16(int16_t volatile *ptr, int16_t val); +void atomic_store_32(int32_t volatile *ptr, int32_t val); +void atomic_store_64(int64_t volatile *ptr, int64_t val); +void atomic_store_ptr(void *ptr, void *val); +int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val); +void* atomic_exchange_ptr(void *ptr, void *val); +int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval); +int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval); +int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval); +int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval); +void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval); +int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_add_fetch_ptr(void *ptr, int32_t val); +int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_add_ptr(void *ptr, int32_t val); +int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_sub_fetch_ptr(void *ptr, int32_t val); +int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_sub_ptr(void *ptr, int32_t val); +int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_and_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_and_ptr(void *ptr, void *val); +int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_or_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_or_ptr(void *ptr, void *val); +int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val); +void* atomic_xor_fetch_ptr(void *ptr, void *val); +int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val); +int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val); +int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val); +int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val); +void* atomic_fetch_xor_ptr(void *ptr, void *val); + #ifdef __cplusplus } #endif diff --git a/include/os/osDir.h b/include/os/osDir.h index 6cf28fb878..d3597cab36 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -17,6 +17,7 @@ #define _TD_OS_DIR_H_ // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define opendir OPENDIR_FUNC_TAOS_FORBID #define readdir READDIR_FUNC_TAOS_FORBID diff --git a/include/os/osFile.h b/include/os/osFile.h index 508a522679..143b4bf2f8 100644 --- a/include/os/osFile.h +++ b/include/os/osFile.h @@ -23,6 +23,7 @@ extern "C" { #include "osSocket.h" // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following sectio #ifndef ALLOW_FORBID_FUNC #define open OPEN_FUNC_TAOS_FORBID #define fopen FOPEN_FUNC_TAOS_FORBID diff --git a/include/os/osLocale.h b/include/os/osLocale.h index ddafd2e93c..74922cc0b9 100644 --- a/include/os/osLocale.h +++ b/include/os/osLocale.h @@ -23,6 +23,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define setlocale SETLOCALE_FUNC_TAOS_FORBID #endif diff --git a/include/os/osRand.h b/include/os/osRand.h index 09e1f1b41d..bce2c08a2e 100644 --- a/include/os/osRand.h +++ b/include/os/osRand.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define rand RAND_FUNC_TAOS_FORBID #define srand SRAND_FUNC_TAOS_FORBID diff --git a/include/os/osSleep.h b/include/os/osSleep.h index feb9472995..1d0ccf32bd 100644 --- a/include/os/osSleep.h +++ b/include/os/osSleep.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define Sleep SLEEP_FUNC_TAOS_FORBID #define sleep SLEEP_FUNC_TAOS_FORBID diff --git a/include/os/osSocket.h b/include/os/osSocket.h index fb330a9e2f..3c8cc8abd1 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -17,6 +17,7 @@ #define _TD_OS_SOCKET_H_ // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define socket SOCKET_FUNC_TAOS_FORBID #define bind BIND_FUNC_TAOS_FORBID @@ -52,9 +53,6 @@ extern "C" { #if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) #define htobe64 htonll - #if defined(_TD_GO_DLL_) - uint64_t htonll(uint64_t val); - #endif #endif #if defined(_TD_DARWIN_64) diff --git a/include/os/osString.h b/include/os/osString.h index 5846d6b7ed..66d69a849c 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -24,6 +24,7 @@ typedef wchar_t TdWchar; typedef int32_t TdUcs4; // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define iconv_open ICONV_OPEN_FUNC_TAOS_FORBID #define iconv_close ICONV_CLOSE_FUNC_TAOS_FORBID diff --git a/include/os/osSystem.h b/include/os/osSystem.h index f130e9d8f1..48eb6788a7 100644 --- a/include/os/osSystem.h +++ b/include/os/osSystem.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define popen POPEN_FUNC_TAOS_FORBID #define pclose PCLOSE_FUNC_TAOS_FORBID diff --git a/include/os/osThread.h b/include/os/osThread.h index 504814c1c6..6d8ff1de3e 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -37,6 +37,7 @@ typedef pthread_condattr_t TdThreadCondAttr; #define taosThreadCleanupPop pthread_cleanup_pop // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define pthread_t PTHREAD_T_TYPE_TAOS_FORBID #define pthread_spinlock_t PTHREAD_SPINLOCK_T_TYPE_TAOS_FORBID diff --git a/include/os/osTime.h b/include/os/osTime.h index 4904eb9b51..031b9d28f9 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define strptime STRPTIME_FUNC_TAOS_FORBID #define gettimeofday GETTIMEOFDAY_FUNC_TAOS_FORBID @@ -33,11 +34,7 @@ extern "C" { #define CLOCK_REALTIME 0 - #ifdef _TD_GO_DLL_ - #define MILLISECOND_PER_SECOND (1000LL) - #else - #define MILLISECOND_PER_SECOND (1000i64) - #endif + #define MILLISECOND_PER_SECOND (1000i64) #else #define MILLISECOND_PER_SECOND ((int64_t)1000L) #endif diff --git a/include/os/osTimer.h b/include/os/osTimer.h index 9040113b23..5fece43217 100644 --- a/include/os/osTimer.h +++ b/include/os/osTimer.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define timer_create TIMER_CREATE_FUNC_TAOS_FORBID #define timer_settime TIMER_SETTIME_FUNC_TAOS_FORBID diff --git a/include/os/osTimezone.h b/include/os/osTimezone.h index fa0e70f209..c8df8c3f3d 100644 --- a/include/os/osTimezone.h +++ b/include/os/osTimezone.h @@ -21,6 +21,7 @@ extern "C" { #endif // If the error is in a third-party library, place this header file under the third-party library header file. +// When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC #define tzset TZSET_FUNC_TAOS_FORBID #endif diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 07f925bce8..6ebf9e71e0 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -48,8 +48,8 @@ static void registerRequest(SRequestObj *pRequest) { if (pTscObj->pAppInfo) { SInstanceSummary *pSummary = &pTscObj->pAppInfo->summary; - int32_t total = atomic_add_fetch_32(&pSummary->totalRequests, 1); - int32_t currentInst = atomic_add_fetch_32(&pSummary->currentRequests, 1); + int32_t total = atomic_add_fetch_64(&pSummary->totalRequests, 1); + int32_t currentInst = atomic_add_fetch_64(&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); @@ -62,7 +62,7 @@ static void deregisterRequest(SRequestObj *pRequest) { STscObj * pTscObj = pRequest->pTscObj; SInstanceSummary *pActivity = &pTscObj->pAppInfo->summary; - int32_t currentInst = atomic_sub_fetch_32(&pActivity->currentRequests, 1); + int32_t currentInst = atomic_sub_fetch_64(&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); int64_t duration = taosGetTimestampMs() - pRequest->metric.start; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 45d45bef2d..8a07b4fe57 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -166,7 +166,7 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream) { static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream) { mTrace("stream:%s, perform update action", pOldStream->name); - atomic_exchange_32(&pOldStream->updateTime, pNewStream->updateTime); + atomic_exchange_64(&pOldStream->updateTime, pNewStream->updateTime); atomic_exchange_32(&pOldStream->version, pNewStream->version); taosWLockLatch(&pOldStream->lock); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index e94ae4f8ec..6a741840d6 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -176,7 +176,7 @@ static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) { static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) { mTrace("topic:%s, perform update action", pOldTopic->name); - atomic_exchange_32(&pOldTopic->updateTime, pNewTopic->updateTime); + atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); atomic_exchange_32(&pOldTopic->version, pNewTopic->version); taosWLockLatch(&pOldTopic->lock); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 60c5886250..88a7662a76 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -278,7 +278,7 @@ typedef struct SCtgAction { #define CTG_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) #define CTG_API_LEAVE(c) do { int32_t __code = c; CTG_UNLOCK(CTG_READ, &gCtgMgmt.lock); CTG_API_DEBUG("CTG API leave %s", __FUNCTION__); CTG_RET(__code); } while (0) -#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8(&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) +#define CTG_API_ENTER() do { CTG_API_DEBUG("CTG API enter %s", __FUNCTION__); CTG_LOCK(CTG_READ, &gCtgMgmt.lock); if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { CTG_API_LEAVE(TSDB_CODE_CTG_OUT_OF_SERVICE); } } while (0) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index fbb2903a8e..3209b1d80f 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -2044,7 +2044,7 @@ void* ctgUpdateThreadFunc(void* param) { while (true) { tsem_wait(&gCtgMgmt.sem); - if (atomic_load_8(&gCtgMgmt.exit)) { + if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { break; } @@ -2090,7 +2090,7 @@ int32_t catalogInit(SCatalogCfg *cfg) { CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - atomic_store_8(&gCtgMgmt.exit, false); + atomic_store_8((int8_t*)&gCtgMgmt.exit, false); if (cfg) { memcpy(&gCtgMgmt.cfg, cfg, sizeof(*cfg)); @@ -2705,11 +2705,11 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) void catalogDestroy(void) { qInfo("start to destroy catalog"); - if (NULL == gCtgMgmt.pCluster || atomic_load_8(&gCtgMgmt.exit)) { + if (NULL == gCtgMgmt.pCluster || atomic_load_8((int8_t*)&gCtgMgmt.exit)) { return; } - atomic_store_8(&gCtgMgmt.exit, true); + atomic_store_8((int8_t*)&gCtgMgmt.exit, true); tsem_post(&gCtgMgmt.sem); diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 4012c3926e..65067f58e3 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -40,7 +40,7 @@ typedef struct SHNode { typedef struct SSHashObj { SHNode **hashList; size_t capacity; // number of slots - size_t size; // number of elements in hash table + int64_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function _equal_fn_t equalFp; // equal function int32_t keyLen; @@ -91,7 +91,7 @@ int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64(&pHashObj->size); + return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); } static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index f9f2e90770..10ac8bbf43 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -97,7 +97,7 @@ bool fmIsAggFunc(int32_t funcId) { void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; - if (m != NULL && atomic_val_compare_exchange_ptr(&gFunMgtService.pFuncNameHashTable, m, 0) == m) { + if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { taosHashCleanup(m); } } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 1f333e49e7..9abaf5b14f 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -16,8 +16,8 @@ #include #include #include -#include "tdatablock.h" #include "mockCatalogService.h" +#include "tdatablock.h" #include "tname.h" #include "ttypes.h" diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index a45ec41e6c..303ec97f33 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -50,7 +50,7 @@ namespace { int32_t qwtTestMaxExecTaskUsec = 2; int32_t qwtTestReqMaxDelayUsec = 2; -uint64_t qwtTestQueryId = 0; +int64_t qwtTestQueryId = 0; bool qwtTestEnableSleep = true; bool qwtTestStop = false; bool qwtTestDeadLoop = false; diff --git a/source/os/src/osAtomic.c b/source/os/src/osAtomic.c index 6429cdc9d3..8b452947d3 100644 --- a/source/os/src/osAtomic.c +++ b/source/os/src/osAtomic.c @@ -13,25 +13,18 @@ * along with this program. If not, see . */ +#define ALLOW_FORBID_FUNC #include "os.h" #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) // add char interlocked_add_fetch_8(char volatile* ptr, char val) { -#ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; -#else return _InterlockedExchangeAdd8(ptr, val) + val; -#endif } short interlocked_add_fetch_16(short volatile* ptr, short val) { -#ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; -#else return _InterlockedExchangeAdd16(ptr, val) + val; -#endif } long interlocked_add_fetch_32(long volatile* ptr, long val) { @@ -39,11 +32,7 @@ long interlocked_add_fetch_32(long volatile* ptr, long val) { } __int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { -//#ifdef _WIN64 return InterlockedExchangeAdd64(ptr, val) + val; -//#else -// return _InterlockedExchangeAdd(ptr, val) + val; -//#endif } char interlocked_and_fetch_8(char volatile* ptr, char val) { @@ -197,3 +186,709 @@ int64_t atomic_exchange_64_impl(int64_t* ptr, int64_t val ) { return old; } #endif + + +int8_t atomic_load_8(int8_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int8_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_load_16(int16_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int16_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_load_32(int32_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int32_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_load_64(int64_t volatile *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(int64_t volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((ptr), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_load_ptr(void *ptr) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return (*(void* volatile*)(ptr)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), 0); +#else + return __atomic_load_n((void**)(ptr), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int8_t volatile*)(ptr)) = (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int16_t volatile*)(ptr)) = (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int32_t volatile*)(ptr)) = (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(int64_t volatile*)(ptr)) = (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void atomic_store_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + ((*(void* volatile*)(ptr)) = (void*)(val)); +#elif defined(_TD_NINGSI_60) + (*(ptr)=(val)); +#else + __atomic_store_n((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_exchange_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_8_impl((int8_t*)ptr, (int8_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_exchange_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_16_impl((int16_t*)ptr, (int16_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_exchange_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_32_impl((int32_t*)ptr, (int32_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_exchange_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchange64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return atomic_exchange_64_impl((int64_t*)ptr, (int64_t)val); +#else + return __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_exchange_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + #ifdef _WIN64 + return _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); + #else + return _InlineInterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)); + #endif +#elif defined(_TD_NINGSI_60) + return atomic_exchange_ptr_impl((void *)ptr, (void*)val); +#else + return __atomic_exchange_n((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_val_compare_exchange_8(int8_t volatile *ptr, int8_t oldval, int8_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange8((int8_t volatile*)(ptr), (int8_t)(newval), (int8_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +int16_t atomic_val_compare_exchange_16(int16_t volatile *ptr, int16_t oldval, int16_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange16((int16_t volatile*)(ptr), (int16_t)(newval), (int16_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +int32_t atomic_val_compare_exchange_32(int32_t volatile *ptr, int32_t oldval, int32_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange((int32_t volatile*)(ptr), (int32_t)(newval), (int32_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +int64_t atomic_val_compare_exchange_64(int64_t volatile *ptr, int64_t oldval, int64_t newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchange64((int64_t volatile*)(ptr), (int64_t)(newval), (int64_t)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap(ptr, oldval, newval); +#endif +} + +void* atomic_val_compare_exchange_ptr(void *ptr, void *oldval, void *newval) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)); +#elif defined(_TD_NINGSI_60) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#else + return __sync_val_compare_and_swap((void **)ptr, oldval, newval); +#endif +} + +int8_t atomic_add_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_add_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_add_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_add_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_add_fetch_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_add_and_fetch((ptr), (val)); +#else + return __atomic_add_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_add_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_add_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_add_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_add_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_add_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAddptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_add((ptr), (val)); +#else + return __atomic_fetch_add((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_sub_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_8((int8_t volatile*)(ptr), -(int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_sub_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_16((int16_t volatile*)(ptr), -(int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_sub_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_32((int32_t volatile*)(ptr), -(int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_sub_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_64((int64_t volatile*)(ptr), -(int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_sub_fetch_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_add_fetch_ptr((void* volatile*)(ptr), -(void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_sub_and_fetch((ptr), (val)); +#else + return __atomic_sub_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_sub_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd8((int8_t volatile*)(ptr), -(int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_sub_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd16((int16_t volatile*)(ptr), -(int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_sub_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd((int32_t volatile*)(ptr), -(int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_sub_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAdd64((int64_t volatile*)(ptr), -(int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_sub_ptr(void *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedExchangeAddptr((void* volatile*)(ptr), -(void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_sub((ptr), (val)); +#else + return __atomic_fetch_sub((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_and_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_and_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_and_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_and_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_and_fetch_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_and_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_and_and_fetch((ptr), (val)); +#else + return __atomic_and_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_and_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedAnd8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_and_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedAnd16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_and_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedAnd((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_and_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_and_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_and_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_and_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_and((ptr), (val)); +#else + return __atomic_fetch_and((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_or_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_or_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_or_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_or_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_or_fetch_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_or_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_or_and_fetch((ptr), (val)); +#else + return __atomic_or_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_or_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedOr8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_or_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedOr16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_or_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedOr((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_or_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_or_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_or_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_or_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_or((ptr), (val)); +#else + return __atomic_fetch_or((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_xor_fetch_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_xor_fetch_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_xor_fetch_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_32((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_xor_fetch_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_xor_fetch_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_xor_fetch_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_xor_and_fetch((ptr), (val)); +#else + return __atomic_xor_fetch((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int8_t atomic_fetch_xor_8(int8_t volatile *ptr, int8_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedXor8((int8_t volatile*)(ptr), (int8_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int16_t atomic_fetch_xor_16(int16_t volatile *ptr, int16_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedXor16((int16_t volatile*)(ptr), (int16_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int32_t atomic_fetch_xor_32(int32_t volatile *ptr, int32_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return _InterlockedXor((int32_t volatile*)(ptr), (int32_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +int64_t atomic_fetch_xor_64(int64_t volatile *ptr, int64_t val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_xor_64((int64_t volatile*)(ptr), (int64_t)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + +void* atomic_fetch_xor_ptr(void *ptr, void *val) { +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) + return interlocked_fetch_xor_ptr((void* volatile*)(ptr), (void*)(val)); +#elif defined(_TD_NINGSI_60) + return __sync_fetch_and_xor((ptr), (val)); +#else + return __atomic_fetch_xor((void **)(ptr), (val), __ATOMIC_SEQ_CST); +#endif +} + diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index e5f83fa167..0e6abb5785 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -216,12 +216,6 @@ int32_t taosShutDownSocketServerRDWR(TdSocketServerPtr pSocketServer) { #endif } -#if (defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32)) - #if defined(_TD_GO_DLL_) - uint64_t htonll(uint64_t val) { return (((uint64_t)htonl(val)) << 32) + htonl(val >> 32); } - #endif -#endif - void taosWinSocketInit1() { #if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) static char flag = 0; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 9a1b213534..cf5b623678 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -456,7 +456,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen } if (pCacheObj->numOfElems == 0) { - atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); + atomic_add_fetch_64(&pCacheObj->statistics.missCount, 1); return NULL; } @@ -475,15 +475,15 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen void *pData = (pNode != NULL) ? pNode->data : NULL; if (pData != NULL) { - atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1); + atomic_add_fetch_64(&pCacheObj->statistics.hitCount, 1); uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData, T_REF_VAL_GET(pNode)); } else { - atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1); + atomic_add_fetch_64(&pCacheObj->statistics.missCount, 1); uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key); } - atomic_add_fetch_32(&pCacheObj->statistics.totalAccess, 1); + atomic_add_fetch_64(&pCacheObj->statistics.totalAccess, 1); return pData; } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 8b437b9797..a2e20f1762 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -55,7 +55,7 @@ typedef struct SHashEntry { struct SHashObj { SHashEntry ** hashList; size_t capacity; // number of slots - size_t size; // number of elements in hash table + int64_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function _equal_fn_t equalFp; // equal function _hash_free_fn_t freeFp; // hash node free callback function @@ -186,7 +186,7 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - atomic_sub_fetch_32(&pNode->refCount, 1); + atomic_sub_fetch_16(&pNode->refCount, 1); if (prev != NULL) { prev->next = pNewNode; } else { @@ -302,7 +302,7 @@ int32_t taosHashGetSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; } - return (int32_t)atomic_load_64(&pHashObj->size); + return (int32_t)atomic_load_64((int64_t*)&pHashObj->size); } int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { @@ -508,7 +508,7 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pNode->removed == 0) { code = 0; // it is found - atomic_sub_fetch_32(&pNode->refCount, 1); + atomic_sub_fetch_16(&pNode->refCount, 1); pNode->removed = 1; if (pNode->refCount <= 0) { if (prevNode == NULL) { @@ -755,7 +755,7 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { pNode = pNode->next; } - atomic_sub_fetch_32(&pOld->refCount, 1); + atomic_sub_fetch_16(&pOld->refCount, 1); if (pOld->refCount <=0) { if (prevNode) { prevNode->next = pOld->next; diff --git a/source/util/src/ttimer.c b/source/util/src/ttimer.c index cc31d9c94a..5e59e61ea1 100644 --- a/source/util/src/ttimer.c +++ b/source/util/src/ttimer.c @@ -132,7 +132,7 @@ static timer_map_t timerMap; static uintptr_t getNextTimerId() { uintptr_t id; do { - id = atomic_add_fetch_ptr(&nextTimerId, 1); + id = (uintptr_t)atomic_add_fetch_ptr((void **)&nextTimerId, 1); } while (id == 0); return id; } From fa4c40e519ab3f54b2bf34193fec4c49ba2025bb Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 22 Mar 2022 09:45:32 +0800 Subject: [PATCH 11/22] [modify vnode path] --- tests/test/c/tmqDemo.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test/c/tmqDemo.c b/tests/test/c/tmqDemo.c index cfbbf52057..f8759f703b 100644 --- a/tests/test/c/tmqDemo.c +++ b/tests/test/c/tmqDemo.c @@ -65,11 +65,11 @@ typedef struct { static SConfInfo g_stConfInfo = { "tmqdb", "stb", - "./tmqResult.txt", // output_file - "/data2/dnode/data/vnode/vnode2/wal", + "./tmqResult.txt", // output_file + "/data2/dnode/data/vnodes/vnode2/wal", 1, // threads 1, // tables - 1, // vgroups + 1, // vgroups 0, // run mode 1, // columns 1, // ratio From 5f9f8ac7e2f7ffae137e0c9f0e7a47884f0296a2 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Mar 2022 10:03:17 +0800 Subject: [PATCH 12/22] fix double free --- source/dnode/mgmt/vnode/src/vmWorker.c | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index d5f0a9d20e..035b805d0e 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -62,11 +62,10 @@ static void vmProcessQueryQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, &pMsg->rpcMsg); if (code != 0) { vmSendRsp(pVnode->pWrapper, pMsg, code); + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); } - - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - rpcFreeCont(pMsg->rpcMsg.pCont); - taosFreeQitem(pMsg); } static void vmProcessFetchQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { @@ -74,12 +73,10 @@ static void vmProcessFetchQueue(SVnodeObj *pVnode, SNodeMsg *pMsg) { int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, &pMsg->rpcMsg); if (code != 0) { vmSendRsp(pVnode->pWrapper, pMsg, code); + dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); + rpcFreeCont(pMsg->rpcMsg.pCont); + taosFreeQitem(pMsg); } - - dTrace("msg:%p, is freed, result:0x%04x:%s", pMsg, code & 0XFFFF, tstrerror(code)); - // TODO: handle invalid write - /*rpcFreeCont(pMsg->rpcMsg.pCont);*/ - /*taosFreeQitem(pMsg);*/ } static void vmProcessWriteQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t numOfMsgs) { From 0025aed0d55ba0a64b221f4526e7650a2e01f55a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 21 Mar 2022 22:42:01 -0400 Subject: [PATCH 13/22] TD-14045 bugfix --- source/libs/parser/src/parAstCreater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index eed6391d5c..1e51288e02 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -892,7 +892,7 @@ SDataType createDataType(uint8_t type) { } SDataType createVarLenDataType(uint8_t type, const SToken* pLen) { - SDataType dt = { .type = type, .precision = 0, .scale = 0, .bytes = tDataTypes[type].bytes }; + SDataType dt = { .type = type, .precision = 0, .scale = 0, .bytes = strtol(pLen->z, NULL, 10) }; return dt; } From c5c00d595568e667a609c85e1d1c107fe9336517 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Tue, 22 Mar 2022 10:42:02 +0800 Subject: [PATCH 14/22] sync refactor --- source/libs/sync/inc/syncEnv.h | 6 +++--- source/libs/sync/inc/syncInt.h | 5 +++++ source/libs/sync/inc/syncUtil.h | 2 +- source/libs/sync/src/syncMain.c | 15 ++++++++++----- source/libs/sync/src/syncUtil.c | 5 ++++- source/libs/sync/test/syncUtilTest.cpp | 2 +- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 66c7c8620d..7940d7f436 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -31,10 +31,10 @@ extern "C" { #define TIMER_MAX_MS 0x7FFFFFFF #define ENV_TICK_TIMER_MS 1000 #define PING_TIMER_MS 1000 -#define ELECT_TIMER_MS_MIN 1500 -#define ELECT_TIMER_MS_MAX 3000 +#define ELECT_TIMER_MS_MIN 150 +#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) #define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 300 +#define HEARTBEAT_TIMER_MS 30 #define EMPTY_RAFT_ID ((SRaftId){.addr = 0, .vgId = 0}) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index eaf4d56f67..cfa87048a7 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -160,6 +160,11 @@ typedef struct SSyncNode { SSyncLogStore* pLogStore; SyncIndex commitIndex; + // timer ms init + int32_t pingBaseLine; + int32_t electBaseLine; + int32_t hbBaseLine; + // ping timer tmr_h pPingTimer; int32_t pingTimerMS; diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h index 1b702c2528..a4bb11afc6 100644 --- a/source/libs/sync/inc/syncUtil.h +++ b/source/libs/sync/inc/syncUtil.h @@ -44,7 +44,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest); // ---- misc ---- int32_t syncUtilRand(int32_t max); -int32_t syncUtilElectRandomMS(); +int32_t syncUtilElectRandomMS(int32_t min, int32_t max); int32_t syncUtilQuorum(int32_t replicaNum); cJSON* syncUtilNodeInfo2Json(const SNodeInfo* p); cJSON* syncUtilRaftId2Json(const SRaftId* p); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index facb0180f9..b92317ef3f 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -242,9 +242,14 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { assert(pSyncNode->pLogStore != NULL); pSyncNode->commitIndex = SYNC_INDEX_INVALID; + // timer ms init + pSyncNode->pingBaseLine = PING_TIMER_MS; + pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN; + pSyncNode->hbBaseLine = HEARTBEAT_TIMER_MS; + // init ping timer pSyncNode->pPingTimer = NULL; - pSyncNode->pingTimerMS = PING_TIMER_MS; + pSyncNode->pingTimerMS = pSyncNode->pingBaseLine; atomic_store_64(&pSyncNode->pingTimerLogicClock, 0); atomic_store_64(&pSyncNode->pingTimerLogicClockUser, 0); pSyncNode->FpPingTimerCB = syncNodeEqPingTimer; @@ -252,7 +257,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init elect timer pSyncNode->pElectTimer = NULL; - pSyncNode->electTimerMS = syncUtilElectRandomMS(); + pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); atomic_store_64(&pSyncNode->electTimerLogicClock, 0); atomic_store_64(&pSyncNode->electTimerLogicClockUser, 0); pSyncNode->FpElectTimerCB = syncNodeEqElectTimer; @@ -260,7 +265,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pSyncInfo) { // init heartbeat timer pSyncNode->pHeartbeatTimer = NULL; - pSyncNode->heartbeatTimerMS = HEARTBEAT_TIMER_MS; + pSyncNode->heartbeatTimerMS = pSyncNode->hbBaseLine; atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, 0); atomic_store_64(&pSyncNode->heartbeatTimerLogicClockUser, 0); pSyncNode->FpHeartbeatTimerCB = syncNodeEqHeartbeatTimer; @@ -394,7 +399,7 @@ int32_t syncNodeRestartElectTimer(SSyncNode* pSyncNode, int32_t ms) { int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - int32_t electMS = syncUtilElectRandomMS(); + int32_t electMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); ret = syncNodeRestartElectTimer(pSyncNode, electMS); return ret; } @@ -763,7 +768,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { syncTimeoutDestroy(pSyncMsg); // reset timer ms - pSyncNode->electTimerMS = syncUtilElectRandomMS(); + pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, &pSyncNode->pPingTimer); } else { diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 9486405331..8fc17ccb51 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -96,7 +96,10 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { int32_t syncUtilRand(int32_t max) { return taosRand() % max; } -int32_t syncUtilElectRandomMS() { return ELECT_TIMER_MS_MIN + syncUtilRand(ELECT_TIMER_MS_RANGE); } +int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { + assert(min > 0 && max > 0 && max >= min); + return min + syncUtilRand(max - min); +} int32_t syncUtilQuorum(int32_t replicaNum) { return replicaNum / 2 + 1; } diff --git a/source/libs/sync/test/syncUtilTest.cpp b/source/libs/sync/test/syncUtilTest.cpp index 663db3a7b3..ae2afd2f53 100644 --- a/source/libs/sync/test/syncUtilTest.cpp +++ b/source/libs/sync/test/syncUtilTest.cpp @@ -16,7 +16,7 @@ void logTest() { void electRandomMSTest() { for (int i = 0; i < 10; ++i) { - int32_t ms = syncUtilElectRandomMS(); + int32_t ms = syncUtilElectRandomMS(150, 300); printf("syncUtilElectRandomMS: %d \n", ms); } } From fe8714fe56b14b8fd71251faa75a3e08cf5da78f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Mar 2022 10:46:16 +0800 Subject: [PATCH 15/22] refact worker --- include/util/tqueue.h | 3 -- include/util/tworker.h | 33 +++++++---------- source/dnode/mgmt/vnode/inc/vmInt.h | 2 +- source/dnode/mgmt/vnode/src/vmWorker.c | 10 ++--- source/util/src/tqueue.c | 48 ------------------------ source/util/src/tworker.c | 51 +------------------------- 6 files changed, 22 insertions(+), 125 deletions(-) diff --git a/include/util/tqueue.h b/include/util/tqueue.h index d51184edfc..9faf90113e 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -70,10 +70,7 @@ int32_t taosGetQueueNumber(STaosQset *qset); int32_t taosReadQitemFromQset(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp); int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahandle, FItems *itemsFp); - -int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId); void taosResetQsetThread(STaosQset *qset, void *pItem); - int32_t taosGetQueueItemsNumber(STaosQueue *queue); int32_t taosGetQsetItemsNumber(STaosQset *qset); diff --git a/include/util/tworker.h b/include/util/tworker.h index dabd1ac9b3..3da8a1db63 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -27,33 +27,33 @@ typedef struct SWWorkerPool SWWorkerPool; typedef struct SQWorker { int32_t id; // worker ID - TdThread thread; // thread + TdThread thread; // thread SQWorkerPool *pool; -} SQWorker, SFWorker; +} SQWorker; typedef struct SQWorkerPool { - int32_t max; // max number of workers - int32_t min; // min number of workers - int32_t num; // current number of workers - STaosQset *qset; - const char *name; - SQWorker *workers; + int32_t max; // max number of workers + int32_t min; // min number of workers + int32_t num; // current number of workers + STaosQset *qset; + const char *name; + SQWorker *workers; TdThreadMutex mutex; -} SQWorkerPool, SFWorkerPool; +} SQWorkerPool; typedef struct SWWorker { int32_t id; // worker id - TdThread thread; // thread + TdThread thread; // thread STaosQall *qall; STaosQset *qset; // queue set SWWorkerPool *pool; } SWWorker; typedef struct SWWorkerPool { - int32_t max; // max number of workers - int32_t nextId; // from 0 to max-1, cyclic - const char *name; - SWWorker *workers; + int32_t max; // max number of workers + int32_t nextId; // from 0 to max-1, cyclic + const char *name; + SWWorker *workers; TdThreadMutex mutex; } SWWorkerPool; @@ -62,11 +62,6 @@ void tQWorkerCleanup(SQWorkerPool *pool); STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp); void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue); -int32_t tFWorkerInit(SFWorkerPool *pool); -void tFWorkerCleanup(SFWorkerPool *pool); -STaosQueue *tFWorkerAllocQueue(SFWorkerPool *pool, void *ahandle, FItem fp); -void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue); - int32_t tWWorkerInit(SWWorkerPool *pool); void tWWorkerCleanup(SWWorkerPool *pool); STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); diff --git a/source/dnode/mgmt/vnode/inc/vmInt.h b/source/dnode/mgmt/vnode/inc/vmInt.h index d670e073da..ba59258d07 100644 --- a/source/dnode/mgmt/vnode/inc/vmInt.h +++ b/source/dnode/mgmt/vnode/inc/vmInt.h @@ -30,7 +30,7 @@ typedef struct SVnodesMgmt { SVnodesStat state; STfs *pTfs; SQWorkerPool queryPool; - SFWorkerPool fetchPool; + SQWorkerPool fetchPool; SWWorkerPool syncPool; SWWorkerPool writePool; const char *path; diff --git a/source/dnode/mgmt/vnode/src/vmWorker.c b/source/dnode/mgmt/vnode/src/vmWorker.c index 035b805d0e..e0632cee68 100644 --- a/source/dnode/mgmt/vnode/src/vmWorker.c +++ b/source/dnode/mgmt/vnode/src/vmWorker.c @@ -262,7 +262,7 @@ int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessWriteQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode, (FItems)vmProcessApplyQueue); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); - pVnode->pFetchQ = tFWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); + pVnode->pFetchQ = tQWorkerAllocQueue(&pMgmt->fetchPool, pVnode, (FItem)vmProcessFetchQueue); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); if (pVnode->pApplyQ == NULL || pVnode->pWriteQ == NULL || pVnode->pSyncQ == NULL || pVnode->pFetchQ == NULL || @@ -277,7 +277,7 @@ int32_t vmAllocQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { void vmFreeQueue(SVnodesMgmt *pMgmt, SVnodeObj *pVnode) { tQWorkerFreeQueue(&pMgmt->queryPool, pVnode->pQueryQ); - tFWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); + tQWorkerFreeQueue(&pMgmt->fetchPool, pVnode->pFetchQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pWriteQ); tWWorkerFreeQueue(&pMgmt->writePool, pVnode->pApplyQ); tWWorkerFreeQueue(&pMgmt->syncPool, pVnode->pSyncQ); @@ -303,11 +303,11 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { pQPool->max = maxQueryThreads; if (tQWorkerInit(pQPool) != 0) return -1; - SFWorkerPool *pFPool = &pMgmt->fetchPool; + SQWorkerPool *pFPool = &pMgmt->fetchPool; pFPool->name = "vnode-fetch"; pFPool->min = minFetchThreads; pFPool->max = maxFetchThreads; - if (tFWorkerInit(pFPool) != 0) return -1; + if (tQWorkerInit(pFPool) != 0) return -1; SWWorkerPool *pWPool = &pMgmt->writePool; pWPool->name = "vnode-write"; @@ -330,7 +330,7 @@ int32_t vmStartWorker(SVnodesMgmt *pMgmt) { void vmStopWorker(SVnodesMgmt *pMgmt) { dndCleanupWorker(&pMgmt->mgmtWorker); - tFWorkerCleanup(&pMgmt->fetchPool); + tQWorkerCleanup(&pMgmt->fetchPool); tQWorkerCleanup(&pMgmt->queryPool); tWWorkerCleanup(&pMgmt->writePool); tWWorkerCleanup(&pMgmt->syncPool); diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 99675630f4..258ca1402f 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -68,7 +68,6 @@ STaosQueue *taosOpenQueue() { return NULL; } - queue->threadId = -1; uDebug("queue:%p is opened", queue); return queue; } @@ -437,53 +436,6 @@ int32_t taosReadAllQitemsFromQset(STaosQset *qset, STaosQall *qall, void **ahand return code; } -int32_t taosReadQitemFromQsetByThread(STaosQset *qset, void **ppItem, void **ahandle, FItem *itemFp, int32_t threadId) { - STaosQnode *pNode = NULL; - int32_t code = -1; - - tsem_wait(&qset->sem); - - taosThreadMutexLock(&qset->mutex); - - for (int32_t i = 0; i < qset->numOfQueues; ++i) { - if (qset->current == NULL) qset->current = qset->head; - STaosQueue *queue = qset->current; - if (queue) qset->current = queue->next; - if (queue == NULL) break; - if (queue->head == NULL) continue; - if (queue->threadId != -1 && queue->threadId != threadId) { - code = 0; - continue; - } - - taosThreadMutexLock(&queue->mutex); - - if (queue->head) { - pNode = queue->head; - pNode->queue = queue; - queue->threadId = threadId; - *ppItem = pNode->item; - - if (ahandle) *ahandle = queue->ahandle; - if (itemFp) *itemFp = queue->itemFp; - - queue->head = pNode->next; - if (queue->head == NULL) queue->tail = NULL; - queue->numOfItems--; - atomic_sub_fetch_32(&qset->numOfItems, 1); - code = 1; - uTrace("item:%p is read out from queue:%p, items:%d", *ppItem, queue, queue->numOfItems); - } - - taosThreadMutexUnlock(&queue->mutex); - if (pNode) break; - } - - taosThreadMutexUnlock(&qset->mutex); - - return code; -} - void taosResetQsetThread(STaosQset *qset, void *pItem) { if (pItem == NULL) return; STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 78098af3bd..7d7f819dec 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -93,7 +93,7 @@ static void *tQWorkerThreadFp(SQWorker *worker) { return NULL; } -STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, ThreadFp threadFp) { +STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { taosThreadMutexLock(&pool->mutex); STaosQueue *queue = taosOpenQueue(); if (queue == NULL) { @@ -114,7 +114,7 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (taosThreadCreate(&worker->thread, &thAttr, threadFp, worker) != 0) { + if (taosThreadCreate(&worker->thread, &thAttr, (ThreadFp)tQWorkerThreadFp, worker) != 0) { uError("worker:%s:%d failed to create thread to process since %s", pool->name, worker->id, strerror(errno)); taosCloseQueue(queue); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -134,58 +134,11 @@ STaosQueue *tWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp, Threa return queue; } -STaosQueue *tQWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { - return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); -} - void tQWorkerFreeQueue(SQWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } -int32_t tFWorkerInit(SFWorkerPool *pool) { return tQWorkerInit((SQWorkerPool *)pool); } - -void tFWorkerCleanup(SFWorkerPool *pool) { tQWorkerCleanup(pool); } - -static void *tFWorkerThreadFp(SQWorker *worker) { - SQWorkerPool *pool = worker->pool; - - FItem fp = NULL; - void *msg = NULL; - void *ahandle = NULL; - int32_t code = 0; - - taosBlockSIGPIPE(); - setThreadName(pool->name); - uDebug("worker:%s:%d is running", pool->name, worker->id); - - while (1) { - code = taosReadQitemFromQsetByThread(pool->qset, (void **)&msg, &ahandle, &fp, worker->id); - - if (code < 0) { - uDebug("worker:%s:%d qset:%p, got no message and exiting", pool->name, worker->id, pool->qset); - break; - } else if (code == 0) { - // uTrace("worker:%s:%d qset:%p, got no message and continue", pool->name, worker->id, pool->qset); - continue; - } - - if (fp != NULL) { - (*fp)(ahandle, msg); - } - - taosResetQsetThread(pool->qset, msg); - } - - return NULL; -} - -STaosQueue *tFWorkerAllocQueue(SQWorkerPool *pool, void *ahandle, FItem fp) { - return tWorkerAllocQueue(pool, ahandle, fp, (ThreadFp)tQWorkerThreadFp); -} - -void tFWorkerFreeQueue(SFWorkerPool *pool, STaosQueue *queue) { tQWorkerFreeQueue(pool, queue); } - int32_t tWWorkerInit(SWWorkerPool *pool) { pool->nextId = 0; pool->workers = calloc(pool->max, sizeof(SWWorker)); From 25f2b374cccb53dfae4deb56e056cf78231d11a0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 21 Mar 2022 23:50:27 -0400 Subject: [PATCH 16/22] TD-13757 bugfix --- source/libs/parser/src/parInsert.c | 13 ++++++++++--- source/libs/parser/src/parTokenizer.c | 4 ++-- source/libs/parser/src/parTranslater.c | 12 +++++++++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 803d8e00d8..1111ff2887 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -28,6 +28,13 @@ pSql += index; \ } while (0) +#define NEXT_TOKEN_WITH_PREV(pSql, sToken) \ + do { \ + int32_t index = 0; \ + sToken = tStrGetToken(pSql, &index, true); \ + pSql += index; \ + } while (0) + #define NEXT_TOKEN_KEEP_SQL(pSql, sToken, index) \ do { \ sToken = tStrGetToken(pSql, &index, false); \ @@ -352,7 +359,7 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time sToken = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; - if (sToken.type == TK_MINUS || sToken.type == TK_NK_PLUS) { + if (sToken.type == TK_NK_MINUS || sToken.type == TK_NK_PLUS) { index = 0; valueToken = tStrGetToken(pTokenEnd, &index, false); pTokenEnd += index; @@ -748,7 +755,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pTagsSchema, SToken sToken; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" for (int i = 0; i < pCxt->tags.numOfBound; ++i) { - NEXT_TOKEN(pCxt->pSql, sToken); + NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); SSchema* pSchema = &pTagsSchema[pCxt->tags.boundedColumns[i]]; param.schema = pSchema; CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); @@ -814,7 +821,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, SToken sToken = {0}; // 1. set the parsed value from sql string for (int i = 0; i < spd->numOfBound; ++i) { - NEXT_TOKEN(pCxt->pSql, sToken); + NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); SSchema *pSchema = &schema[spd->boundedColumns[i] - 1]; param.schema = pSchema; getMemRowAppendInfo(schema, pBuilder->rowType, spd, i, ¶m.toffset, ¶m.colIdx); diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 3cd4549ad0..7f16157090 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -320,7 +320,7 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) { *tokenId = TK_NK_COMMENT; return i; } - *tokenId = TK_MINUS; + *tokenId = TK_NK_MINUS; return 1; } case '(': { @@ -674,7 +674,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) { } else { // support parse the -/+number format - if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_NK_PLUS)) { + if ((isPrevOptr) && (t0.type == TK_NK_MINUS || t0.type == TK_NK_PLUS)) { len = tGetToken(&str[*i + t0.n], &type); if (type == TK_NK_INTEGER || type == TK_NK_FLOAT) { t0.type = type; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 70652901ef..f20bc5f2e6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -873,12 +873,22 @@ static int32_t translateAlterDatabase(STranslateContext* pCxt, SAlterDatabaseStm return TSDB_CODE_SUCCESS; } +static int32_t calcTypeBytes(SDataType dt) { + if (TSDB_DATA_TYPE_BINARY == dt.type) { + return dt.bytes + VARSTR_HEADER_SIZE; + } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { + return dt.bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } else { + return dt.bytes; + } +} + static int32_t columnNodeToField(SNodeList* pList, SArray** pArray) { *pArray = taosArrayInit(LIST_LENGTH(pList), sizeof(SField)); SNode* pNode; FOREACH(pNode, pList) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; - SField field = { .type = pCol->dataType.type, .bytes = pCol->dataType.bytes }; + SField field = { .type = pCol->dataType.type, .bytes = calcTypeBytes(pCol->dataType) }; strcpy(field.name, pCol->colName); taosArrayPush(*pArray, &field); } From a4e6423b51747ba405846252989a1241f42e5d42 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Tue, 22 Mar 2022 12:01:26 +0800 Subject: [PATCH 17/22] [add tmq cases] --- tests/script/jenkins/basic.txt | 3 ++ tests/script/tsim/tmq/basic.sim | 75 +++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 tests/script/tsim/tmq/basic.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 0c2b4fe1f4..cc0576e1f3 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -21,4 +21,7 @@ # ---- query ./test.sh -f tsim/query/interval.sim + +# ---- tmq +./test.sh -f tsim/tmq/basic.sim #======================b1-end=============== diff --git a/tests/script/tsim/tmq/basic.sim b/tests/script/tsim/tmq/basic.sim new file mode 100644 index 0000000000..a091e45061 --- /dev/null +++ b/tests/script/tsim/tmq/basic.sim @@ -0,0 +1,75 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wal -v 1 +system sh/exec.sh -n dnode1 -s start +sleep 500 +sql connect + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 100 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +#root@trd02 /data2/dnode $ tmq_demo --help +#Used to tmq_demo +# -c Configuration directory, default is +# -d The name of the database to be created, default is tmqdb +# -s The name of the super table to be created, default is stb +# -f The file of result, default is ./tmqResult.txt +# -w The path of vnode of wal, default is /data2/dnode/data/vnodes/vnode2/wal +# -t numOfThreads, default is 1 +# -n numOfTables, default is 1 +# -v numOfVgroups, default is 1 +# -a runMode, default is 0 +# -l numOfColumn, default is 1 +# -q ratio, default is 1.000000 +# -b batchNumOfRow, default is 1 +# -r totalRowsOfPerTbl, default is 10000 +# -m startTimestamp, default is 1640966400000 [2022-01-01 00:00:00] +# -g showMsgFlag, default is 0 +# +#system_content ../../debug/tests/test/c/tmq_demo -c ../../sim/tsim/cfg +system ../../debug/tests/test/c/tmq_demo -c ../../sim/tsim/cfg +print result-> $system_content + +sql show databases +print ===> $rows $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != tmqdb then + return -1 +endi + +sql use tmqdb +sql show tables +print ===> $rows $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != stb0 then + return -1 +endi + +sql select count(*) from stb0 +print ===> $rows $data00 $data01 $data02 $data03 +if $rows != 1 then + return -1 +endi +if $data00 != 10000 then + return -1 +endi +#system sh/exec.sh -n dnode1 -s stop -x SIGINT From a73a79e711cf47aa50b0e69445fe63f7098bc3b5 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 22 Mar 2022 01:07:21 -0400 Subject: [PATCH 18/22] TD-13757 bugfix --- tests/script/tsim/insert/basic0.sim | 53 ++++++++++++++--------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/tests/script/tsim/insert/basic0.sim b/tests/script/tsim/insert/basic0.sim index 6c24662be7..b83cc224f8 100644 --- a/tests/script/tsim/insert/basic0.sim +++ b/tests/script/tsim/insert/basic0.sim @@ -42,9 +42,8 @@ sql insert into ct1 values(now+0s, 10, 2.0, 3.0) sql insert into ct1 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) sql insert into ct2 values(now+0s, 10, 2.0, 3.0) sql insert into ct2 values(now+1s, 11, 2.1, 3.1)(now+2s, 12, 2.2, 3.2)(now+3s, 13, 2.3, 3.3) -# after fix bug, modify sql_error to sql -sql_error insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) -sql_error insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) +sql insert into ct1 values(now+4s, -14, -2.4, -3.4) ct2 values(now+4s, -14, -2.4, -3.4) +sql insert into ct1 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) ct2 values(now+5s, -15, -2.5, -3.5)(now+6s, -16, -2.6, -3.6) sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) @@ -52,7 +51,7 @@ sql insert into ct3 values('2021-01-01 00:00:00.000', 10, 2.0, 3.0) #=================================================================== print =============== query data from child table sql select * from ct1 -if $rows != 4 then # after fix bug, modify 4 to 7 +if $rows != 7 then return -1 endi if $data01 != 10 then @@ -82,23 +81,23 @@ if $rows != 1 then endi print $data00 $data01 $data02 -if $data00 != 4 then +if $data00 != 7 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 print $data00 $data01 $data02 $data03 -if $data00 != 4 then +if $data00 != 7 then return -1 endi -if $data01 != 4 then +if $data01 != 7 then return -1 endi -if $data02 != 4 then +if $data02 != 7 then return -1 endi -if $data03 != 4 then +if $data03 != 7 then return -1 endi @@ -112,13 +111,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -16 then return -1 endi -if $data01 != 2.00000 then +if $data01 != -2.60000 then return -1 endi -if $data02 != 3.000000000 then +if $data02 != -3.600000000 then return -1 endi @@ -144,13 +143,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != 1 then return -1 endi -if $data01 != 8.599999905 then +if $data01 != 1.099999905 then return -1 endi -if $data02 != 12.600000000 then +if $data02 != 2.100000000 then return -1 endi @@ -232,7 +231,7 @@ system sh/exec.sh -n dnode1 -s start sleep 2000 sql select * from ct1 -if $rows != 4 then # after fix bug, modify 4 to 7 +if $rows != 7 then return -1 endi if $data01 != 10 then @@ -262,23 +261,23 @@ if $rows != 1 then endi print $data00 $data01 $data02 -if $data00 != 4 then +if $data00 != 7 then return -1 endi print =============== select count(column) from child table sql select count(ts), count(c1), count(c2), count(c3) from ct1 print $data00 $data01 $data02 $data03 -if $data00 != 4 then +if $data00 != 7 then return -1 endi -if $data01 != 4 then +if $data01 != 7 then return -1 endi -if $data02 != 4 then +if $data02 != 7 then return -1 endi -if $data03 != 4 then +if $data03 != 7 then return -1 endi @@ -292,13 +291,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 10 then +if $data00 != -16 then return -1 endi -if $data01 != 2.00000 then +if $data01 != -2.60000 then return -1 endi -if $data02 != 3.000000000 then +if $data02 != -3.600000000 then return -1 endi @@ -324,13 +323,13 @@ print $data00 $data01 $data02 $data03 if $rows != 1 then return -1 endi -if $data00 != 46 then +if $data00 != 1 then return -1 endi -if $data01 != 8.599999905 then +if $data01 != 1.099999905 then return -1 endi -if $data02 != 12.600000000 then +if $data02 != 2.100000000 then return -1 endi From 4091771274f95910111e46c41d9b5579363b62bf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 22 Mar 2022 14:38:26 +0800 Subject: [PATCH 19/22] refact queue --- include/util/tworker.h | 32 ++++++++++++++ source/dnode/mgmt/mnode/inc/mmInt.h | 6 +-- source/dnode/mgmt/mnode/src/mmWorker.c | 28 ++++++------ source/util/src/tqueue.c | 1 - source/util/src/tworker.c | 59 +++++++++++++++++++++++++- 5 files changed, 108 insertions(+), 18 deletions(-) diff --git a/include/util/tworker.h b/include/util/tworker.h index 3da8a1db63..91f4fbf7ff 100644 --- a/include/util/tworker.h +++ b/include/util/tworker.h @@ -67,6 +67,38 @@ void tWWorkerCleanup(SWWorkerPool *pool); STaosQueue *tWWorkerAllocQueue(SWWorkerPool *pool, void *ahandle, FItems fp); void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue); +typedef struct { + const char *name; + int32_t minNum; + int32_t maxNum; + FItem fp; + void *param; +} SQWorkerAllCfg; + +typedef struct { + const char *name; + STaosQueue *queue; + SQWorkerPool pool; +} SQWorkerAll; + +typedef struct { + const char *name; + int32_t maxNum; + FItems fp; + void *param; +} SWWorkerAllCfg; + +typedef struct { + const char *name; + STaosQueue *queue; + SWWorkerPool pool; +} SWWorkerAll; + +int32_t tQWorkerAllInit(SQWorkerAll *pWorker, const SQWorkerAllCfg *pCfg); +void tQWorkerAllCleanup(SQWorkerAll *pWorker); +int32_t tWWorkerAllInit(SWWorkerAll *pWorker, const SWWorkerAllCfg *pCfg); +void tWWorkerAllCleanup(SWWorkerAll *pWorker); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/mnode/inc/mmInt.h b/source/dnode/mgmt/mnode/inc/mmInt.h index 1751131764..06ed637791 100644 --- a/source/dnode/mgmt/mnode/inc/mmInt.h +++ b/source/dnode/mgmt/mnode/inc/mmInt.h @@ -28,9 +28,9 @@ typedef struct SMnodeMgmt { SDnode *pDnode; SMgmtWrapper *pWrapper; const char *path; - SDnodeWorker readWorker; - SDnodeWorker writeWorker; - SDnodeWorker syncWorker; + SQWorkerAll readWorker; + SQWorkerAll writeWorker; + SQWorkerAll syncWorker; SReplica replicas[TSDB_MAX_REPLICA]; int8_t replica; int8_t selfIndex; diff --git a/source/dnode/mgmt/mnode/src/mmWorker.c b/source/dnode/mgmt/mnode/src/mmWorker.c index ab9f46e323..470128940b 100644 --- a/source/dnode/mgmt/mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mnode/src/mmWorker.c @@ -42,9 +42,9 @@ static void mmProcessQueue(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { taosFreeQitem(pMsg); } -static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SNodeMsg *pMsg) { +static int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SQWorkerAll *pWorker, SNodeMsg *pMsg) { dTrace("msg:%p, put into worker %s", pMsg, pWorker->name); - return dndWriteMsgToWorker(pWorker, pMsg); + return taosWriteQitem(pWorker->queue, pMsg); } int32_t mmProcessWriteMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { @@ -59,7 +59,7 @@ int32_t mmProcessReadMsg(SMnodeMgmt *pMgmt, SNodeMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->readWorker, pMsg); } -static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRpcMsg *pRpc) { +static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SQWorkerAll *pWorker, SRpcMsg *pRpc) { SNodeMsg *pMsg = taosAllocateQitem(sizeof(SNodeMsg)); if (pMsg == NULL) { return -1; @@ -68,7 +68,7 @@ static int32_t mmPutRpcMsgToWorker(SMnodeMgmt *pMgmt, SDnodeWorker *pWorker, SRp dTrace("msg:%p, is created and put into worker:%s, type:%s", pMsg, pWorker->name, TMSG_INFO(pRpc->msgType)); pMsg->rpcMsg = *pRpc; - int32_t code = dndWriteMsgToWorker(pWorker, pMsg); + int32_t code = taosWriteQitem(pWorker->queue, pMsg); if (code != 0) { dTrace("msg:%p, is freed", pMsg); taosFreeQitem(pMsg); @@ -89,18 +89,20 @@ int32_t mmPutMsgToReadQueue(SMgmtWrapper *pWrapper, SRpcMsg *pRpc) { } int32_t mmStartWorker(SMnodeMgmt *pMgmt) { - if (dndInitWorker(pMgmt, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, mmProcessQueue) != 0) { - dError("failed to start mnode read worker since %s", terrstr()); + SQWorkerAllCfg cfg = {.minNum = 0, .maxNum = 1, .name = "mnode-read", .fp = (FItem)mmProcessQueue, .param = pMgmt}; + + if (tQWorkerAllInit(&pMgmt->readWorker, &cfg) != 0) { + dError("failed to start mnode-read worker since %s", terrstr()); return -1; } - if (dndInitWorker(pMgmt, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, mmProcessQueue) != 0) { - dError("failed to start mnode write worker since %s", terrstr()); + if (tQWorkerAllInit(&pMgmt->writeWorker, &cfg) != 0) { + dError("failed to start mnode-write worker since %s", terrstr()); return -1; } - if (dndInitWorker(pMgmt, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, mmProcessQueue) != 0) { - dError("failed to start mnode sync worker since %s", terrstr()); + if (tQWorkerAllInit(&pMgmt->syncWorker, &cfg) != 0) { + dError("failed to start mnode sync-worker since %s", terrstr()); return -1; } @@ -108,7 +110,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { } void mmStopWorker(SMnodeMgmt *pMgmt) { - dndCleanupWorker(&pMgmt->readWorker); - dndCleanupWorker(&pMgmt->writeWorker); - dndCleanupWorker(&pMgmt->syncWorker); + tQWorkerAllCleanup(&pMgmt->readWorker); + tQWorkerAllCleanup(&pMgmt->writeWorker); + tQWorkerAllCleanup(&pMgmt->syncWorker); } diff --git a/source/util/src/tqueue.c b/source/util/src/tqueue.c index 258ca1402f..70f2871f41 100644 --- a/source/util/src/tqueue.c +++ b/source/util/src/tqueue.c @@ -441,7 +441,6 @@ void taosResetQsetThread(STaosQset *qset, void *pItem) { STaosQnode *pNode = (STaosQnode *)((char *)pItem - sizeof(STaosQnode)); taosThreadMutexLock(&qset->mutex); - pNode->queue->threadId = -1; for (int32_t i = 0; i < pNode->queue->numOfItems; ++i) { tsem_post(&qset->sem); } diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 7d7f819dec..e05c4e0a78 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -22,7 +22,7 @@ typedef void *(*ThreadFp)(void *param); int32_t tQWorkerInit(SQWorkerPool *pool) { pool->qset = taosOpenQset(); - pool->workers = calloc(sizeof(SQWorker), pool->max); + pool->workers = calloc(pool->max, sizeof(SQWorker)); if (pool->workers == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -279,3 +279,60 @@ void tWWorkerFreeQueue(SWWorkerPool *pool, STaosQueue *queue) { taosCloseQueue(queue); uDebug("worker:%s, queue:%p is freed", pool->name, queue); } + +int32_t tQWorkerAllInit(SQWorkerAll *pWorker, const SQWorkerAllCfg *pCfg) { + SQWorkerPool *pPool = &pWorker->pool; + pPool->name = pCfg->name; + pPool->min = pCfg->minNum; + pPool->max = pCfg->maxNum; + if (tQWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->queue = tQWorkerAllocQueue(pPool, pCfg->param, pCfg->fp); + if (pWorker->queue == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->name = pCfg->name; + return 0; +} + +void tQWorkerAllCleanup(SQWorkerAll *pWorker) { + if (pWorker->queue == NULL) return; + + while (!taosQueueEmpty(pWorker->queue)) { + taosMsleep(10); + } + + tQWorkerCleanup(&pWorker->pool); + tQWorkerFreeQueue(&pWorker->pool, pWorker->queue); +} + +int32_t tWWorkerAllInit(SWWorkerAll *pWorker, const SWWorkerAllCfg *pCfg) { + SWWorkerPool *pPool = &pWorker->pool; + pPool->name = pCfg->name; + pPool->max = pCfg->maxNum; + if (tWWorkerInit(pPool) != 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->queue = tWWorkerAllocQueue(pPool, pCfg->param, pCfg->fp); + if (pWorker->queue == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pWorker->name = pCfg->name; + return 0; +} + +void tWWorkerAllCleanup(SWWorkerAll *pWorker) { + if (pWorker->queue == NULL) return; + + while (!taosQueueEmpty(pWorker->queue)) { + taosMsleep(10); + } + + tWWorkerCleanup(&pWorker->pool); + tWWorkerFreeQueue(&pWorker->pool, pWorker->queue); +} From 55338753262bbc14e85a7bed00bed3eb1ba57588 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 22 Mar 2022 16:03:42 +0800 Subject: [PATCH 20/22] [td-13039] support group by. --- include/common/tcommon.h | 6 +- source/libs/executor/inc/executorimpl.h | 6 +- source/libs/executor/src/executorimpl.c | 236 +++++++++++++----------- source/libs/function/src/builtinsimpl.c | 4 +- 4 files changed, 141 insertions(+), 111 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index fb787a32df..8faccf7815 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -160,10 +160,8 @@ typedef struct SColumn { int64_t dataBlockId; }; - union { - int16_t colId; - int16_t slotId; - }; + int16_t colId; + int16_t slotId; char name[TSDB_COL_NAME_LEN]; int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 66d5f8b5a7..f170d5c529 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -543,9 +543,10 @@ typedef struct SFillOperatorInfo { typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; - int32_t colIndex; + SArray* pGroupCols; char* prevData; // previous group by value SGroupResInfo groupResInfo; + SAggSupporter aggSup; } SGroupbyOperatorInfo; typedef struct SSessionAggOperatorInfo { @@ -643,6 +644,8 @@ SOperatorInfo* createLimitOperatorInfo(SOperatorInfo* downstream, int32_t numOfD SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, + SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, @@ -650,7 +653,6 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index a1b11c8161..d1aa7cd99a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include "os.h" @@ -242,8 +243,8 @@ static void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity, int32_t* rowCellOffset); static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); -static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); - +static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupId, + SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, int64_t keyFirst, int64_t keyLast, STimeWindow *win); @@ -1012,8 +1013,8 @@ static void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, int32_t of pCtx[k].startTs = pWin->skey; // keep it temporarialy - int32_t startOffset = pCtx[k].input.startRowIndex; bool hasAgg = pCtx[k].input.colDataAggIsSet; + int32_t startOffset = pCtx[k].input.startRowIndex; int32_t numOfRows = pCtx[k].input.numOfRows; int32_t pos = (order == TSDB_ORDER_ASC) ? offset : offset - (forwardStep - 1); @@ -1622,35 +1623,30 @@ static void hashAllIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // updateResultRowInfoActiveIndex(pResultRowInfo, pQueryAttr, pRuntimeEnv->current->lastKey); } +static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pBlock) { + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + // TODO multiple group by columns + SColumn* pCol = taosArrayGet(pInfo->pGroupCols, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); -static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pInfo, SSDataBlock *pSDataBlock) { - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - STableQueryInfo* item = pRuntimeEnv->current; - - SColumnInfoData* pColInfoData = taosArrayGet(pSDataBlock->pDataBlock, pInfo->colIndex); - - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int16_t bytes = pColInfoData->info.bytes; - int16_t type = pColInfoData->info.type; - + int16_t bytes = pColInfoData->info.bytes; + int16_t type = pColInfoData->info.type; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); return; } - SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; - STimeWindow w = TSWINDOW_INITIALIZER; int32_t num = 0; - for (int32_t j = 0; j < pSDataBlock->info.rows; ++j) { - char* val = ((char*)pColInfoData->pData) + bytes * j; - if (isNull(val, type)) { + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + if (colDataIsNull(pColInfoData, pBlock->info.rows, j, NULL)) { // TODO continue; } + char* val = colDataGetData(pColInfoData, j); + // Compare with the previous row of this column, and do not set the output buffer again if they are identical. if (pInfo->prevData == NULL) { pInfo->prevData = malloc(bytes); @@ -1672,35 +1668,28 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } } - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); - } - - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); + int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, 0, + pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } -// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, j - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + doApplyFunctions(pInfo->binfo.pCtx, &w, j - num, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); num = 1; memcpy(pInfo->prevData, val, bytes); } if (num > 0) { - char* val = ((char*)pColInfoData->pData) + bytes * (pSDataBlock->info.rows - num); + char* val = ((char*)pColInfoData->pData) + bytes * (pBlock->info.rows - num); memcpy(pInfo->prevData, val, bytes); - - if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); + int32_t ret = setGroupResultOutputBuf_rv(&(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, 0, + pInfo->aggSup.pResultBuf, pTaskInfo, &pInfo->aggSup); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); - if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); - } - -// doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + doApplyFunctions(pInfo->binfo.pCtx, &w, pBlock->info.rows - num, num, NULL, pBlock->info.rows, pOperator->numOfOutput, TSDB_ORDER_ASC); } tfree(pInfo->prevData); @@ -1792,9 +1781,8 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { } } -static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - SDiskbasedBuf *pResultBuf = pRuntimeEnv->pResultBuf; - +static int32_t setGroupResultOutputBuf_rv(SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupId, + SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup) { int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx *pCtx = binfo->pCtx; @@ -1808,19 +1796,12 @@ static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicI } int64_t tid = 0; - SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, d, len, true, groupIndex); + SResultRow *pResultRow = doSetResultOutBufByKey_rv(pBuf, pResultRowInfo, groupId, (char *)pData, TSDB_KEYSIZE, true, groupId, pTaskInfo, true, pAggSup); assert (pResultRow != NULL); setResultRowKey(pResultRow, pData, type); - if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->pQueryAttr->resultRowSize); - if (ret != 0) { - return -1; - } - } - setResultOutputBuf(pRuntimeEnv, pResultRow, pCtx, numOfCols, rowCellInfoOffset); - initCtxOutputBuffer(pCtx, numOfCols); + setResultRowOutputBufInitCtx_rv(pBuf, pResultRow, pCtx, numOfCols, binfo->rowCellInfoOffset); return TSDB_CODE_SUCCESS; } @@ -1851,6 +1832,10 @@ static bool functionNeedToExecute(SqlFunctionCtx *pCtx) { // in case of timestamp column, always generated results. int32_t functionId = pCtx->functionId; + if (functionId == -1) { + return false; + } + if (functionId == FUNCTION_TS) { return true; } @@ -2063,9 +2048,13 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SExprInfo* pExprInfo, int32_t num if (pExpr->pExpr->_function.pFunctNode != NULL) { SFuncExecEnv env = {0}; - fmGetFuncExecFuncs(pExpr->pExpr->_function.pFunctNode->funcId, &pCtx->fpSet); + pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; + + fmGetFuncExecFuncs(pCtx->functionId, &pCtx->fpSet); pCtx->fpSet.getEnv(pExpr->pExpr->_function.pFunctNode, &env); pCtx->resDataInfo.interBufSize = env.calcMemSize; + } else { + pCtx->functionId = -1; } pCtx->input.numOfInputCols = pFunct->numOfParams; @@ -3318,8 +3307,6 @@ void setFunctionResultOutput(SOptrBasicInfo* pInfo, SAggSupporter* pSup, int32_t SResultRow* pRow = doSetResultOutBufByKey_rv(pSup->pResultBuf, pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pSup); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { - SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); - struct SResultRowEntryInfo* pEntry = getResultCell(pRow, i, rowCellInfoOffset); cleanupResultRowEntry(pEntry); @@ -3417,7 +3404,7 @@ void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity) { void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size) { for (int32_t j = 0; j < size; ++j) { struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(&pCtx[j]); - if (isRowEntryInitialized(pResInfo)) { + if (isRowEntryInitialized(pResInfo) || pCtx[j].functionId == -1) { continue; } @@ -3458,6 +3445,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SqlFunctionCt void finalizeQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput) { for (int32_t j = 0; j < numOfOutput; ++j) { + if (pCtx[j].functionId == -1) { + continue; + } + pCtx[j].fpSet.finalize(&pCtx[j]); } } @@ -3480,7 +3471,9 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD continue; } - pCtx[j].fpSet.finalize(&pCtx[j]); + if (pCtx[j].fpSet.process) { // TODO set the dummy function. + pCtx[j].fpSet.finalize(&pCtx[j]); + } if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; @@ -3586,20 +3579,13 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pRes void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - SFilePage* bufPage = getBufPage(pBuf, pResult->pageId); - -// int32_t offset = 0; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { -// offset += pCtx[i].resDataInfo.bytes; continue; } - -// offset += pCtx[i].resDataInfo.bytes; - // int32_t functionId = pCtx[i].functionId; // if (functionId < 0) { // continue; @@ -3608,7 +3594,7 @@ void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, // if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; // } - if (!pResInfo->initialized) { + if (!pResInfo->initialized && pCtx[i].functionId != -1) { pCtx[i].fpSet.init(&pCtx[i], pResInfo); } } @@ -6941,15 +6927,15 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo *pOperator, bool* newgrou SGroupbyOperatorInfo *pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { -// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - if (pInfo->binfo.pRes->info.rows == 0/*|| !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)*/) { + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); + if (pInfo->binfo.pRes->info.rows == 0|| !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } return pInfo->binfo.pRes; } - SOperatorInfo* downstream = pOperator->pDownstream[0]; int32_t order = TSDB_ORDER_ASC; + SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); @@ -6962,29 +6948,23 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo *pOperator, bool* newgrou // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, order); // setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); - if (pInfo->colIndex == -1) { - pInfo->colIndex = getGroupbyColumnIndex(NULL/*pInfo->pGroupbyExpr*/, pBlock); - } - doHashGroupbyAgg(pOperator, pInfo, pBlock); } pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->binfo.resultRowInfo); + finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); // if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows - finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); +// finalizeQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput); // } else { // updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); // } + blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); -// if (!pRuntimeEnv->pQueryAttr->stableQuery) { -// sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); -// } - -// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - if (pInfo->binfo.pRes->info.rows == 0/* || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)*/) { + toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -7663,25 +7643,25 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun return pOperator; } -SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SSDataBlock* pResBlock, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, + SArray* pGroupColList, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SGroupbyOperatorInfo* pInfo = calloc(1, sizeof(SGroupbyOperatorInfo)); - pInfo->colIndex = -1; // group by column index + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + pInfo->pGroupCols = pGroupColList; + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, 4096, pResultBlock, pTaskInfo->id.str); -// pQueryAttr->resultRowSize = (pQueryAttr->resultRowSize * -// (int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery))); - - pInfo->binfo.pRes = pResBlock; initResultRowInfo(&pInfo->binfo.resultRowInfo, 8); - SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "GroupbyAggOperator"; pOperator->blockingOptr = true; pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; + pOperator->pExpr = pExprInfo; + pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; pOperator->_openFn = operatorDummyOpenFn; pOperator->getNextFn = hashGroupbyAggregate; @@ -7689,6 +7669,11 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; + + _error: + tfree(pInfo); + tfree(pOperator); + return NULL; } SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult) { @@ -8152,17 +8137,29 @@ static SResSchema createResSchema(int32_t type, int32_t bytes, int32_t slotId, i return s; } -SExprInfo* createExprInfo(SNodeList* pNodeList, int32_t* numOfExprs) { +SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); + int32_t numOfGroupKeys = 0; + if (pGroupKeys != NULL) { + numOfGroupKeys = LIST_LENGTH(pGroupKeys); + } - *numOfExprs = numOfFuncs; - SExprInfo* pExprs = calloc(numOfFuncs, sizeof(SExprInfo)); + *numOfExprs = numOfFuncs + numOfGroupKeys; + SExprInfo* pExprs = calloc(*numOfExprs, sizeof(SExprInfo)); - for(int32_t i = 0; i < numOfFuncs; ++i) { - SExprInfo* pExp = &pExprs[i]; + for(int32_t i = 0; i < (*numOfExprs); ++i) { + STargetNode* pTargetNode = NULL; + if (i < numOfFuncs) { + pTargetNode = (STargetNode*)nodesListGetNode(pNodeList, i); + } else { + pTargetNode = (STargetNode*)nodesListGetNode(pGroupKeys, i - numOfFuncs); + } + + SExprInfo* pExp = &pExprs[pTargetNode->slotId]; pExp->pExpr = calloc(1, sizeof(tExprNode)); pExp->pExpr->_function.num = 1; + pExp->pExpr->_function.functionId = -1; pExp->base.pParam = calloc(1, sizeof(SFunctParam)); pExp->base.numOfParams = 1; @@ -8170,10 +8167,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, int32_t* numOfExprs) { pExp->base.pParam[0].pCol = calloc(1, sizeof(SColumn)); SColumn* pCol = pExp->base.pParam[0].pCol; - STargetNode* pTargetNode = (STargetNode*)nodesListGetNode(pNodeList, i); - ASSERT(pTargetNode->slotId == i); - - // it is a project query + // it is a project query, or group by column if (nodeType(pTargetNode->pExpr) == QUERY_NODE_COLUMN) { SColumnNode* pColNode = (SColumnNode*) pTargetNode->pExpr; @@ -8190,6 +8184,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, int32_t* numOfExprs) { SDataType* pType = &pFuncNode->node.resType; pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, pType->precision, pFuncNode->node.aliasName); + pExp->pExpr->_function.functionId = pFuncNode->funcId; pExp->pExpr->_function.pFunctNode = pFuncNode; strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, tListLen(pExp->pExpr->_function.functionName)); @@ -8199,11 +8194,11 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, int32_t* numOfExprs) { SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); SColumnNode* pcn = (SColumnNode*)p1; - pCol->slotId = pcn->slotId; - pCol->bytes = pcn->node.resType.bytes; - pCol->type = pcn->node.resType.type; - pCol->scale = pcn->node.resType.scale; - pCol->precision = pcn->node.resType.precision; + pCol->slotId = pcn->slotId; + pCol->bytes = pcn->node.resType.bytes; + pCol->type = pcn->node.resType.type; + pCol->scale = pcn->node.resType.scale; + pCol->precision = pcn->node.resType.precision; pCol->dataBlockId = pcn->dataBlockId; } } @@ -8231,6 +8226,7 @@ static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SRead static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); static SArray* extractTableIdList(const STableGroupInfo* pTableGroupInfo); static SArray* extractScanColumnId(SNodeList* pNodeList); +static SArray* extractColumnInfo(SNodeList* pNodeList); SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { // if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_PROJECT) { // ignore the project node @@ -8289,7 +8285,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(((SProjectPhysiNode*)pPhyNode)->pProjections, &num); + SExprInfo* pExprInfo = createExprInfo(((SProjectPhysiNode*)pPhyNode)->pProjections, NULL, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); return createProjectOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo); } @@ -8302,9 +8298,17 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(((SAggPhysiNode*)pPhyNode)->pAggFuncs, &num); + + SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; + SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); - return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); + + if (pAggNode->pGroupKeys != NULL) { + SArray* pColList = extractColumnInfo(pAggNode->pGroupKeys); + return createGroupOperatorInfo(op, pExprInfo, num, pResBlock, pColList, pTaskInfo, NULL); + } else { + return createAggregateOperatorInfo(op, pExprInfo, num, pResBlock, pTaskInfo, pTableGroupInfo); + } } } else if (QUERY_NODE_PHYSICAL_PLAN_INTERVAL == nodeType(pPhyNode)) { size_t size = LIST_LENGTH(pPhyNode->pChildren); @@ -8317,7 +8321,7 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhysiNode* pPhyNode, SExecTaskInfo* pTa SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; int32_t num = 0; - SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->pFuncs, &num); + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->pFuncs, NULL, &num); SSDataBlock* pResBlock = createOutputBuf_rv1(pPhyNode->pOutputDataBlockDesc); SInterval interval = {.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = 'a', .slidingUnit = 'a'}; @@ -8385,6 +8389,32 @@ SArray* extractScanColumnId(SNodeList* pNodeList) { return pList; } +SArray* extractColumnInfo(SNodeList* pNodeList) { + size_t numOfCols = LIST_LENGTH(pNodeList); + SArray* pList = taosArrayInit(numOfCols, sizeof(SColumn)); + if (pList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + STargetNode* pNode = (STargetNode*) nodesListGetNode(pNodeList, i); + SColumnNode* pColNode = (SColumnNode*) pNode->pExpr; + + SColumn c = {0}; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.precision = pColNode->node.resType.precision; + c.scale = pColNode->node.resType.scale; + + taosArrayPush(pList, &c); + } + + return pList; +} + int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId) { int32_t code = 0; if (tableType == TSDB_SUPER_TABLE) { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index fa83068755..14f63bead8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -456,7 +456,7 @@ void firstFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = 0; - struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); char* buf = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; @@ -500,7 +500,7 @@ void lastFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = 0; - struct SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); char* buf = GET_ROWCELL_INTERBUF(pResInfo); SInputColumnInfoData* pInput = &pCtx->input; From 72d121b3e7b555bf23d472f361e3927cb15312a3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 22 Mar 2022 04:38:10 -0400 Subject: [PATCH 21/22] merge 3.0 --- tests/script/tmqResult.txt | 120 ------------------------------------- 1 file changed, 120 deletions(-) delete mode 100644 tests/script/tmqResult.txt diff --git a/tests/script/tmqResult.txt b/tests/script/tmqResult.txt deleted file mode 100644 index ad3da044c8..0000000000 --- a/tests/script/tmqResult.txt +++ /dev/null @@ -1,120 +0,0 @@ -################################################################### -# configDir: ../../sim/tsim/cfg -# dbName: tmqdb -# stbName: stb -# vnodeWalPath: /data2/dnode/data/vnodes/vnode2/wal -# numOfTables: 1 -# numOfThreads: 1 -# numOfVgroups: 1 -# runMode: 0 -# ratio: 1.000000 -# numOfColumn: 1 -# batchNumOfRow: 1 -# totalRowsOfPerTbl: 10000 -# totalRowsOfT2: 10000 -# Test time: 2022-03-22 08:06:41 -################################################################### -|-------------------------------insert info-----------------------------|--------------------------------consume info---------------------------------| -|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume time(s) | msgs/s | MB/s | avg msg size(KB) | -| 1| 10000 | 4.245 | 2355.60 | -0.000 | 0 | 6.840 | 0.00 | -0.00| -inf | - -################################################################### -# configDir: ../../sim/tsim/cfg -# dbName: tmqdb -# stbName: stb -# vnodeWalPath: /data2/dnode/data/vnodes/vnode2/wal -# numOfTables: 1 -# numOfThreads: 1 -# numOfVgroups: 1 -# runMode: 0 -# ratio: 1.000000 -# numOfColumn: 1 -# batchNumOfRow: 1 -# totalRowsOfPerTbl: 10000 -# totalRowsOfT2: 10000 -# Test time: 2022-03-22 08:09:18 -################################################################### -|-------------------------------insert info-----------------------------|--------------------------------consume info---------------------------------| -|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume time(s) | msgs/s | MB/s | avg msg size(KB) | -| 1| 10000 | 4.297 | 2327.25 | -0.000 | 0 | 0.037 | 0.00 | -0.00| -inf | - -################################################################### -# configDir: ../../sim/tsim/cfg -# dbName: tmqdb -# stbName: stb -# vnodeWalPath: /data2/dnode/data/vnodes/vnode2/wal -# numOfTables: 1 -# numOfThreads: 1 -# numOfVgroups: 1 -# runMode: 0 -# ratio: 1.000000 -# numOfColumn: 1 -# batchNumOfRow: 1 -# totalRowsOfPerTbl: 10000 -# totalRowsOfT2: 10000 -# Test time: 2022-03-22 08:15:38 -################################################################### -|-------------------------------insert info-----------------------------|--------------------------------consume info---------------------------------| -|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume time(s) | msgs/s | MB/s | avg msg size(KB) | -| 1| 10000 | 4.349 | 2299.28 | -0.000 | 0 | 0.051 | 0.00 | -0.00| -inf | - -################################################################### -# configDir: ../../sim/tsim/cfg -# dbName: tmqdb -# stbName: stb -# vnodeWalPath: /data2/dnode/data/vnodes/vnode2/wal -# numOfTables: 1 -# numOfThreads: 1 -# numOfVgroups: 1 -# runMode: 0 -# ratio: 1.000000 -# numOfColumn: 1 -# batchNumOfRow: 1 -# totalRowsOfPerTbl: 10000 -# totalRowsOfT2: 10000 -# Test time: 2022-03-22 08:24:23 -################################################################### -|-------------------------------insert info-----------------------------|--------------------------------consume info---------------------------------| -|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume time(s) | msgs/s | MB/s | avg msg size(KB) | -| 1| 10000 | 4.243 | 2356.83 | -0.000 | 0 | 0.019 | 0.00 | -0.00| -inf | - -################################################################### -# configDir: ../../sim/tsim/cfg -# dbName: tmqdb -# stbName: stb -# vnodeWalPath: /data2/dnode/data/vnodes/vnode2/wal -# numOfTables: 1 -# numOfThreads: 1 -# numOfVgroups: 1 -# runMode: 0 -# ratio: 1.000000 -# numOfColumn: 1 -# batchNumOfRow: 1 -# totalRowsOfPerTbl: 10000 -# totalRowsOfT2: 10000 -# Test time: 2022-03-22 08:26:09 -################################################################### -|-------------------------------insert info-----------------------------|--------------------------------consume info---------------------------------| -|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume time(s) | msgs/s | MB/s | avg msg size(KB) | -| 1| 10000 | 4.022 | 2486.15 | -0.000 | 10000 | 4.218 | 2370.99 | -0.00| -0.00 | - -################################################################### -# configDir: ../../sim/tsim/cfg -# dbName: tmqdb -# stbName: stb -# vnodeWalPath: /data2/dnode/data/vnodes/vnode2/wal -# numOfTables: 1 -# numOfThreads: 1 -# numOfVgroups: 1 -# runMode: 0 -# ratio: 1.000000 -# numOfColumn: 1 -# batchNumOfRow: 1 -# totalRowsOfPerTbl: 10000 -# totalRowsOfT2: 10000 -# Test time: 2022-03-22 08:29:04 -################################################################### -|-------------------------------insert info-----------------------------|--------------------------------consume info---------------------------------| -|batch size| insert msgs | insert time(s) | msgs/s | walLogSize(MB) | consume msgs | consume time(s) | msgs/s | MB/s | avg msg size(KB) | -| 1| 10000 | 5.247 | 1905.75 | -0.000 | 10000 | 5.454 | 1833.68 | -0.00| -0.00 | - From ef50049a8b1fa6446370695c18fa5fb074d81dd3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 22 Mar 2022 04:54:40 -0400 Subject: [PATCH 22/22] merge 3.0 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3eda69f60d..1bfbf00cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,7 @@ tests/examples/JDBC/JDBCDemo/.classpath tests/examples/JDBC/JDBCDemo/.project tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* +tests/script/tmqResult.txt # Emacs # -*- mode: gitignore; -*-