diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 2b5994e95c..83b7ade407 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 7321fbb + GIT_TAG 16eb34f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/include/common/tglobal.h b/include/common/tglobal.h index cb4426f8a9..7df6124152 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -119,6 +119,7 @@ extern SDiskCfg tsDiskCfg[]; // udf extern bool tsStartUdfd; extern char tsUdfdResFuncs[]; +extern char tsUdfdLdLibPath[]; // schemaless extern char tsSmlChildTableName[]; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 282cee015e..67c2a9081c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -57,7 +57,8 @@ extern int32_t tMsgDict[]; #define TMSG_SEG_SEQ(TYPE) ((TYPE)&0xff) #define TMSG_INFO(TYPE) \ ((TYPE) < TDMT_DND_MAX_MSG || (TYPE) < TDMT_MND_MAX_MSG || (TYPE) < TDMT_VND_MAX_MSG || (TYPE) < TDMT_SCH_MAX_MSG || \ - (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) \ + (TYPE) < TDMT_VND_TMQ_MAX_MSG || (TYPE) < TDMT_STREAM_MAX_MSG || (TYPE) < TDMT_VND_STREAM_MAX_MSG || \ + (TYPE) < TDMT_MON_MAX_MSG || (TYPE) < TDMT_SYNC_MAX_MSG) \ ? tMsgInfo[tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)] \ : 0 #define TMSG_INDEX(TYPE) (tMsgDict[TMSG_SEG_CODE(TYPE)] + TMSG_SEG_SEQ(TYPE)) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 571d14fe3c..3d9d55b101 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -138,17 +138,15 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_INDEX, "get-table-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_BATCH_META, "batch-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TABLE_CFG, "table-cfg", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) - TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "alter-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "drop-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_HB, "consumer-hb", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_COMMIT_OFFSET, "mnode-commit-offset", SMqCMCommitOffsetReq, SMqCMCommitOffsetRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_TOPIC, "drop-topic", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_SUBSCRIBE, "subscribe", SCMSubscribeReq, SCMSubscribeRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_ASK_EP, "ask-ep", SMqAskEpReq, SMqAskEpRsp) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CONSUMER_LOST, "consumer-lost", SMqConsumerLostMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_CONSUMER_RECOVER, "consumer-recover", SMqConsumerRecoverMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_HB, "consumer-hb", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DO_REBALANCE, "do-rebalance", SMqDoRebalanceMsg, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TMQ_DROP_CGROUP, "drop-cgroup", SMqDropCGroupReq, SMqDropCGroupRsp) TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) @@ -186,21 +184,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_CHANGE, "vnode-mq-vg-change", SMqRebVgReq, SMqRebVgRsp) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_VG_DELETE, "vnode-mq-vg-delete", SMqVDeleteReq, SMqVDeleteRsp) - TD_DEF_MSG_TYPE(TDMT_VND_MQ_COMMIT_OFFSET, "vnode-commit-offset", STqOffset, STqOffset) - TD_DEF_MSG_TYPE(TDMT_VND_ADD_CHECK_INFO, "vnode-add-check-info", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DELETE_CHECK_INFO, "vnode-delete-check-info", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_CREATE_TOPIC, "vnode-create-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_ALTER_TOPIC, "vnode-alter-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_DROP_TOPIC, "vnode-drop-topic", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_SUBSCRIBE, "vnode-subscribe", SMVSubscribeReq, SMVSubscribeRsp) - TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqPollReq, SMqDataBlkRsp) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_DISPATCH_WRITE, "vnode-stream-task-dispatch-write", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP1, "vnode-stream-recover1", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP2, "vnode-stream-recover2", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) @@ -232,15 +215,30 @@ enum { TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SCH_MAX_MSG, "sch-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_ADD_CHECKINFO, "vnode-tmq-add-checkinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DEL_CHECKINFO, "vnode-del-checkinfo", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_RECOVER_FINISH, "vnode-stream-finish", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_VND_STREAM_MSG) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP1, "vnode-stream-recover1", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_RECOVER_STEP2, "vnode-stream-recover2", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) + TD_NEW_MSG_SEG(TDMT_MON_MSG) TD_DEF_MSG_TYPE(TDMT_MON_MAX_MSG, "monitor-max", NULL, NULL) diff --git a/include/common/tname.h b/include/common/tname.h index 8c594a2b5e..666a25303e 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -72,13 +72,13 @@ bool tNameTbNameEqual(SName* left, SName* right); typedef struct { // input - SArray* tags; // element is SSmlKv - const char* sTableName; // super table name - uint8_t sTableNameLen; // the length of super table name + SArray* tags; // element is SSmlKv + const char* stbFullName; // super table name + uint8_t stbFullNameLen; // the length of super table name // output - char* childTableName; // must have size of TSDB_TABLE_NAME_LEN; - uint64_t uid; // child table uid, may be useful + char* ctbShortName; // must have size of TSDB_TABLE_NAME_LEN; + uint64_t uid; // child table uid, may be useful } RandTableName; void buildChildTableName(RandTableName* rName); diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index c9e47c25b7..cdb1642a5c 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -99,7 +99,6 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); */ int32_t mndProcessRpcMsg(SRpcMsg *pMsg); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); -int32_t mndProcessSyncCtrlMsg(SRpcMsg *pMsg); int32_t mndPreProcessQueryMsg(SRpcMsg *pMsg); void mndPostProcessQueryMsg(SRpcMsg *pMsg); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index dfeb68ce43..b957be4267 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -309,6 +309,9 @@ int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* f int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, + bool* pass, bool* exists); + int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); int32_t catalogUpdateVgEpSet(SCatalog* pCtg, const char* dbFName, int32_t vgId, SEpSet* epSet); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 3b1b63496f..1fe30a2d66 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -79,6 +79,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers); */ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* readers, int32_t* numOfCols, SSchemaWrapper** pSchema); +int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); /** * Set multiple input data blocks for the stream scan. * @param tinfo diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 6968f1712c..a50af18b44 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -216,6 +216,7 @@ bool fmIsMultiRowsFunc(int32_t funcId); bool fmIsKeepOrderFunc(int32_t funcId); bool fmIsCumulativeFunc(int32_t funcId); bool fmIsInterpPseudoColumnFunc(int32_t funcId); +bool fmIsGroupKeyFunc(int32_t funcId); void getLastCacheDataType(SDataType* pType); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 534d86b1f1..e2616567f5 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -441,8 +441,9 @@ typedef struct { } SStreamRetrieveRsp; typedef struct { - int64_t streamId; - int32_t taskId; + SMsgHead msgHead; + int64_t streamId; + int32_t taskId; } SStreamRecoverStep1Req, SStreamRecoverStep2Req; typedef struct { diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 95ee2ca2bc..fc926cb1b4 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -200,10 +200,10 @@ typedef struct SSyncInfo { int32_t syncInit(); void syncCleanUp(); +bool syncIsInit(); int64_t syncOpen(SSyncInfo* pSyncInfo); void syncStart(int64_t rid); void syncStop(int64_t rid); -int32_t syncSetStandby(int64_t rid); ESyncState syncGetMyRole(int64_t rid); bool syncIsReady(int64_t rid); const char* syncGetMyRoleStr(int64_t rid); @@ -216,24 +216,18 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet); void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); // int32_t syncProposeBatch(int64_t rid, SRpcMsg** pMsgPArr, bool* pIsWeakArr, int32_t arrSize); -bool syncEnvIsStart(); const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapshot); -int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg); - -// build SRpcMsg, need to call syncPropose with SRpcMsg -int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg); - +int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); int32_t syncLeaderTransfer(int64_t rid); -int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader); - int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); int32_t syncEndSnapshot(int64_t rid); - int32_t syncStepDown(int64_t rid, SyncTerm newTerm); +int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); + #ifdef __cplusplus } #endif diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index a1cff2b738..9cb8a9d564 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -28,10 +28,6 @@ typedef struct SRaftId { SyncGroupId vgId; } SRaftId; -// ------------------ control ------------------- -SSyncNode* syncNodeAcquire(int64_t rid); -void syncNodeRelease(SSyncNode* pNode); - int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg); int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo); void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index bbddb539c6..563820f60b 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -451,6 +451,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) #define TSDB_CODE_WAL_CHKSUM_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x1006) +#define TSDB_CODE_WAL_LOG_INCOMPLETE TAOS_DEF_ERROR_CODE(0, 0x1007) // tfs #define TSDB_CODE_FS_INVLD_CFG TAOS_DEF_ERROR_CODE(0, 0x2201) diff --git a/include/util/tref.h b/include/util/tref.h index c2cc54cb07..c4b2ec8fa7 100644 --- a/include/util/tref.h +++ b/include/util/tref.h @@ -25,7 +25,8 @@ extern "C" { // open a reference set, max is the mod used by hash, fp is the pointer to free resource function // return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately -int32_t taosOpenRef(int32_t max, void (*fp)(void *)); +typedef void (*RefFp)(void *); +int32_t taosOpenRef(int32_t max, RefFp fp); // close the reference set, refId is the return value by taosOpenRef // return 0 if success. On error, -1 is returned, and terrno is set appropriately diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index c18e1e3e40..ba639476d8 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -508,7 +508,7 @@ static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pT pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb; - pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; + pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; // send msg atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); @@ -750,7 +750,7 @@ void tmqSendHbReq(void* param, void* tmrId) { sendInfo->requestObjRefId = 0; sendInfo->param = NULL; sendInfo->fp = tmqHbCb; - sendInfo->msgType = TDMT_MND_MQ_HB; + sendInfo->msgType = TDMT_MND_TMQ_HB; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -1038,7 +1038,7 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { sendInfo->requestObjRefId = 0; sendInfo->param = ¶m; sendInfo->fp = tmqSubscribeCb; - sendInfo->msgType = TDMT_MND_SUBSCRIBE; + sendInfo->msgType = TDMT_MND_TMQ_SUBSCRIBE; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -1420,7 +1420,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqAskEpCb; - sendInfo->msgType = TDMT_MND_MQ_ASK_EP; + sendInfo->msgType = TDMT_MND_TMQ_ASK_EP; SEpSet epSet = getEpSet_s(&tmq->pTscObj->pAppInfo->mgmtEp); @@ -1573,7 +1573,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqPollCb; - sendInfo->msgType = TDMT_VND_CONSUME; + sendInfo->msgType = TDMT_VND_TMQ_CONSUME; int64_t transporterId = 0; /*printf("send poll\n");*/ diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0c38d43543..b8436237c0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1132,6 +1132,10 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { pDataBlock->info.window.ekey = 0; pDataBlock->info.window.skey = 0; + if (pDataBlock->info.capacity == 0) { + return; + } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); @@ -1186,6 +1190,7 @@ static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* } void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { + pColumn->hasNull = false; if (IS_VAR_DATA_TYPE(pColumn->info.type)) { pColumn->varmeta.length = 0; if (pColumn->varmeta.offset != NULL) { @@ -2063,8 +2068,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB default: if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { if (colDataIsNull_s(pColInfoData, j)) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NULL, NULL, false, offset, - k); + tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NULL, NULL, false, + offset, k); } else if (pCol->type == pColInfoData->info.type) { tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, k); @@ -2137,10 +2142,26 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB return TSDB_CODE_SUCCESS; } -char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { - ASSERT(stbName[0] != 0); +char* buildCtbNameByGroupId(const char* stbFullName, uint64_t groupId) { + ASSERT(stbFullName[0] != 0); SArray* tags = taosArrayInit(0, sizeof(void*)); + if (tags == NULL) { + return NULL; + } + SSmlKv* pTag = taosMemoryCalloc(1, sizeof(SSmlKv)); + if (pTag == NULL) { + taosArrayDestroy(tags); + return NULL; + } + + void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); + if (cname == NULL) { + taosArrayDestroy(tags); + taosMemoryFree(pTag); + return NULL; + } + pTag->key = "group_id"; pTag->keyLen = strlen(pTag->key); pTag->type = TSDB_DATA_TYPE_UBIGINT; @@ -2148,13 +2169,11 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { pTag->length = sizeof(uint64_t); taosArrayPush(tags, &pTag); - void* cname = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + 1); - RandTableName rname = { .tags = tags, - .sTableName = stbName, - .sTableNameLen = strlen(stbName), - .childTableName = cname, + .stbFullName = stbFullName, + .stbFullNameLen = strlen(stbFullName), + .ctbShortName = cname, }; buildChildTableName(&rname); @@ -2162,8 +2181,8 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { taosMemoryFree(pTag); taosArrayDestroy(tags); - ASSERT(rname.childTableName && rname.childTableName[0]); - return rname.childTableName; + ASSERT(rname.ctbShortName && rname.ctbShortName[0]); + return rname.ctbShortName; } void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index fbb9e04a25..660730f0dc 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -163,7 +163,8 @@ int32_t tsTtlUnit = 86400; int32_t tsTtlPushInterval = 86400; int32_t tsGrantHBInterval = 60; int32_t tsUptimeInterval = 300; // seconds -char tsUdfdResFuncs[1024] = ""; // udfd resident funcs that teardown when udfd exits +char tsUdfdResFuncs[512] = ""; // udfd resident funcs that teardown when udfd exits +char tsUdfdLdLibPath[512] = ""; #ifndef _STORAGE int32_t taosSetTfsCfg(SConfig *pCfg) { @@ -424,6 +425,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, 0) != 0) return -1; + if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, 0) != 0) return -1; GRANT_CFG_ADD; return 0; } @@ -722,7 +724,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; tstrncpy(tsUdfdResFuncs, cfgGetItem(pCfg, "udfdResFuncs")->str, sizeof(tsUdfdResFuncs)); - + tstrncpy(tsUdfdLdLibPath, cfgGetItem(pCfg, "udfdLdLibPath")->str, sizeof(tsUdfdLdLibPath)); if (tsQueryBufferSize >= 0) { tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; } diff --git a/source/common/src/tname.c b/source/common/src/tname.c index 70682d6cd4..4d83b6e3d8 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -315,7 +315,7 @@ static int compareKv(const void* p1, const void* p2) { */ void buildChildTableName(RandTableName* rName) { SStringBuilder sb = {0}; - taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); + taosStringBuilderAppendStringLen(&sb, rName->stbFullName, rName->stbFullNameLen); taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { taosStringBuilderAppendChar(&sb, ','); @@ -336,11 +336,11 @@ void buildChildTableName(RandTableName* rName) { tMD5Final(&context); char temp[8] = {0}; - rName->childTableName[0] = 't'; - rName->childTableName[1] = '_'; + rName->ctbShortName[0] = 't'; + rName->ctbShortName[1] = '_'; for (int i = 0; i < 16; i++) { sprintf(temp, "%02x", context.digest[i]); - strcat(rName->childTableName, temp); + strcat(rName->ctbShortName, temp); } taosStringBuilderDestroy(&sb); rName->uid = *(uint64_t*)(context.digest); diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 423028b167..762b8fd529 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -13,4 +13,18 @@ target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" ) -target_link_libraries(taosd dnode) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(taosd jemalloc) + target_link_libraries(taosd dnode ${LINK_JEMALLOC}) +ELSE () + target_link_libraries(taosd dnode) +ENDIF () + diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 12e678886d..2c4a8e9b70 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -141,15 +141,13 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_GET_TABLE_INDEX, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_MQ_COMMIT_OFFSET, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_CREATE_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_TOPIC, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_SUBSCRIBE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_ASK_EP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_HB, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TMQ_DROP_CGROUP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_TRANS, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_QUERY, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_KILL_CONN, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -171,10 +169,10 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ADD_CHECK_INFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE_CHECK_INFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index e0a39a6bf1..e50b527bac 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -67,24 +67,6 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { taosFreeQitem(pMsg); } -static void mmProcessSyncCtrlMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SMnodeMgmt *pMgmt = pInfo->ahandle; - pMsg->info.node = pMgmt->pMnode; - - const STraceId *trace = &pMsg->info.traceId; - dGTrace("msg:%p, get from mnode-sync-ctrl queue", pMsg); - - SMsgHead *pHead = pMsg->pCont; - pHead->contLen = ntohl(pHead->contLen); - pHead->vgId = ntohl(pHead->vgId); - - int32_t code = mndProcessSyncCtrlMsg(pMsg); - - dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); -} - static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; pMsg->info.node = pMgmt->pMnode; @@ -252,7 +234,7 @@ int32_t mmStartWorker(SMnodeMgmt *pMgmt) { .min = 1, .max = 1, .name = "mnode-sync-ctrl", - .fp = (FItem)mmProcessSyncCtrlMsg, + .fp = (FItem)mmProcessSyncMsg, .param = pMgmt, }; if (tSingleWorkerInit(&pMgmt->syncCtrlWorker, &scCfg) != 0) { diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 1ce7f1a84b..7ecb6fb208 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -72,8 +72,6 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d682fa9cc5..e2812e0d31 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -392,12 +392,12 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_SMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_SMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_SUBMIT_RSMA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_CHANGE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_ADD_CHECK_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE_CHECK_INFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -409,10 +409,9 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RECOVER_FINISH, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index d5b1e18c7b..71e2ece733 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -133,22 +133,6 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf } } -static void vmProcessSyncCtrlQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { - SVnodeObj *pVnode = pInfo->ahandle; - SRpcMsg *pMsg = NULL; - - for (int32_t i = 0; i < numOfMsgs; ++i) { - if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; - const STraceId *trace = &pMsg->info.traceId; - dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg); - - int32_t code = vnodeProcessSyncCtrlMsg(pVnode->pImpl, pMsg, NULL); // no response here - dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); - rpcFreeCont(pMsg->pCont); - taosFreeQitem(pMsg); - } -} - static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { const STraceId *trace = &pMsg->info.traceId; SMsgHead *pHead = pMsg->pCont; @@ -317,7 +301,7 @@ int32_t vmGetQueueSize(SVnodeMgmt *pMgmt, int32_t vgId, EQueueType qtype) { int32_t vmAllocQueue(SVnodeMgmt *pMgmt, SVnodeObj *pVnode) { pVnode->pWriteQ = tWWorkerAllocQueue(&pMgmt->writePool, pVnode->pImpl, (FItems)vnodeProposeWriteMsg); pVnode->pSyncQ = tWWorkerAllocQueue(&pMgmt->syncPool, pVnode, (FItems)vmProcessSyncQueue); - pVnode->pSyncCtrlQ = tWWorkerAllocQueue(&pMgmt->syncCtrlPool, pVnode, (FItems)vmProcessSyncCtrlQueue); + pVnode->pSyncCtrlQ = tWWorkerAllocQueue(&pMgmt->syncCtrlPool, pVnode, (FItems)vmProcessSyncQueue); pVnode->pApplyQ = tWWorkerAllocQueue(&pMgmt->applyPool, pVnode->pImpl, (FItems)vnodeApplyWriteMsg); pVnode->pQueryQ = tQWorkerAllocQueue(&pMgmt->queryPool, pVnode, (FItem)vmProcessQueryQueue); pVnode->pStreamQ = tQWorkerAllocQueue(&pMgmt->streamPool, pVnode, (FItem)vmProcessStreamQueue); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index fa3651c7d7..aee1a7a75c 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -63,12 +63,12 @@ int32_t mndInitConsumer(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndConsumerActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_HB, mndProcessMqHbReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_ASK_EP, mndProcessAskEpReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_SUBSCRIBE, mndProcessSubscribeReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_HB, mndProcessMqHbReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_ASK_EP, mndProcessAskEpReq); mndSetMsgHandle(pMnode, TDMT_MND_MQ_TIMER, mndProcessMqTimerMsg); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_LOST, mndProcessConsumerLostMsg); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_LOST, mndProcessConsumerLostMsg); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CONSUMER_RECOVER, mndProcessConsumerRecoverMsg); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndRetrieveConsumer); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONSUMERS, mndCancelGetNextConsumer); @@ -207,7 +207,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { pLostMsg->consumerId = pConsumer->consumerId; SRpcMsg pRpcMsg = { - .msgType = TDMT_MND_MQ_CONSUMER_LOST, + .msgType = TDMT_MND_TMQ_CONSUMER_LOST, .pCont = pLostMsg, .contLen = sizeof(SMqConsumerLostMsg), }; @@ -256,7 +256,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { if (taosHashGetSize(pRebMsg->rebSubHash) != 0) { mInfo("mq rebalance will be triggered"); SRpcMsg rpcMsg = { - .msgType = TDMT_MND_MQ_DO_REBALANCE, + .msgType = TDMT_MND_TMQ_DO_REBALANCE, .pCont = pRebMsg, .contLen = sizeof(SMqDoRebalanceMsg), }; @@ -292,7 +292,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) { pRecoverMsg->consumerId = consumerId; SRpcMsg pRpcMsg = { - .msgType = TDMT_MND_MQ_CONSUMER_RECOVER, + .msgType = TDMT_MND_TMQ_CONSUMER_RECOVER, .pCont = pRecoverMsg, .contLen = sizeof(SMqConsumerRecoverMsg), }; @@ -331,7 +331,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { pRecoverMsg->consumerId = consumerId; SRpcMsg pRpcMsg = { - .msgType = TDMT_MND_MQ_CONSUMER_RECOVER, + .msgType = TDMT_MND_TMQ_CONSUMER_RECOVER, .pCont = pRecoverMsg, .contLen = sizeof(SMqConsumerRecoverMsg), }; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 98a24286f6..891f2bbcd8 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -474,133 +474,18 @@ void mndStop(SMnode *pMnode) { mndCleanupTimer(pMnode); } -int32_t mndProcessSyncCtrlMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SSyncMgmt *pMgmt = &pMnode->syncMgmt; - int32_t code = 0; - - mInfo("vgId:%d, process sync ctrl msg", 1); - - if (!syncEnvIsStart()) { - mError("failed to process sync msg:%p type:%s since syncEnv stop", pMsg, TMSG_INFO(pMsg->msgType)); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - - SSyncNode *pSyncNode = syncNodeAcquire(pMgmt->sync); - if (pSyncNode == NULL) { - mError("failed to process sync msg:%p type:%s since syncNode is null", pMsg, TMSG_INFO(pMsg->msgType)); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - - if (pMsg->msgType == TDMT_SYNC_HEARTBEAT) { - SyncHeartbeat *pSyncMsg = syncHeartbeatFromRpcMsg2(pMsg); - code = syncNodeOnHeartbeat(pSyncNode, pSyncMsg); - syncHeartbeatDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_HEARTBEAT_REPLY) { - SyncHeartbeatReply *pSyncMsg = syncHeartbeatReplyFromRpcMsg2(pMsg); - code = syncNodeOnHeartbeatReply(pSyncNode, pSyncMsg); - syncHeartbeatReplyDestroy(pSyncMsg); - } - - syncNodeRelease(pSyncNode); - - if (code != 0) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - } - return code; -} - int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SSyncMgmt *pMgmt = &pMnode->syncMgmt; - int32_t code = 0; - if (!syncEnvIsStart()) { - mError("failed to process sync msg:%p type:%s since syncEnv stop", pMsg, TMSG_INFO(pMsg->msgType)); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - - SSyncNode *pSyncNode = syncNodeAcquire(pMgmt->sync); - if (pSyncNode == NULL) { - mError("failed to process sync msg:%p type:%s since syncNode is null", pMsg, TMSG_INFO(pMsg->msgType)); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - - if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { - SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); - code = syncNodeOnTimer(pSyncNode, pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_PING) { - SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); - code = syncNodeOnPing(pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); - code = syncNodeOnPingReply(pSyncNode, pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { - SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); - code = syncNodeOnRequestVote(pSyncNode, pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); - code = syncNodeOnRequestVoteReply(pSyncNode, pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { - SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); - code = syncNodeOnAppendEntries(pSyncNode, pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); - code = syncNodeOnAppendEntriesReply(pSyncNode, pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { - SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); - code = syncNodeOnSnapshot(pSyncNode, pSyncMsg); - syncSnapshotSendDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { - SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); - code = syncNodeOnSnapshotReply(pSyncNode, pSyncMsg); - syncSnapshotRspDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_SET_MNODE_STANDBY) { - code = syncSetStandby(pMgmt->sync); - SRpcMsg rsp = {.code = code, .info = pMsg->info}; - tmsgSendRsp(&rsp); - - } else if (pMsg->msgType == TDMT_SYNC_LOCAL_CMD) { - SyncLocalCmd *pSyncMsg = syncLocalCmdFromRpcMsg2(pMsg); - code = syncNodeOnLocalCmd(pSyncNode, pSyncMsg); - syncLocalCmdDestroy(pSyncMsg); - - } else { - mError("failed to process msg:%p since invalid type:%s", pMsg, TMSG_INFO(pMsg->msgType)); - code = -1; - } - - syncNodeRelease(pSyncNode); + const STraceId *trace = &pMsg->info.traceId; + mGTrace("vgId:1, sync msg:%p will be processed, type:%s", pMsg, TMSG_INFO(pMsg->msgType)); + int32_t code = syncProcessMsg(pMgmt->sync, pMsg); if (code != 0) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + mGError("vgId:1, failed to process sync msg:%p type:%s since %s", pMsg, TMSG_INFO(pMsg->msgType), terrstr()); } + return code; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 5ee4925a96..7e7c6ee0b6 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -487,6 +487,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { qDestroyQueryPlan(pPlan); return -1; } + pTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(taskOneLevel, pTask); // source diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 3ff989e670..9f433f3322 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -557,78 +557,6 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } -#if 0 -static int32_t mndPersistTaskRecoverReq(STrans *pTrans, SStreamTask *pTask) { - SMStreamTaskRecoverReq *pReq = taosMemoryCalloc(1, sizeof(SMStreamTaskRecoverReq)); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; - int32_t len; - int32_t code; - tEncodeSize(tEncodeSMStreamTaskRecoverReq, pReq, len, code); - if (code != 0) { - return -1; - } - void *buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - if (buf == NULL) { - return -1; - } - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - SEncoder encoder; - tEncoderInit(&encoder, abuf, len); - tEncodeSMStreamTaskRecoverReq(&encoder, pReq); - ((SMsgHead *)buf)->vgId = pTask->nodeId; - - STransAction action = {0}; - memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); - action.pCont = buf; - action.contLen = sizeof(SMsgHead) + len; - action.msgType = TDMT_STREAM_TASK_RECOVER; - if (mndTransAppendRedoAction(pTrans, &action) != 0) { - taosMemoryFree(buf); - return -1; - } - return 0; -} - -int32_t mndRecoverStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { - if (pStream->isDistributed) { - int32_t lv = taosArrayGetSize(pStream->tasks); - for (int32_t i = 0; i < lv; i++) { - SArray *pTasks = taosArrayGetP(pStream->tasks, i); - int32_t sz = taosArrayGetSize(pTasks); - SStreamTask *pTask = taosArrayGetP(pTasks, 0); - if (pTask->taskLevel == TASK_LEVEL__AGG) { - ASSERT(sz == 1); - if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { - return -1; - } - } else { - continue; - } - } - } else { - int32_t lv = taosArrayGetSize(pStream->tasks); - for (int32_t i = 0; i < lv; i++) { - SArray *pTasks = taosArrayGetP(pStream->tasks, i); - int32_t sz = taosArrayGetSize(pTasks); - for (int32_t j = 0; j < sz; j++) { - SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (pTask->taskLevel != TASK_LEVEL__SOURCE) break; - ASSERT(pTask->taskLevel != TASK_LEVEL__SINK); - if (mndPersistTaskRecoverReq(pTrans, pTask) < 0) { - return -1; - } - } - } - } - return 0; -} -#endif - int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index a6661e73c9..74f2b1288e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -73,11 +73,11 @@ int32_t mndInitSubscribe(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndSubActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_CHANGE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_VND_MQ_VG_DELETE_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DO_REBALANCE, mndProcessRebalanceReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP, mndProcessDropCgroupReq); - mndSetMsgHandle(pMnode, TDMT_MND_MQ_DROP_CGROUP_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_SUBSCRIBE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_DELETE_SUB_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DO_REBALANCE, mndProcessRebalanceReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP, mndProcessDropCgroupReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_CGROUP_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_SUBSCRIPTIONS, mndRetrieveSubscribe); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextSubscribe); @@ -164,7 +164,7 @@ static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, const SM action.epSet = mndGetVgroupEpset(pMnode, pVgObj); action.pCont = buf; action.contLen = tlen; - action.msgType = TDMT_VND_MQ_VG_CHANGE; + action.msgType = TDMT_VND_TMQ_SUBSCRIBE; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -920,7 +920,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) action.epSet = pVgEp->epSet; action.pCont = pReq; action.contLen = sizeof(SMqVDeleteReq); - action.msgType = TDMT_VND_MQ_VG_DELETE; + action.msgType = TDMT_VND_TMQ_DELETE_SUB; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index df4b526775..6853ff5ccc 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -17,11 +17,44 @@ #include "mndSync.h" #include "mndTrans.h" -static int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { +static int32_t mndSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + SMsgHead *pHead = pMsg->pCont; pHead->contLen = htonl(pHead->contLen); pHead->vgId = htonl(pHead->vgId); + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + return -1; + } + + int32_t code = tmsgPutToQueue(msgcb, SYNC_CTRL_QUEUE, pMsg); + if (code != 0) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + } + return code; +} + +static int32_t mndSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + SMsgHead *pHead = pMsg->pCont; + pHead->contLen = htonl(pHead->contLen); + pHead->vgId = htonl(pHead->vgId); + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; + return -1; + } + int32_t code = tmsgPutToQueue(msgcb, SYNC_QUEUE, pMsg); if (code != 0) { rpcFreeCont(pMsg->pCont); @@ -212,7 +245,7 @@ int32_t mndInitSync(SMnode *pMnode) { .msgcb = NULL, .FpSendMsg = mndSyncSendMsg, .FpEqMsg = mndSyncEqMsg, - .FpEqCtrlMsg = NULL, + .FpEqCtrlMsg = mndSyncEqCtrlMsg, }; snprintf(syncInfo.path, sizeof(syncInfo.path), "%s%ssync", pMnode->path, TD_DIRSEP); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 324404ce1b..e1ca1d2708 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -53,11 +53,10 @@ int32_t mndInitTopic(SMnode *pMnode) { .deleteFp = (SdbDeleteFp)mndTopicActionDelete, }; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); - mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_VND_ADD_CHECK_INFO_RSP, mndTransProcessRsp); - mndSetMsgHandle(pMnode, TDMT_VND_DELETE_CHECK_INFO_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_CREATE_TOPIC, mndProcessCreateTopicReq); + mndSetMsgHandle(pMnode, TDMT_MND_TMQ_DROP_TOPIC, mndProcessDropTopicReq); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_ADD_CHECKINFO_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_TMQ_DEL_CHECKINFO_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndRetrieveTopic); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TOPICS, mndCancelGetNextTopic); @@ -506,7 +505,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = buf; action.contLen = sizeof(SMsgHead) + len; - action.msgType = TDMT_VND_ADD_CHECK_INFO; + action.msgType = TDMT_VND_TMQ_ADD_CHECKINFO; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); @@ -715,7 +714,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { action.epSet = mndGetVgroupEpset(pMnode, pVgroup); action.pCont = buf; action.contLen = sizeof(SMsgHead) + TSDB_TOPIC_FNAME_LEN; - action.msgType = TDMT_VND_DELETE_CHECK_INFO; + action.msgType = TDMT_VND_TMQ_DEL_CHECKINFO; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); sdbRelease(pSdb, pVgroup); diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 8dbfc14841..7e6bd75b2d 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -90,12 +90,12 @@ int32_t qndProcessQueryMsg(SQnode *pQnode, int64_t ts, SRpcMsg *pMsg) { code = qWorkerProcessFetchMsg(pQnode, pQnode->pQuery, pMsg, ts); break; case TDMT_SCH_CANCEL_TASK: - //code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts); + // code = qWorkerProcessCancelMsg(pQnode, pQnode->pQuery, pMsg, ts); break; case TDMT_SCH_DROP_TASK: code = qWorkerProcessDropMsg(pQnode, pQnode->pQuery, pMsg, ts); break; - case TDMT_VND_CONSUME: + case TDMT_VND_TMQ_CONSUME: // code = tqProcessConsumeReq(pQnode->pTq, pMsg); // break; case TDMT_SCH_QUERY_HEARTBEAT: diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 719e2d8fad..d34159d312 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -261,18 +261,52 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { return 0; } +int32_t sndProcessTaskRecoverFinishReq(SSnode *pSnode, SRpcMsg *pMsg) { + char *msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + + // deserialize + SStreamRecoverFinishReq req; + + SDecoder decoder; + tDecoderInit(&decoder, msg, msgLen); + tDecodeSStreamRecoverFinishReq(&decoder, &req); + tDecoderClear(&decoder); + + // find task + SStreamTask *pTask = streamMetaGetTask(pSnode->pMeta, req.taskId); + if (pTask == NULL) { + return -1; + } + // do process request + if (streamProcessRecoverFinishReq(pTask, req.childId) < 0) { + return -1; + } + + return 0; +} + +int32_t sndProcessTaskRecoverFinishRsp(SSnode *pSnode, SRpcMsg *pMsg) { + // + return 0; +} + int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_RUN: return sndProcessTaskRunReq(pSnode, pMsg); case TDMT_STREAM_TASK_DISPATCH: return sndProcessTaskDispatchReq(pSnode, pMsg, true); - case TDMT_STREAM_RETRIEVE: - return sndProcessTaskRetrieveReq(pSnode, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return sndProcessTaskDispatchRsp(pSnode, pMsg); + case TDMT_STREAM_RETRIEVE: + return sndProcessTaskRetrieveReq(pSnode, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return sndProcessTaskRetrieveRsp(pSnode, pMsg); + case TDMT_STREAM_RECOVER_FINISH: + return sndProcessTaskRecoverFinishReq(pSnode, pMsg); + case TDMT_STREAM_RECOVER_FINISH_RSP: + return sndProcessTaskRecoverFinishRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6200c88d75..020d2b6049 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -83,7 +83,6 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg *pRsp); int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); -int32_t vnodeProcessSyncCtrlMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp); int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); @@ -159,7 +158,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableL void tsdbReaderClose(STsdbReader *pReader); bool tsdbNextDataBlock(STsdbReader *pReader); bool tsdbTableNextDataBlock(STsdbReader *pReader, uint64_t uid); -void tsdbRetrieveDataBlockInfo(STsdbReader *pReader, SDataBlockInfo *pDataBlockInfo); +void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SColumnDataAgg ***pBlockStatis, bool *allHave); SArray *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 94178727be..9ec8bb2cfd 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -176,8 +176,8 @@ int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId); // tq-mq int32_t tqProcessAddCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessVgDeleteReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); // tq-stream @@ -187,11 +187,15 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); -int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); -int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); +// int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); +// int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg); SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchema* pSchema, SSchemaWrapper* pTagSchemaWrapper, bool createTb, int64_t suid, const char* stbFullName, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2c01645389..47a40ea495 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -709,7 +709,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { return 0; } -int32_t tqProcessVgDeleteReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { +int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; taosWLockLatch(&pTq->pushLock); @@ -767,7 +767,7 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t m return 0; } -int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { +int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { SMqRebVgReq req = {0}; tDecodeSMqRebVgReq(msg, &req); // todo lock @@ -982,25 +982,33 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg // 3.go through recover steps to fill history if (pTask->fillHistory) { - streamSetParamForRecover(pTask); if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + streamSetParamForRecover(pTask); streamSourceRecoverPrepareStep1(pTask, version); SStreamRecoverStep1Req req; streamBuildSourceRecover1Req(pTask, &req); - - void* serialziedReq = (void*)&req; int32_t len = sizeof(SStreamRecoverStep1Req); + void* serializedReq = rpcMallocCont(len); + if (serializedReq == NULL) { + return -1; + } + + memcpy(serializedReq, &req, len); + SRpcMsg rpcMsg = { .contLen = len, - .pCont = serialziedReq, + .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_STEP1, }; - tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &rpcMsg); + if (tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &rpcMsg) < 0) { + /*ASSERT(0);*/ + } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { + streamSetParamForRecover(pTask); streamAggRecoverPrepare(pTask); } else if (pTask->taskLevel == TASK_LEVEL__SINK) { // do nothing @@ -1010,8 +1018,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg return 0; } -int32_t tqProcessTaskRecover1Req(STQ* pTq, char* msg, int32_t msgLen) { - int32_t code; +int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { + int32_t code; + char* msg = pMsg->pCont; + int32_t msgLen = pMsg->contLen; + SStreamRecoverStep1Req* pReq = (SStreamRecoverStep1Req*)msg; SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, pReq->taskId); if (pTask == NULL) { @@ -1035,16 +1046,24 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, char* msg, int32_t msgLen) { return -1; } + ASSERT(pReq->taskId == pTask->taskId); + // serialize msg - int32_t len = sizeof(SStreamRecoverStep2Req); - void* serializedReq = (void*)&req; + int32_t len = sizeof(SStreamRecoverStep1Req); + + void* serializedReq = rpcMallocCont(len); + if (serializedReq == NULL) { + return -1; + } + + memcpy(serializedReq, &req, len); // dispatch msg SRpcMsg rpcMsg = { .code = 0, .contLen = len, .msgType = TDMT_VND_STREAM_RECOVER_STEP2, - .pCont = (void*)serializedReq, + .pCont = serializedReq, }; tmsgPutToQueue(&pTq->pVnode->msgCb, WRITE_QUEUE, &rpcMsg); @@ -1087,15 +1106,15 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t version, char* msg, int32_t m return 0; } -int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, char* msg, int32_t msgLen) { - int32_t code; +int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, SRpcMsg* pMsg) { + char* msg = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); // deserialize - int32_t len; SStreamRecoverFinishReq req; SDecoder decoder; - tDecoderInit(&decoder, msg, sizeof(SStreamRecoverFinishReq)); + tDecoderInit(&decoder, msg, msgLen); tDecodeSStreamRecoverFinishReq(&decoder, &req); tDecoderClear(&decoder); @@ -1112,6 +1131,11 @@ int32_t tqProcessTaskRecoverFinishReq(STQ* pTq, char* msg, int32_t msgLen) { return 0; } +int32_t tqProcessTaskRecoverFinishRsp(STQ* pTq, SRpcMsg* pMsg) { + // + return 0; +} + int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { bool failed = false; SDecoder* pCoder = &(SDecoder){0}; @@ -1235,7 +1259,7 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { pSubmit = streamDataSubmitNew(pReq); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - qError("failed to create data submit for stream since out of memory"); + tqError("failed to create data submit for stream since out of memory"); failed = true; } @@ -1244,18 +1268,21 @@ int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__RECOVER1) continue; + if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__RECOVER1) { + tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus); + continue; + } - qDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { - qError("stream task input failed, task id %d", pTask->taskId); + tqError("stream task input failed, task id %d", pTask->taskId); continue; } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + tqError("stream task launch failed, task id %d", pTask->taskId); continue; } } else { @@ -1306,33 +1333,6 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { } } -#if 0 -int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { - SStreamTaskRecoverReq* pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; - SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessRecoverReq(pTask, pReq, pMsg); - return 0; - } else { - return -1; - } -} - -int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { - SStreamTaskRecoverRsp* pRsp = pMsg->pCont; - int32_t taskId = pRsp->rspTaskId; - - SStreamTask* pTask = streamMetaGetTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessRecoverRsp(pTask, pRsp); - return 0; - } else { - return -1; - } -} -#endif - int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = pRsp->taskId; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5f90c567be..a963b8eb8b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -35,11 +35,17 @@ typedef struct { int32_t numOfLastFiles; } SBlockNumber; +typedef struct SBlockIndex { + int32_t ordinalIndex; + int64_t inFileOffset; + STimeWindow window; +} SBlockIndex; + typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; SMapData mapData; // block info (compressed) - SArray* pBlockList; // block data index list + SArray* pBlockList; // block data index list, SArray SIterInfo iter; // mem buffer skip list iterator SIterInfo iiter; // imem buffer skip list iterator SArray* delSkyline; // delete info for this table @@ -72,6 +78,7 @@ typedef struct SIOCostSummary { double lastBlockLoadTime; int64_t composedBlocks; double buildComposedBlockTime; + double createScanInfoList; } SIOCostSummary; typedef struct SBlockLoadSuppInfo { @@ -234,6 +241,8 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK return NULL; } + int64_t st = taosGetTimestampUs(); + for (int32_t j = 0; j < numOfTables; ++j) { STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { @@ -249,8 +258,10 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK pTsdbReader->idStr); } - tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, %s", pTsdbReader, numOfTables, - (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->idStr); + pTsdbReader->cost.createScanInfoList = (taosGetTimestampUs() - st) / 1000.0; + tsdbDebug("%p create %d tables scan-info, size:%.2f Kb, elapsed time:%.2f ms, %s", pTsdbReader, numOfTables, + (sizeof(STableBlockScanInfo) * numOfTables) / 1024.0, pTsdbReader->cost.createScanInfoList, + pTsdbReader->idStr); return pTableMap; } @@ -568,7 +579,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, STableBlockScanInfo* pScanInfo = p; if (pScanInfo->pBlockList == NULL) { - pScanInfo->pBlockList = taosArrayInit(4, sizeof(int32_t)); + pScanInfo->pBlockList = taosArrayInit(4, sizeof(SBlockIndex)); } taosArrayPush(pIndexList, pBlockIdx); @@ -630,7 +641,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN continue; } - void* p = taosArrayPush(pScanInfo->pBlockList, &j); + SBlockIndex bIndex = {.ordinalIndex = j, .inFileOffset = block.aSubBlock->offset}; + bIndex.window = (STimeWindow) {.skey = block.minKey.ts, .ekey = block.maxKey.ts}; + + void* p = taosArrayPush(pScanInfo->pBlockList, &bIndex); if (p == NULL) { tMapDataClear(&pScanInfo->mapData); return TSDB_CODE_OUT_OF_MEMORY; @@ -821,7 +835,8 @@ int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SData } else if (!asc && pReader->window.skey <= pBlock->minKey.ts) { endPos = 0; } else { - endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.ekey, pReader->order); + int64_t key = asc ? pReader->window.ekey : pReader->window.skey; + endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, pReader->order); } return endPos; @@ -852,8 +867,9 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // pDumpInfo->rowIndex = pBlock->nRow - 1; } else { int32_t pos = asc ? pBlock->nRow - 1 : 0; - int32_t order = (pReader->order == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.skey, order); + int32_t order = asc ? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int64_t key = asc ? pReader->window.skey : pReader->window.ekey; + pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, key, order); } } @@ -1059,8 +1075,8 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) return TSDB_CODE_INVALID_PARA; } - int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); - tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk); + SBlockIndex* pIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); + tMapDataGetItemByIdx(&pScanInfo->mapData, pIndex->ordinalIndex, &pBlockIter->block, tGetDataBlk); } #if 0 @@ -1073,6 +1089,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { bool asc = ASCENDING_TRAVERSE(pReader->order); + SBlockOrderSupporter sup = {0}; pBlockIter->numOfBlocks = numOfBlocks; taosArrayClear(pBlockIter->blockList); pBlockIter->pTableMap = pReader->status.pTableMap; @@ -1081,9 +1098,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); int64_t st = taosGetTimestampUs(); - - SBlockOrderSupporter sup = {0}; - int32_t code = initBlockOrderSupporter(&sup, numOfTables); + int32_t code = initBlockOrderSupporter(&sup, numOfTables); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1111,17 +1126,11 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; - SDataBlk block = {0}; + for (int32_t k = 0; k < num; ++k) { - SBlockOrderWrapper wrapper = {0}; - - int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k); - tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetDataBlk); - - wrapper.uid = pTableScanInfo->uid; - wrapper.offset = block.aSubBlock[0].offset; - - sup.pDataBlockInfo[sup.numOfTables][k] = wrapper; + SBlockIndex* pIndex = taosArrayGet(pTableScanInfo->pBlockList, k); + sup.pDataBlockInfo[sup.numOfTables][k] = + (SBlockOrderWrapper){.uid = pTableScanInfo->uid, .offset = pIndex->inFileOffset}; cnt++; } @@ -1212,25 +1221,22 @@ static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* p (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } -static SDataBlk* getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, - int32_t* nextIndex, int32_t order) { +static bool getNeighborBlockOfSameTable(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, + int32_t* nextIndex, int32_t order, SBlockIndex* pBlockIndex) { bool asc = ASCENDING_TRAVERSE(order); if (asc && pBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { - return NULL; + return false; } if (!asc && pBlockInfo->tbBlockIdx == 0) { - return NULL; + return false; } int32_t step = asc ? 1 : -1; *nextIndex = pBlockInfo->tbBlockIdx + step; - - SDataBlk* pBlock = taosMemoryCalloc(1, sizeof(SDataBlk)); - int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); - - tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk); - return pBlock; + *pBlockIndex = *(SBlockIndex*) taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); +// tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, pIndex->ordinalIndex, pBlock, tGetDataBlk); + return true; } static int32_t findFileBlockInfoIndex(SDataBlockIter* pBlockIter, SFileDataBlockInfo* pFBlockInfo) { @@ -1272,12 +1278,12 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t return TSDB_CODE_SUCCESS; } -static bool overlapWithNeighborBlock(SDataBlk* pBlock, SDataBlk* pNeighbor, int32_t order) { +static bool overlapWithNeighborBlock(SDataBlk* pBlock, SBlockIndex* pNeighborBlockIndex, int32_t order) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { - return pBlock->maxKey.ts == pNeighbor->minKey.ts; + return pBlock->maxKey.ts == pNeighborBlockIndex->window.skey; } else { - return pBlock->minKey.ts == pNeighbor->maxKey.ts; + return pBlock->minKey.ts == pNeighborBlockIndex->window.ekey; } } @@ -1364,13 +1370,14 @@ typedef struct { static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, STsdbReader* pReader) { - int32_t neighborIndex = 0; - SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order); + int32_t neighborIndex = 0; + SBlockIndex bIndex = {0}; + + bool hasNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order, &bIndex); // overlap with neighbor - if (pNeighbor) { - pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order); - taosMemoryFree(pNeighbor); + if (hasNeighbor) { + pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, &bIndex, pReader->order); } // has duplicated ts of different version in this block @@ -3067,15 +3074,15 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int32_t nextIndex = -1; - SDataBlk* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); - if (pNeighborBlock == NULL) { // do nothing + int32_t nextIndex = -1; + SBlockIndex bIndex = {0}; + + bool hasNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order, &bIndex); + if (!hasNeighbor) { // do nothing return 0; } - bool overlap = overlapWithNeighborBlock(pBlock, pNeighborBlock, pReader->order); - taosMemoryFree(pNeighborBlock); - + bool overlap = overlapWithNeighborBlock(pBlock, &bIndex, pReader->order); if (overlap) { // load next block SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -3496,7 +3503,6 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order); -// resetDataBlockScanInfo(pReader->status.pTableMap, pReader->window.skey); // no data in files, let's try buffer in memory if (pReader->status.fileIter.numOfFiles == 0) { @@ -3697,15 +3703,16 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pLReader); } - tsdbDebug( - "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, fileBlocks:%" PRId64 - ", fileBlocks-load-time:%.2f ms, " - "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 - ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb %s", - pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, - pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, - pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); + tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 + " SMA-time:%.2f ms, fileBlocks:%" PRId64 + ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 + ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb, creatTime:%.2f ms, %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, + pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, + pCost->lastBlockLoadTime, pCost->composedBlocks, pCost->buildComposedBlockTime, + numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pCost->createScanInfoList, pReader->idStr); taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); @@ -3799,24 +3806,24 @@ bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) { return true; } -static void setBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { - ASSERT(pDataBlockInfo != NULL && pReader != NULL); - pDataBlockInfo->rows = pReader->pResBlock->info.rows; - pDataBlockInfo->uid = pReader->pResBlock->info.uid; - pDataBlockInfo->window = pReader->pResBlock->info.window; +static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) { + ASSERT(pReader != NULL); + *rows = pReader->pResBlock->info.rows; + *uid = pReader->pResBlock->info.uid; + *pWindow = pReader->pResBlock->info.window; } -void tsdbRetrieveDataBlockInfo(STsdbReader* pReader, SDataBlockInfo* pDataBlockInfo) { +void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) { if (pReader->type == TIMEWINDOW_RANGE_EXTERNAL) { if (pReader->step == EXTERNAL_ROWS_MAIN) { - setBlockInfo(pReader, pDataBlockInfo); + setBlockInfo(pReader, rows, uid, pWindow); } else if (pReader->step == EXTERNAL_ROWS_PREV) { - setBlockInfo(pReader->innerReader[0], pDataBlockInfo); + setBlockInfo(pReader->innerReader[0], rows, uid, pWindow); } else { - setBlockInfo(pReader->innerReader[1], pDataBlockInfo); + setBlockInfo(pReader->innerReader[1], rows, uid, pWindow); } } else { - setBlockInfo(pReader, pDataBlockInfo); + setBlockInfo(pReader, rows, uid, pWindow); } } @@ -3838,7 +3845,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); - int64_t stime = taosGetTimestampUs(); +// int64_t stime = taosGetTimestampUs(); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; @@ -3869,7 +3876,9 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS size_t numOfCols = blockDataGetNumOfCols(pReader->pResBlock); int32_t i = 0, j = 0; - while (j < numOfCols && i < taosArrayGetSize(pSup->pColAgg)) { + size_t size = taosArrayGetSize(pSup->pColAgg); + + while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i); if (pAgg->colId == pSup->colIds[j]) { if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) { @@ -3886,14 +3895,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS } } - double elapsed = (taosGetTimestampUs() - stime) / 1000.0; - pReader->cost.smaLoadTime += elapsed; pReader->cost.smaDataLoad += 1; - *pBlockStatis = pSup->plist; - tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", elapsed time:%.2f ms, %s", 0, pFBlock->uid, - elapsed, pReader->idStr); + tsdbDebug("vgId:%d, succeed to load block SMA for uid %" PRIu64 ", %s", 0, pFBlock->uid, pReader->idStr); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4885dc85b1..cfedd70711 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -228,30 +228,30 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp if (vnodeProcessBatchDeleteReq(pVnode, version, pReq, len, pRsp) < 0) goto _err; break; /* TQ */ - case TDMT_VND_MQ_VG_CHANGE: - if (tqProcessVgChangeReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), - pMsg->contLen - sizeof(SMsgHead)) < 0) { + case TDMT_VND_TMQ_SUBSCRIBE: + if (tqProcessSubscribeReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), + pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; - case TDMT_VND_MQ_VG_DELETE: - if (tqProcessVgDeleteReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + case TDMT_VND_TMQ_DELETE_SUB: + if (tqProcessDeleteSubReq(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } break; - case TDMT_VND_MQ_COMMIT_OFFSET: + case TDMT_VND_TMQ_COMMIT_OFFSET: if (tqProcessOffsetCommitReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; - case TDMT_VND_ADD_CHECK_INFO: + case TDMT_VND_TMQ_ADD_CHECKINFO: if (tqProcessAddCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; } break; - case TDMT_VND_DELETE_CHECK_INFO: + case TDMT_VND_TMQ_DEL_CHECKINFO: if (tqProcessDelCheckInfoReq(pVnode->pTq, version, POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), pMsg->contLen - sizeof(SMsgHead)) < 0) { goto _err; @@ -268,6 +268,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } } break; + case TDMT_VND_STREAM_RECOVER_STEP2: { + if (tqProcessTaskRecover2Req(pVnode->pTq, version, pMsg->pCont, pMsg->contLen) < 0) { + goto _err; + } + } break; case TDMT_VND_ALTER_CONFIRM: vnodeProcessAlterConfirmReq(pVnode, version, pReq, len, pRsp); break; @@ -356,14 +361,11 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return 0; } - if (pMsg->msgType == TDMT_VND_CONSUME && !pVnode->restored) { + if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) { vnodeRedirectRpcMsg(pVnode, pMsg); return 0; } - char *msgstr = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); - int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); - switch (pMsg->msgType) { case TDMT_SCH_FETCH: case TDMT_SCH_MERGE_FETCH: @@ -382,7 +384,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetTableCfg(pVnode, pMsg, true); case TDMT_VND_BATCH_META: return vnodeGetBatchMeta(pVnode, pMsg); - case TDMT_VND_CONSUME: + case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); @@ -390,16 +392,18 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); #endif - /*case TDMT_STREAM_TASK_RECOVER:*/ - /*return tqProcessTaskRecoverReq(pVnode->pTq, pMsg);*/ - case TDMT_STREAM_RETRIEVE: - return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); - /*case TDMT_STREAM_TASK_RECOVER_RSP:*/ - /*return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg);*/ + case TDMT_STREAM_RETRIEVE: + return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_RETRIEVE_RSP: return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); + case TDMT_VND_STREAM_RECOVER_STEP1: + return tqProcessTaskRecover1Req(pVnode->pTq, pMsg); + case TDMT_STREAM_RECOVER_FINISH: + return tqProcessTaskRecoverFinishReq(pVnode->pTq, pMsg); + case TDMT_STREAM_RECOVER_FINISH_RSP: + return tqProcessTaskRecoverFinishRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index bf665fd6db..478e20d1ab 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -230,147 +230,27 @@ void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) { } } -int32_t vnodeProcessSyncCtrlMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - int32_t code = 0; - const STraceId *trace = &pMsg->info.traceId; - - if (!syncEnvIsStart()) { - vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId, pMsg); - terrno = TSDB_CODE_APP_ERROR; - return -1; - } - - SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); - if (pSyncNode == NULL) { - vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId, pMsg); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - - vGTrace("vgId:%d, sync msg:%p will be processed, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); - - if (pMsg->msgType == TDMT_SYNC_HEARTBEAT) { - SyncHeartbeat *pSyncMsg = syncHeartbeatFromRpcMsg2(pMsg); - code = syncNodeOnHeartbeat(pSyncNode, pSyncMsg); - syncHeartbeatDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_HEARTBEAT_REPLY) { - SyncHeartbeatReply *pSyncMsg = syncHeartbeatReplyFromRpcMsg2(pMsg); - code = syncNodeOnHeartbeatReply(pSyncNode, pSyncMsg); - syncHeartbeatReplyDestroy(pSyncMsg); - - } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); - code = -1; - } - - vTrace("vgId:%d, sync msg:%p is processed, type:%s code:0x%x", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType), - code); - syncNodeRelease(pSyncNode); - if (code != 0 && terrno == 0) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - } - return code; -} - int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { - int32_t code = 0; const STraceId *trace = &pMsg->info.traceId; - - if (!syncEnvIsStart()) { - vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId, pMsg); - terrno = TSDB_CODE_APP_ERROR; - return -1; - } - - SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); - if (pSyncNode == NULL) { - vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId, pMsg); - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - vGTrace("vgId:%d, sync msg:%p will be processed, type:%s", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType)); - if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { - SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnTimer(pSyncNode, pSyncMsg); - syncTimeoutDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_PING) { - SyncPing *pSyncMsg = syncPingFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnPing(pSyncNode, pSyncMsg); - syncPingDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) { - SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnPingReply(pSyncNode, pSyncMsg); - syncPingReplyDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { - SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, NULL); - syncClientRequestDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { - SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnRequestVote(pSyncNode, pSyncMsg); - syncRequestVoteDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { - SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnRequestVoteReply(pSyncNode, pSyncMsg); - syncRequestVoteReplyDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { - SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnAppendEntries(pSyncNode, pSyncMsg); - syncAppendEntriesDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { - SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); - ASSERT(pSyncMsg != NULL); - code = syncNodeOnAppendEntriesReply(pSyncNode, pSyncMsg); - syncAppendEntriesReplyDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { - SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); - code = syncNodeOnSnapshot(pSyncNode, pSyncMsg); - syncSnapshotSendDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { - SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); - code = syncNodeOnSnapshotReply(pSyncNode, pSyncMsg); - syncSnapshotRspDestroy(pSyncMsg); - - } else if (pMsg->msgType == TDMT_SYNC_LOCAL_CMD) { - SyncLocalCmd *pSyncMsg = syncLocalCmdFromRpcMsg2(pMsg); - code = syncNodeOnLocalCmd(pSyncNode, pSyncMsg); - syncLocalCmdDestroy(pSyncMsg); - - } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); - code = -1; + int32_t code = syncProcessMsg(pVnode->sync, pMsg); + if (code != 0) { + vGError("vgId:%d, failed to process sync msg:%p type:%s since %s", pVnode->config.vgId, pMsg, + TMSG_INFO(pMsg->msgType), terrstr()); } - vTrace("vgId:%d, sync msg:%p is processed, type:%s code:0x%x", pVnode->config.vgId, pMsg, TMSG_INFO(pMsg->msgType), - code); - syncNodeRelease(pSyncNode); - if (code != 0 && terrno == 0) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - } return code; } static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { - if (msgcb == NULL) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; return -1; } @@ -383,7 +263,13 @@ static int32_t vnodeSyncEqCtrlMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { } static int32_t vnodeSyncEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { - if (msgcb == NULL) { + if (pMsg == NULL || pMsg->pCont == NULL) { + return -1; + } + + if (msgcb == NULL || msgcb->putToQueueFp == NULL) { + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; return -1; } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b6a22b5fa7..e66cdb14ce 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -320,7 +320,7 @@ _return: } int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass) { + bool* pass, bool* exists) { bool inCache = false; int32_t code = 0; @@ -329,6 +329,13 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, co CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); if (inCache) { + if (exists) { + *exists = true; + } + + return TSDB_CODE_SUCCESS; + } else if (exists) { + *exists = false; return TSDB_CODE_SUCCESS; } @@ -1032,7 +1039,7 @@ int32_t catalogChkTbMetaVersion(SCatalog* pCtg, SRequestConnInfo* pConn, SArray* switch (tbType) { case TSDB_CHILD_TABLE: { SName stb = name; - strcpy(stb.tname, stbName); + tstrncpy(stb.tname, stbName, sizeof(stb.tname)); ctgRemoveTbMeta(pCtg, &stb); break; } @@ -1373,13 +1380,30 @@ int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass)); + CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, NULL)); _return: CTG_API_LEAVE(code); } +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, + bool* pass, bool* exists) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, exists)); + +_return: + + CTG_API_LEAVE(code); +} + + int32_t catalogGetServerVersion(SCatalog* pCtg, SRequestConnInfo* pConn, char** pVersion) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 4a64a8666e..3d21bbbcd9 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -924,6 +924,11 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* if (1 == vgNum) { void* pIter = taosHashIterate(dbInfo->vgHash, NULL); + if (NULL == pIter) { + ctgError("empty vgHash, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + for (int32_t i = 0; i < tbNum; ++i) { vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); if (NULL == vgInfo) { diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 0bdd9841ab..489d174e17 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -2771,10 +2771,30 @@ TEST(apiTest, catalogChkAuth_test) { ASSERT_EQ(code, 0); bool pass = false; + bool exists = false; + code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + ASSERT_EQ(code, 0); + ASSERT_EQ(exists, false); + code = catalogChkAuth(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass); ASSERT_EQ(code, 0); ASSERT_EQ(pass, true); + while (true) { + uint64_t n = 0; + ctgdGetStatNum("runtime.numOfOpDequeue", (void *)&n); + if (n != 1) { + taosMsleep(50); + } else { + break; + } + } + + code = catalogChkAuthFromCache(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + ASSERT_EQ(code, 0); + ASSERT_EQ(pass, true); + ASSERT_EQ(exists, true); + catalogDestroy(); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 76a3359b5b..fd8a357a8b 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -96,11 +96,12 @@ typedef struct SColMatchInfo { int32_t matchType; // determinate the source according to col id or slot id } SColMatchInfo; +typedef struct SExecTaskInfo SExecTaskInfo; typedef struct STableListInfo STableListInfo; struct SqlFunctionCtx; int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* id); + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, SExecTaskInfo* pTaskInfo); STableListInfo* tableListCreate(); void* tableListDestroy(STableListInfo* pTableListInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1191b6a485..14159c2da2 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -89,25 +89,10 @@ typedef struct STableScanAnalyzeInfo SFileBlockLoadRecorder; typedef struct STaskCostInfo { int64_t created; int64_t start; - uint64_t loadStatisTime; - uint64_t loadFileBlockTime; - uint64_t loadDataInCacheTime; - uint64_t loadStatisSize; - uint64_t loadFileBlockSize; - uint64_t loadDataInCacheSize; - - uint64_t loadDataTime; - + uint64_t elapsedTime; + double extractListTime; + double groupIdMapTime; SFileBlockLoadRecorder* pRecoder; - uint64_t elapsedTime; - - uint64_t winInfoSize; - uint64_t tableInfoSize; - uint64_t hashSize; - uint64_t numOfTimeWindows; - - SArray* queryProfEvents; // SArray - SHashObj* operatorProfResults; // map } STaskCostInfo; typedef struct SOperatorCostInfo { @@ -150,18 +135,13 @@ typedef struct { SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; - SSDataBlock* pullOverBlk; // for streaming - SWalFilterCond cond; - int64_t lastScanUid; int8_t recoverStep; SQueryTableDataCond tableCond; - int64_t recoverStartVer; - int64_t recoverEndVer; int64_t fillHistoryVer1; int64_t fillHistoryVer2; - int8_t triggerSaved; - int64_t deleteMarkSaved; + // int8_t triggerSaved; + // int64_t deleteMarkSaved; SStreamState* pState; } SStreamTaskInfo; @@ -461,8 +441,10 @@ typedef struct SPartitionDataInfo { typedef struct STimeWindowAggSupp { int8_t calTrigger; - int64_t waterMark; + int8_t calTriggerSaved; int64_t deleteMark; + int64_t deleteMarkSaved; + int64_t waterMark; TSKEY maxTs; TSKEY minTs; SColumnInfoData timeWindowData; // query time window info for scalar function execution. @@ -921,7 +903,7 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaul void doSetOperatorCompleted(SOperatorInfo* pOperator); void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, SColMatchInfo* pColMatchInfo, SFilterInfo* pFilterInfo); int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock, const char* idStr); + SSDataBlock* pBlock, int32_t rows, const char* idStr); void cleanupAggSup(SAggSupporter* pAggSup); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 92e98d3eab..bcd59e83f0 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -152,25 +152,27 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->indexOfBufferedRes = 0; } + SSDataBlock* pRes = pInfo->pRes; + if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { SColMatchItem* pMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i); int32_t slotId = pMatchInfo->dstSlotId; SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId); - SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, slotId); + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId); char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); bool isNull = colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes); colDataAppend(pDst, 0, p, isNull); } - pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); - pInfo->pRes->info.rows = 1; + pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes); + pRes->info.rows = 1; if (pInfo->pseudoExprSup.numOfExprs > 0) { SExprSupp* pSup = &pInfo->pseudoExprSup; - int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, pRes->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; @@ -178,10 +180,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - pInfo->pRes->info.groupId = getTableGroupId(pTableList, pInfo->pRes->info.uid); - + pRes->info.groupId = getTableGroupId(pTableList, pRes->info.uid); pInfo->indexOfBufferedRes += 1; - return pInfo->pRes; + return pRes; } else { doSetOperatorCompleted(pOperator); return NULL; @@ -221,7 +222,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW); pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); - code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index bb87b5029a..32386a72fd 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -775,75 +775,6 @@ end: return code; } -static int tableUidCompare(const void* a, const void* b) { - int64_t u1 = *(uint64_t*)a; - int64_t u2 = *(uint64_t*)b; - if (u1 == u2) { - return 0; - } - return u1 < u2 ? -1 : 1; -} - -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond, SHashObj* tags) { - int32_t ret = -1; - if (nodeType(cond) == QUERY_NODE_OPERATOR) { - ret = optimizeTbnameInCondImpl(metaHandle, suid, list, cond); - if (ret != -1) { - metaGetTableTagsByUids(metaHandle, suid, list, tags); - removeInvalidTable(list, tags); - } - } - - if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { - return ret; - } - - bool hasTbnameCond = false; - SLogicConditionNode* pNode = (SLogicConditionNode*)cond; - SNodeList* pList = (SNodeList*)pNode->pParameterList; - - int32_t len = LIST_LENGTH(pList); - if (len <= 0) return ret; - - SListCell* cell = pList->pHead; - for (int i = 0; i < len; i++) { - if (cell == NULL) break; - if (optimizeTbnameInCondImpl(metaHandle, suid, list, cell->pNode) == 0) { - hasTbnameCond = true; - break; - } - cell = cell->pNext; - } - taosArraySort(list, tableUidCompare); - taosArrayRemoveDuplicate(list, tableUidCompare, NULL); - - if (hasTbnameCond) { - ret = metaGetTableTagsByUids(metaHandle, suid, list, tags); - removeInvalidTable(list, tags); - } - return ret; -} - -/* - * handle invalid uid - */ -static int32_t removeInvalidTable(SArray* uids, SHashObj* tags) { - if (taosArrayGetSize(uids) <= 0) return 0; - - SArray* validUid = taosArrayInit(taosArrayGetSize(uids), sizeof(int64_t)); - - for (int32_t i = 0; i < taosArrayGetSize(uids); i++) { - int64_t* uid = taosArrayGet(uids, i); - if (taosHashGet(tags, uid, sizeof(int64_t)) != NULL) { - taosArrayPush(validUid, uid); - } - } - - taosArraySwap(uids, validUid); - taosArrayDestroy(validUid); - return 0; -} - static int32_t nameComparFn(const void* p1, const void* p2) { const char* pName1 = *(const char**)p1; const char* pName2 = *(const char**)p2; @@ -897,6 +828,81 @@ static SArray* getTableNameList(const SNodeListNode* pList) { return pNewList; } +static int tableUidCompare(const void* a, const void* b) { + uint64_t u1 = *(uint64_t*)a; + uint64_t u2 = *(uint64_t*)b; + + if (u1 == u2) { + return 0; + } + + return u1 < u2 ? -1 : 1; +} + +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond, SHashObj* tags) { + int32_t ret = -1; + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + ret = optimizeTbnameInCondImpl(metaHandle, suid, list, cond); + if (ret != -1) { + metaGetTableTagsByUids(metaHandle, suid, list, tags); + removeInvalidTable(list, tags); + } + } + + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return ret; + } + + bool hasTbnameCond = false; + SLogicConditionNode* pNode = (SLogicConditionNode*)cond; + SNodeList* pList = (SNodeList*)pNode->pParameterList; + + int32_t len = LIST_LENGTH(pList); + if (len <= 0) { + return ret; + } + + SListCell* cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + if (optimizeTbnameInCondImpl(metaHandle, suid, list, cell->pNode) == 0) { + hasTbnameCond = true; + break; + } + cell = cell->pNext; + } + + taosArraySort(list, tableUidCompare); + taosArrayRemoveDuplicate(list, tableUidCompare, NULL); + + if (hasTbnameCond) { + ret = metaGetTableTagsByUids(metaHandle, suid, list, tags); + removeInvalidTable(list, tags); + } + + return ret; +} + +/* + * handle invalid uid + */ +static int32_t removeInvalidTable(SArray* uids, SHashObj* tags) { + if (taosArrayGetSize(uids) <= 0) return 0; + + SArray* validUid = taosArrayInit(taosArrayGetSize(uids), sizeof(int64_t)); + + for (int32_t i = 0; i < taosArrayGetSize(uids); i++) { + int64_t* uid = taosArrayGet(uids, i); + if (taosHashGet(tags, uid, sizeof(int64_t)) != NULL) { + taosArrayPush(validUid, uid); + } + } + + taosArraySwap(uids, validUid); + taosArrayDestroy(validUid); + return 0; +} + static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { return -1; @@ -1757,7 +1763,7 @@ STableListInfo* tableListCreate() { goto _error; } - pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); if (pListInfo->map == NULL) { goto _error; } @@ -1882,8 +1888,9 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, - const char* idStr) { + struct SExecTaskInfo* pTaskInfo) { int64_t st = taosGetTimestampUs(); + const char* idStr = GET_TASKID(pTaskInfo); if (pHandle == NULL) { qError("invalid handle, in creating operator tree, %s", idStr); @@ -1899,7 +1906,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags ASSERT(pTableListInfo->numOfOuputGroups == 1); int64_t st1 = taosGetTimestampUs(); - qDebug("generate queried table list completed, elapsed time:%.2f ms %s", (st1 - st) / 1000.0, idStr); + pTaskInfo->cost.extractListTime = (st1 - st) / 1000.0; + qDebug("extract queried table list completed, elapsed time:%.2f ms %s", pTaskInfo->cost.extractListTime, idStr); if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { qDebug("no table qualified for query, %s" PRIx64, idStr); @@ -1911,8 +1919,8 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags return code; } - int64_t st2 = taosGetTimestampUs(); - qDebug("generate group id map completed, elapsed time:%.2f ms %s", (st2 - st1) / 1000.0, idStr); + pTaskInfo->cost.groupIdMapTime = (taosGetTimestampUs() - st1)/1000.0; + qDebug("generate group id map completed, elapsed time:%.2f ms %s", pTaskInfo->cost.groupIdMapTime, idStr); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index bbd62a9912..2964878a2c 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -70,6 +70,26 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } } +static int32_t doSetStreamOpOpen(SOperatorInfo* pOperator, char* id) { + { + ASSERT(pOperator != NULL); + if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + if (pOperator->numOfDownstream == 0) { + qError("failed to find stream scan operator to set the input data block, %s" PRIx64, id); + return TSDB_CODE_QRY_APP_ERROR; + } + + if (pOperator->numOfDownstream > 1) { // not handle this in join query + qError("join not supported for stream block scan, %s" PRIx64, id); + return TSDB_CODE_QRY_APP_ERROR; + } + pOperator->status = OP_NOT_OPENED; + return doSetStreamOpOpen(pOperator->pDownstream[0], id); + } + } + return 0; +} + static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -117,7 +137,22 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } -static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); } +int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { + if (tinfo == NULL) { + return TSDB_CODE_QRY_APP_ERROR; + } + + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + + int32_t code = doSetStreamOpOpen(pTaskInfo->pRoot, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); + } else { + qDebug("%s set the stream block successfully", GET_TASKID(pTaskInfo)); + } + + return code; +} int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { @@ -706,8 +741,7 @@ int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.recoverStartVer = 0; - pTaskInfo->streamInfo.recoverEndVer = ver; + pTaskInfo->streamInfo.fillHistoryVer1 = ver; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE1; return 0; } @@ -715,8 +749,7 @@ int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver) { int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - pTaskInfo->streamInfo.recoverStartVer = pTaskInfo->streamInfo.recoverEndVer; - pTaskInfo->streamInfo.recoverEndVer = ver; + pTaskInfo->streamInfo.fillHistoryVer2 = ver; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__PREPARE2; return 0; } @@ -737,22 +770,44 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; - pTaskInfo->streamInfo.triggerSaved = pInfo->twAggSup.calTrigger; - pTaskInfo->streamInfo.deleteMarkSaved = pInfo->twAggSup.deleteMark; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0); + ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + + qInfo("save stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + + pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; + pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - pTaskInfo->streamInfo.triggerSaved = pInfo->twAggSup.calTrigger; - pTaskInfo->streamInfo.deleteMarkSaved = pInfo->twAggSup.deleteMark; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0); + ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + + qInfo("save stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + + pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; + pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; - pTaskInfo->streamInfo.triggerSaved = pInfo->twAggSup.calTrigger; - pTaskInfo->streamInfo.deleteMarkSaved = pInfo->twAggSup.deleteMark; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0); + ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + + qInfo("save stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); + + pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; + pInfo->twAggSup.deleteMarkSaved = pInfo->twAggSup.deleteMark; pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; pInfo->twAggSup.deleteMark = INT64_MAX; } @@ -782,21 +837,36 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX); - pInfo->twAggSup.calTrigger = pTaskInfo->streamInfo.triggerSaved; - pInfo->twAggSup.deleteMark = pTaskInfo->streamInfo.deleteMarkSaved; + pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; + pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + qInfo("restore stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION || pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX); - pInfo->twAggSup.calTrigger = pTaskInfo->streamInfo.triggerSaved; - pInfo->twAggSup.deleteMark = pTaskInfo->streamInfo.deleteMarkSaved; + pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; + pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + qInfo("restore stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + ASSERT(pInfo->twAggSup.deleteMark == INT64_MAX); - pInfo->twAggSup.calTrigger = pTaskInfo->streamInfo.triggerSaved; - pInfo->twAggSup.deleteMark = pTaskInfo->streamInfo.deleteMarkSaved; + pInfo->twAggSup.calTrigger = pInfo->twAggSup.calTriggerSaved; + pInfo->twAggSup.deleteMark = pInfo->twAggSup.deleteMarkSaved; + ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || + pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); + qInfo("restore stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); } // iterate operator tree diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 0c0d5549f2..13a3712f0c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -574,6 +574,9 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc // if the source equals to the destination, it is to create a new column as the result of scalar // function or some operators. bool createNewColModel = (pResult == pSrcBlock); + if (createNewColModel) { + blockDataEnsureCapacity(pResult, pResult->info.rows); + } int32_t numOfRows = 0; @@ -623,6 +626,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; ASSERT(pResult->info.capacity > 0); + colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows); colDataDestroy(&idata); @@ -844,57 +848,6 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int return win; } -#if 0 -static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { - STimeWindow w = {0}; - - TSKEY sk = TMIN(pQueryAttr->window.skey, pQueryAttr->window.ekey); - TSKEY ek = TMAX(pQueryAttr->window.skey, pQueryAttr->window.ekey); - - if (true) { - // getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.skey, sk, ek, &w); - assert(w.ekey >= pBlockInfo->window.skey); - - if (w.ekey < pBlockInfo->window.ekey) { - return true; - } - - while (1) { - // getNextTimeWindow(pQueryAttr, &w); - if (w.skey > pBlockInfo->window.ekey) { - break; - } - - assert(w.ekey > pBlockInfo->window.ekey); - if (w.skey <= pBlockInfo->window.ekey && w.skey > pBlockInfo->window.skey) { - return true; - } - } - } else { - // getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w); - assert(w.skey <= pBlockInfo->window.ekey); - - if (w.skey > pBlockInfo->window.skey) { - return true; - } - - while (1) { - // getNextTimeWindow(pQueryAttr, &w); - if (w.ekey < pBlockInfo->window.skey) { - break; - } - - assert(w.skey < pBlockInfo->window.skey); - if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) { - return true; - } - } - } - - return false; -} -#endif - int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NOT_LOAD; @@ -1447,40 +1400,18 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -static int32_t compressQueryColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, int8_t compressed) { - int32_t colSize = pColRes->info.bytes * numOfRows; - return (*(tDataTypes[pColRes->info.type].compFunc))(pColRes->pData, colSize, numOfRows, data, - colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); -} - void queryCostStatis(SExecTaskInfo* pTaskInfo) { STaskCostInfo* pSummary = &pTaskInfo->cost; - // uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); - // hashSize += taosHashGetMemSize(pRuntimeEnv->tableqinfoGroupInfo.map); - // pSummary->hashSize = hashSize; - - // SResultRowPool* p = pTaskInfo->pool; - // if (p != NULL) { - // pSummary->winInfoSize = getResultRowPoolMemSize(p); - // pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); - // } else { - // pSummary->winInfoSize = 0; - // pSummary->numOfTimeWindows = 0; - // } - SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder; if (pSummary->pRecoder != NULL) { qDebug( - "%s :cost summary: elapsed time:%.2f ms, total blocks:%d, load block SMA:%d, load data block:%d, total " - "rows:%" PRId64 ", check rows:%" PRId64, - GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pRecorder->totalBlocks, pRecorder->loadBlockStatis, - pRecorder->loadBlocks, pRecorder->totalRows, pRecorder->totalCheckedRows); + "%s :cost summary: elapsed time:%.2f ms, extract tableList:%.2f ms, createGroupIdMap:%.2f ms, total blocks:%d, " + "load block SMA:%d, load data block:%d, total rows:%" PRId64 ", check rows:%" PRId64, + GET_TASKID(pTaskInfo), pSummary->elapsedTime / 1000.0, pSummary->extractListTime, pSummary->groupIdMapTime, + pRecorder->totalBlocks, pRecorder->loadBlockStatis, pRecorder->loadBlocks, pRecorder->totalRows, + pRecorder->totalCheckedRows); } - - // qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, - // hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, - // pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } // static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { @@ -3267,6 +3198,11 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model, char* dbFName) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); + if (pTaskInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->schemaInfo.dbname = strdup(dbFName); @@ -3397,7 +3333,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, - pTableListInfo, pTagCond, pTagIndexCond, idstr); + pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr); @@ -3415,9 +3351,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - int32_t code = - createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, /*pTableScanNode->groupSort*/true, pHandle, - pTableListInfo, pTagCond, pTagIndexCond, idstr); + + int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, + pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); @@ -3442,7 +3378,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo if (pHandle->vnode) { int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, - pHandle, pTableListInfo, pTagCond, pTagIndexCond, idstr); + pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); if (code) { pTaskInfo->code = code; qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); @@ -3469,7 +3405,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, - pTagIndexCond, idstr); + pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; qError("failed to getTableList, code: %s", tstrerror(code)); @@ -3502,7 +3438,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, - pTagCond, pTagIndexCond, idstr); + pTagCond, pTagIndexCond, pTaskInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -3834,10 +3770,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead char* sql, EOPTR_EXEC_MODEL model) { uint64_t queryId = pPlan->id.queryId; - int32_t code = TSDB_CODE_SUCCESS; *pTaskInfo = createExecTaskInfo(queryId, taskId, model, pPlan->dbFName); if (*pTaskInfo == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; } @@ -3856,17 +3790,16 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { - code = (*pTaskInfo)->code; + terrno = (*pTaskInfo)->code; goto _complete; } - return code; + return TSDB_CODE_SUCCESS; - _complete: +_complete: taosMemoryFree(sql); doDestroyTask(*pTaskInfo); - terrno = code; - return code; + return terrno; } void doDestroyTask(SExecTaskInfo* pTaskInfo) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 582c79e8d3..5229b46815 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -331,7 +331,8 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, } } - for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->matchInfo.pList); ++i) { + size_t num = taosArrayGetSize(pTableScanInfo->matchInfo.pList); + for (int32_t i = 0; i < num; ++i) { SColMatchItem* pColMatchInfo = taosArrayGet(pTableScanInfo->matchInfo.pList, i); if (!pColMatchInfo->needOutput) { continue; @@ -343,11 +344,11 @@ static bool doLoadBlockSMA(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, return true; } -static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { +static void doSetTagColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, int32_t rows) { if (pTableScanInfo->pseudoSup.numOfExprs > 0) { SExprSupp* pSup = &pTableScanInfo->pseudoSup; - int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); @@ -382,6 +383,17 @@ void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo } } +static void ensureBlockCapacity(SSDataBlock* pBlock, int32_t capacity) { + // keep the value of rows temporarily + int32_t rows = pBlock->info.rows; + + pBlock->info.rows = 0; + blockDataEnsureCapacity(pBlock, capacity); + + // restore the rows number + pBlock->info.rows = rows; +} + static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -413,7 +425,10 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + ensureBlockCapacity(pBlock, pBlock->info.rows); + } + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_STATIS_LOAD) { @@ -423,7 +438,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + if (pTableScanInfo->pseudoSup.numOfExprs > 0) { + ensureBlockCapacity(pBlock, pBlock->info.rows); + } + + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); return TSDB_CODE_SUCCESS; } else { qDebug("%s failed to load SMA, since not all columns have SMA", GET_TASKID(pTaskInfo)); @@ -472,8 +491,9 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca return terrno; } + ensureBlockCapacity(pBlock, pBlock->info.rows); relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); // restore the previous value pCost->totalRows -= pBlock->info.rows; @@ -513,27 +533,30 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction } int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock, const char* idStr) { + SSDataBlock* pBlock, int32_t rows, const char* idStr) { // currently only the tbname pseudo column if (numOfPseudoExpr == 0) { return TSDB_CODE_SUCCESS; } + // backup the rows + int32_t backupRows = pBlock->info.rows; + pBlock->info.rows = rows; + SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); int32_t code = metaGetTableEntryByUid(&mr, pBlock->info.uid); + metaReaderReleaseLock(&mr); + if (code != TSDB_CODE_SUCCESS) { qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); metaReaderClear(&mr); return terrno; } - metaReaderReleaseLock(&mr); - for (int32_t j = 0; j < numOfPseudoExpr; ++j) { SExprInfo* pExpr = &pPseudoExpr[j]; - - int32_t dstSlotId = pExpr->base.resSchema.slotId; + int32_t dstSlotId = pExpr->base.resSchema.slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); colInfoDataCleanup(pColInfoData, pBlock->info.rows); @@ -572,6 +595,9 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int } metaReaderClear(&mr); + + // restore the rows + pBlock->info.rows = backupRows; return TSDB_CODE_SUCCESS; } @@ -616,14 +642,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &binfo); + SDataBlockInfo* pBInfo = &pBlock->info; + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBInfo->rows, &pBInfo->uid, &pBInfo->window); - binfo.capacity = binfo.rows; - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info = binfo; - - ASSERT(binfo.uid != 0); + ASSERT(pBInfo->uid != 0); pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid); uint32_t status = 0; @@ -1146,20 +1168,19 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU bool hasBlock = tsdbNextDataBlock(pReader); if (hasBlock) { - SDataBlockInfo binfo = {0}; - tsdbRetrieveDataBlockInfo(pReader, &binfo); + SDataBlockInfo* pBInfo = &pBlock->info; + + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->uid, &pBInfo->window); SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL); - blockDataEnsureCapacity(pBlock, binfo.rows); - - pBlock->info.window = binfo.window; - pBlock->info.uid = binfo.uid; - pBlock->info.rows = binfo.rows; + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; relocateColumnData(pBlock, pTableScanInfo->matchInfo.pList, pCols, true); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, rows); - pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, binfo.uid); + pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->uid); } tsdbReaderClose(pReader); @@ -1181,12 +1202,6 @@ static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, static uint64_t getGroupIdByUid(SStreamScanInfo* pInfo, uint64_t uid) { return getTableGroupId(pInfo->pTableScanOp->pTaskInfo->pTableInfoList, uid); - // SHashObj* map = pInfo->pTableScanOp->pTaskInfo->pTableInfoList.map; - // uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); - // if (groupId) { - // return *groupId; - // } - // return 0; } static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { @@ -1631,7 +1646,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, - GET_TASKID(pTaskInfo)); + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { blockDataFreeRes((SSDataBlock*)pBlock); T_LONG_JMP(pTaskInfo->env, code); @@ -1641,11 +1656,11 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock if (filter) { doFilter(pInfo->pCondition, pInfo->pRes, NULL, NULL); } + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); blockDataFreeRes((SSDataBlock*)pBlock); calBlockTbName(&pInfo->tbnameCalSup, pInfo->pRes); - return 0; } @@ -1875,6 +1890,11 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTSInfo->cond.startVersion = pTaskInfo->streamInfo.fillHistoryVer1 + 1; pTSInfo->cond.endVersion = pTaskInfo->streamInfo.fillHistoryVer2; } + + /*resetTableScanInfo(pTSInfo, pWin);*/ + tsdbReaderClose(pTSInfo->dataReader); + pTSInfo->dataReader = NULL; + pTSInfo->scanTimes = 0; pTSInfo->currentGroupId = -1; pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__SCAN; @@ -1888,6 +1908,10 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return pBlock; } pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + pTSInfo->cond.startVersion = 0; + pTSInfo->cond.endVersion = -1; + return NULL; } #endif @@ -2142,7 +2166,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } - tsdbRetrieveDataBlockInfo(pInfo->dataReader, &pBlock->info); + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.uid, &pBlock->info.window); + pBlock->info.rows = rows; SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL); pBlock->pDataBlock = pCols; @@ -4359,7 +4385,7 @@ static int32_t loadDataBlockFromOneTable2(SOperatorInfo* pOperator, STableMergeS // currently only the tbname pseudo column if (pTableScanInfo->pseudoSup.numOfExprs > 0) { int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo, - pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo)); + pTableScanInfo->pseudoSup.numOfExprs, pBlock, pBlock->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -4476,7 +4502,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc // currently only the tbname pseudo column if (pTableScanInfo->pseudoSup.numOfExprs > 0) { int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pseudoSup.pExprInfo, - pTableScanInfo->pseudoSup.numOfExprs, pBlock, GET_TASKID(pTaskInfo)); + pTableScanInfo->pseudoSup.numOfExprs, pBlock, pBlock->info.rows, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -4540,14 +4566,11 @@ static SSDataBlock* getTableDataBlockTemp(void* param) { } blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(reader, &binfo); - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info.type = binfo.type; - pBlock->info.uid = binfo.uid; - pBlock->info.window = binfo.window; - pBlock->info.rows = binfo.rows; + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window); + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; if (pQueryCond->order == TSDB_ORDER_ASC) { pQueryCond->twindows.skey = pBlock->info.window.ekey + 1; @@ -4603,14 +4626,11 @@ static SSDataBlock* getTableDataBlock2(void* param) { } blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(reader, &binfo); - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info.type = binfo.type; - pBlock->info.uid = binfo.uid; - pBlock->info.window = binfo.window; - pBlock->info.rows = binfo.rows; + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window); + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable2(pOperator, pTableScanInfo, pBlock, &status); @@ -4656,14 +4676,11 @@ static SSDataBlock* getTableDataBlock(void* param) { } blockDataCleanup(pBlock); - SDataBlockInfo binfo = pBlock->info; - tsdbRetrieveDataBlockInfo(reader, &binfo); - blockDataEnsureCapacity(pBlock, binfo.rows); - pBlock->info.type = binfo.type; - pBlock->info.uid = binfo.uid; - pBlock->info.window = binfo.window; - pBlock->info.rows = binfo.rows; + int32_t rows = 0; + tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window); + blockDataEnsureCapacity(pBlock, rows); + pBlock->info.rows = rows; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ee45d0547f..6be9a0402f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3303,6 +3303,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, // for test 315360000000 .deleteMark = 1000LL * 60LL * 60LL * 24LL * 365LL * 10LL, // .deleteMark = INT64_MAX, + .deleteMarkSaved = 0, + .calTriggerSaved = 0, }; ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 913dd24a49..fa241dc6ef 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -12,6 +12,17 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(function jemalloc) +ENDIF () + target_link_libraries( function PRIVATE os @@ -21,7 +32,7 @@ target_link_libraries( PRIVATE qcom PRIVATE scalar PRIVATE transport - PRIVATE stream + PRIVATE stream ${LINK_JEMALLOC} PUBLIC uv_a ) @@ -37,10 +48,15 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(runUdf jemalloc) +ENDIF () + target_link_libraries( runUdf PUBLIC uv_a - PRIVATE os util common nodes function + PRIVATE os util common nodes function ${LINK_JEMALLOC} ) add_library(udf1 STATIC MODULE test/udf1.c) @@ -54,8 +70,13 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf1 jemalloc) +ENDIF () + target_link_libraries( - udf1 PUBLIC os) + udf1 PUBLIC os ${LINK_JEMALLOC}) add_library(udf2 STATIC MODULE test/udf2.c) target_include_directories( @@ -68,8 +89,13 @@ target_include_directories( "${TD_SOURCE_DIR}/include/os" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udf2 jemalloc) +ENDIF () + target_link_libraries( - udf2 PUBLIC os + udf2 PUBLIC os ${LINK_JEMALLOC} ) #SET(EXECUTABLE_OUTPUT_PATH ${CMAKE_BINARY_DIR}/build/bin) @@ -86,9 +112,13 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - udfd - PUBLIC uv_a - PRIVATE os util common nodes function -) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEPENDENCIES(udfd jemalloc) +ENDIF () + +target_link_libraries( + udfd + PUBLIC uv_a + PRIVATE os util common nodes function ${LINK_JEMALLOC} + ) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 5dcef7cd17..35ad10ffbf 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -497,7 +497,7 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static FORCE_INLINE int32_t getNumOfElems(SqlFunctionCtx* pCtx) { +static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; /* @@ -2194,6 +2194,56 @@ int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { break; } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t* plist = (uint8_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t* plist = (uint16_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_UINT: { + uint32_t* plist = (uint32_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t* plist = (uint64_t*)pCol->pData; + for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { + if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) { + continue; + } + + numOfElem++; + LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { float* plist = (float*)pCol->pData; for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index bae005f5c4..0de914271c 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -251,6 +251,13 @@ bool fmIsSelectValueFunc(int32_t funcId) { return FUNCTION_TYPE_SELECT_VALUE == funcMgtBuiltins[funcId].type; } +bool fmIsGroupKeyFunc(int32_t funcId) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return false; + } + return FUNCTION_TYPE_GROUP_KEY == funcMgtBuiltins[funcId].type; +} + void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index ea59e92e98..459ee583a4 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -117,10 +117,29 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) { char dnodeIdEnvItem[32] = {0}; char thrdPoolSizeEnvItem[32] = {0}; snprintf(dnodeIdEnvItem, 32, "%s=%d", "DNODE_ID", pData->dnodeId); + float numCpuCores = 4; taosGetCpuCores(&numCpuCores); snprintf(thrdPoolSizeEnvItem, 32, "%s=%d", "UV_THREADPOOL_SIZE", (int)numCpuCores * 2); - char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, NULL}; + + char pathTaosdLdLib[512] = {0}; + size_t taosdLdLibPathLen = sizeof(pathTaosdLdLib); + uv_os_getenv("LD_LIBRARY_PATH", pathTaosdLdLib, &taosdLdLibPathLen); + + char udfdPathLdLib[1024] = {0}; + size_t udfdLdLibPathLen = strlen(tsUdfdLdLibPath); + strncpy(udfdPathLdLib, tsUdfdLdLibPath, udfdLdLibPathLen); + udfdPathLdLib[udfdLdLibPathLen] = ':'; + strncpy(udfdPathLdLib + udfdLdLibPathLen + 1, pathTaosdLdLib, sizeof(udfdPathLdLib) - udfdLdLibPathLen); + if (udfdLdLibPathLen + taosdLdLibPathLen < 1024) { + fnInfo("udfd LD_LIBRARY_PATH: %s", udfdPathLdLib); + } else { + fnError("can not set correct udfd LD_LIBRARY_PATH"); + } + char ldLibPathEnvItem[1024 + 32] = {0}; + snprintf(ldLibPathEnvItem, 1024 + 32, "%s=%s", "LD_LIBRARY_PATH", udfdPathLdLib); + + char *envUdfd[] = {dnodeIdEnvItem, thrdPoolSizeEnvItem, ldLibPathEnvItem, NULL}; options.env = envUdfd; int err = uv_spawn(&pData->loop, &pData->process, &options); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d76ee55ef4..d3ff787f50 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -5160,7 +5160,7 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { SMDropSmaReq dropSmaReq = {0}; SName name; - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->indexName, &name), dropSmaReq.name); + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), dropSmaReq.name); dropSmaReq.igNotExists = pStmt->ignoreNotExists; return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq); } @@ -5281,7 +5281,7 @@ static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* p code = buildCreateTopicReq(pCxt, pStmt, &createReq); } if (TSDB_CODE_SUCCESS == code) { - code = buildCmdMsg(pCxt, TDMT_MND_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq); + code = buildCmdMsg(pCxt, TDMT_MND_TMQ_CREATE_TOPIC, (FSerializeFunc)tSerializeSCMCreateTopicReq, &createReq); } tFreeSCMCreateTopicReq(&createReq); return code; @@ -5295,7 +5295,7 @@ static int32_t translateDropTopic(STranslateContext* pCxt, SDropTopicStmt* pStmt tNameGetFullDbName(&name, dropReq.name); dropReq.igNotExists = pStmt->ignoreNotExists; - return buildCmdMsg(pCxt, TDMT_MND_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); + return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_TOPIC, (FSerializeFunc)tSerializeSMDropTopicReq, &dropReq); } static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pStmt) { @@ -5307,7 +5307,7 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt dropReq.igNotExists = pStmt->ignoreNotExists; strcpy(dropReq.cgroup, pStmt->cgroup); - return buildCmdMsg(pCxt, TDMT_MND_MQ_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); + return buildCmdMsg(pCxt, TDMT_MND_TMQ_DROP_CGROUP, (FSerializeFunc)tSerializeSMDropCgroupReq, &dropReq); } static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 74941d95f1..275983f869 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -587,6 +587,10 @@ static int32_t createIndefRowsFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt return code; } +static bool isInterpFunc(int32_t funcId) { + return fmIsInterpFunc(funcId) || fmIsInterpPseudoColumnFunc(funcId) || fmIsGroupKeyFunc(funcId); +} + static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { if (!pSelect->hasInterpFunc) { return TSDB_CODE_SUCCESS; @@ -602,7 +606,7 @@ static int32_t createInterpFuncLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p pInterpFunc->node.resultDataOrder = pInterpFunc->node.requireDataOrder; // interp functions and _group_key functions - int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, fmIsVectorFunc, &pInterpFunc->pFuncs); + int32_t code = nodesCollectFuncs(pSelect, SQL_CLAUSE_SELECT, isInterpFunc, &pInterpFunc->pFuncs); if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pInterpFunc->pFuncs, pSelect, SQL_CLAUSE_SELECT); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 1a7965397e..cf064881c2 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -456,6 +456,7 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == (*pDst)->vgHash) { + taosMemoryFreeClear(*pDst); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 426274a20f..4738e1bbf9 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -1087,7 +1087,7 @@ int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - info->units = tmp; + info->units = (SFilterUnit*)tmp; memset(info->units + psize, 0, sizeof(*info->units) * FILTER_DEFAULT_UNIT_SIZE); } @@ -1633,12 +1633,12 @@ void filterDumpInfoToString(SFilterInfo *info, const char *msg, int32_t options) SValueNode *var = (SValueNode *)field->desc; SDataType *dType = &var->node.resType; - if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { - qDebug("VAL%d => [type:TS][val:[%" PRIi64 "] - [%" PRId64 "]]", i, *(int64_t *)field->data, - *(((int64_t *)field->data) + 1)); - } else { + //if (dType->type == TSDB_DATA_TYPE_VALUE_ARRAY) { + // qDebug("VAL%d => [type:TS][val:[%" PRIi64 "] - [%" PRId64 "]]", i, *(int64_t *)field->data, + // *(((int64_t *)field->data) + 1)); + //} else { qDebug("VAL%d => [type:%d][val:%" PRIx64 "]", i, dType->type, var->datum.i); // TODO - } + //} } else if (field->data) { qDebug("VAL%d => [type:NIL][val:NIL]", i); // TODO } @@ -4059,11 +4059,13 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, SColumnInfoData **p, SC SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); - FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); - *p = output.columnData; - + int32_t code = scalarCalculate(info->sclCtx.node, pList, &output); taosArrayDestroy(pList); + FLT_ERR_RET(code); + + *p = output.columnData; + if (output.numOfQualified == output.numOfRows) { *pResultStatus = FILTER_RESULT_ALL_QUALIFIED; } else if (output.numOfQualified == 0) { diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 70d1bb2685..ea1ce175e3 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -896,6 +896,10 @@ int32_t sclExecCaseWhen(SCaseWhenNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pWhen, ctx, &pWhen)); SCL_ERR_JRET(sclGetNodeRes(pWhenThen->pThen, ctx, &pThen)); + if (NULL == pWhen || NULL == pThen) { + sclError("invalid when/then in whenThen list"); + SCL_ERR_JRET(TSDB_CODE_INVALID_PARA); + } if (pCase) { vectorCompare(pCase, pWhen, &comp, TSDB_ORDER_ASC, OP_TYPE_EQUAL); diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index e9d4fbeaaa..a8f7184bb2 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -43,6 +43,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); +int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, + SEpSet* pEpSet); + SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); void streamFreeQitem(SStreamQueueItem* data); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 417ddfa80d..d2876a22c6 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -239,7 +239,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; - msg.msgType = TDMT_VND_STREAM_RECOVER_FINISH; + msg.msgType = TDMT_STREAM_RECOVER_FINISH; tmsgSendReq(pEpSet, &msg); @@ -250,7 +250,7 @@ FAIL: return code; } -int32_t streamDispatchOneReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { +int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; @@ -292,13 +292,19 @@ FAIL: int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz, int64_t groupId) { - char* ctbName; + char* ctbName = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + if (ctbName == NULL) { + return -1; + } + if (pDataBlock->info.parTbName[0]) { - ctbName = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN); snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); } else { - ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); + char* ctbShortName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->shuffleDispatcher.dbInfo.db, ctbShortName); + taosMemoryFree(ctbShortName); } + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ @@ -365,7 +371,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, downstreamTaskId, vgId); - if (streamDispatchOneReq(pTask, &req, vgId, pEpSet) < 0) { + if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) { goto FAIL_FIXED_DISPATCH; } code = 0; @@ -427,7 +433,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (pReqs[i].blockNum > 0) { // send SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - if (streamDispatchOneReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { + if (streamDispatchOneDataReq(pTask, &pReqs[i], pVgInfo->vgId, &pVgInfo->epSet) < 0) { goto FAIL_SHUFFLE_DISPATCH; } } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 6e37810004..629333b4b4 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -90,6 +90,8 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { void* exec = pTask->exec.executor; + qSetStreamOpOpen(exec); + while (1) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { @@ -127,7 +129,10 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; streamTaskOutput(pTask, qRes); - // TODO stream sched dispatch + + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + streamDispatch(pTask); + } } return 0; } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 12de2fafc1..adeb797721 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -36,6 +36,7 @@ int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) { } int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) { + pReq->msgHead.vgId = pTask->nodeId; pReq->streamId = pTask->streamId; pReq->taskId = pTask->taskId; return 0; @@ -44,10 +45,10 @@ int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { // return streamScanExec(pTask, 100); - // TODO next: dispatch msg to launch scan step2 } int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { + pReq->msgHead.vgId = pTask->nodeId; pReq->streamId = pTask->streamId; pReq->taskId = pTask->taskId; return 0; @@ -64,11 +65,20 @@ int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { SStreamRecoverFinishReq req = { .streamId = pTask->streamId, - .taskId = pTask->taskId, .childId = pTask->selfChildId, }; + // serialize if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + req.taskId = pTask->fixedEpDispatcher.taskId; + streamDispatchOneRecoverFinishReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t vgSz = taosArrayGetSize(vgInfo); + for (int32_t i = 0; i < vgSz; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + req.taskId = pVgInfo->taskId; + streamDispatchOneRecoverFinishReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + } } return 0; } @@ -76,9 +86,9 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { // agg int32_t streamAggRecoverPrepare(SStreamTask* pTask) { void* exec = pTask->exec.executor; - if (qStreamSetParamForRecover(exec) < 0) { - return -1; - } + /*if (qStreamSetParamForRecover(exec) < 0) {*/ + /*return -1;*/ + /*}*/ pTask->recoverWaitingChild = taosArrayGetSize(pTask->childEpInfo); return 0; } @@ -96,10 +106,12 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { } int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { - int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingChild, 1); - ASSERT(left >= 0); - if (left == 0) { - streamAggChildrenRecoverFinish(pTask); + if (pTask->taskLevel == TASK_LEVEL__AGG) { + int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingChild, 1); + ASSERT(left >= 0); + if (left == 0) { + streamAggChildrenRecoverFinish(pTask); + } } return 0; } diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 06da0eb3df..55ce1470ce 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -20,13 +20,7 @@ extern "C" { #endif -#include -#include -#include #include "syncInt.h" -#include "taosdef.h" -#include "trpc.h" -#include "ttimer.h" #define TIMER_MAX_MS 0x7FFFFFFF #define ENV_TICK_TIMER_MS 1000 @@ -57,12 +51,12 @@ typedef struct SSyncEnv { } SSyncEnv; -extern SSyncEnv* gSyncEnv; +SSyncEnv* syncEnv(); -int32_t syncEnvStart(); -int32_t syncEnvStop(); -int32_t syncEnvStartTimer(); -int32_t syncEnvStopTimer(); +int64_t syncNodeAdd(SSyncNode* pNode); +void syncNodeRemove(int64_t rid); +SSyncNode* syncNodeAcquire(int64_t rid); +void syncNodeRelease(SSyncNode* pNode); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index f4949e1016..5e8041e54a 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -24,6 +24,8 @@ extern "C" { #include "syncTools.h" #include "tlog.h" #include "ttimer.h" +#include "taosdef.h" +#include "ttimer.h" // clang-format off #define sFatal(...) do { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} while(0) @@ -255,9 +257,6 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* newConfig, Sync SyncIndex syncMinMatchIndex(SSyncNode* pSyncNode); char* syncNodePeerState2Str(const SSyncNode* pSyncNode); -SSyncNode* syncNodeAcquire(int64_t rid); -void syncNodeRelease(SSyncNode* pNode); - // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term); @@ -302,11 +301,7 @@ bool syncNodeNeedSendAppendEntries(SSyncNode* ths, const SRaftId* pDestId, const int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta); int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct SSnapshotMeta* sMeta); -void syncStartNormal(int64_t rid); -void syncStartStandBy(int64_t rid); - bool syncNodeCanChange(SSyncNode* pSyncNode); -bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pNewCfg); int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode); int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index fb0bfb8bef..cf0d4d25e8 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -13,118 +13,112 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "syncEnv.h" -// #include +#include "tref.h" -SSyncEnv *gSyncEnv = NULL; +static SSyncEnv gSyncEnv = {0}; +static int32_t gNodeRefId = -1; +bool gRaftDetailLog = false; +static void syncEnvTick(void *param, void *tmrId); -// local function ----------------- -static SSyncEnv *doSyncEnvStart(); -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv); -static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv); -static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv); -static void syncEnvTick(void *param, void *tmrId); -// -------------------------------- +SSyncEnv *syncEnv() { return &gSyncEnv; } -bool syncEnvIsStart() { - if (gSyncEnv == NULL) { - return false; - } +bool syncIsInit() { return atomic_load_8(&gSyncEnv.isStart); } - return atomic_load_8(&(gSyncEnv->isStart)); -} +int32_t syncInit() { + if (syncIsInit()) return 0; -int32_t syncEnvStart() { - int32_t ret = 0; uint32_t seed = (uint32_t)(taosGetTimestampNs() & 0x00000000FFFFFFFF); taosSeedRand(seed); - // gSyncEnv = doSyncEnvStart(gSyncEnv); - gSyncEnv = doSyncEnvStart(); - ASSERT(gSyncEnv != NULL); - sTrace("sync env start ok"); - return ret; -} -int32_t syncEnvStop() { - int32_t ret = doSyncEnvStop(gSyncEnv); - return ret; -} - -int32_t syncEnvStartTimer() { - int32_t ret = doSyncEnvStartTimer(gSyncEnv); - return ret; -} - -int32_t syncEnvStopTimer() { - int32_t ret = doSyncEnvStopTimer(gSyncEnv); - return ret; -} - -// local function ----------------- -static void syncEnvTick(void *param, void *tmrId) { - SSyncEnv *pSyncEnv = (SSyncEnv *)param; - if (atomic_load_64(&pSyncEnv->envTickTimerLogicClockUser) <= atomic_load_64(&pSyncEnv->envTickTimerLogicClock)) { - ++(pSyncEnv->envTickTimerCounter); - sTrace("syncEnvTick do ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 - ", envTickTimerCounter:%" PRIu64 - ", " - "envTickTimerMS:%d, tmrId:%p", - pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS, tmrId); - - // do something, tick ... - taosTmrReset(syncEnvTick, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, &pSyncEnv->pEnvTickTimer); - } else { - sTrace("syncEnvTick pass ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 - ", envTickTimerCounter:%" PRIu64 - ", " - "envTickTimerMS:%d, tmrId:%p", - pSyncEnv->envTickTimerLogicClockUser, pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerCounter, - pSyncEnv->envTickTimerMS, tmrId); - } -} - -static SSyncEnv *doSyncEnvStart() { - SSyncEnv *pSyncEnv = (SSyncEnv *)taosMemoryMalloc(sizeof(SSyncEnv)); - ASSERT(pSyncEnv != NULL); - memset(pSyncEnv, 0, sizeof(SSyncEnv)); - - pSyncEnv->envTickTimerCounter = 0; - pSyncEnv->envTickTimerMS = ENV_TICK_TIMER_MS; - pSyncEnv->FpEnvTickTimer = syncEnvTick; - atomic_store_64(&pSyncEnv->envTickTimerLogicClock, 0); - atomic_store_64(&pSyncEnv->envTickTimerLogicClockUser, 0); + memset(&gSyncEnv, 0, sizeof(SSyncEnv)); + gSyncEnv.envTickTimerCounter = 0; + gSyncEnv.envTickTimerMS = ENV_TICK_TIMER_MS; + gSyncEnv.FpEnvTickTimer = syncEnvTick; + atomic_store_64(&gSyncEnv.envTickTimerLogicClock, 0); + atomic_store_64(&gSyncEnv.envTickTimerLogicClockUser, 0); // start tmr thread - pSyncEnv->pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); + gSyncEnv.pTimerManager = taosTmrInit(1000, 50, 10000, "SYNC-ENV"); + atomic_store_8(&gSyncEnv.isStart, 1); - atomic_store_8(&(pSyncEnv->isStart), 1); - return pSyncEnv; -} - -static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { - ASSERT(pSyncEnv == gSyncEnv); - if (pSyncEnv != NULL) { - atomic_store_8(&(pSyncEnv->isStart), 0); - taosTmrCleanUp(pSyncEnv->pTimerManager); - taosMemoryFree(pSyncEnv); + gNodeRefId = taosOpenRef(200, (RefFp)syncNodeClose); + if (gNodeRefId < 0) { + sError("failed to init node ref"); + syncCleanUp(); + return -1; } - gSyncEnv = NULL; + + sDebug("sync rsetId:%d is open", gNodeRefId); return 0; } -static int32_t doSyncEnvStartTimer(SSyncEnv *pSyncEnv) { - int32_t ret = 0; - taosTmrReset(pSyncEnv->FpEnvTickTimer, pSyncEnv->envTickTimerMS, pSyncEnv, pSyncEnv->pTimerManager, - &pSyncEnv->pEnvTickTimer); - atomic_store_64(&pSyncEnv->envTickTimerLogicClock, pSyncEnv->envTickTimerLogicClockUser); - return ret; +void syncCleanUp() { + atomic_store_8(&gSyncEnv.isStart, 0); + taosTmrCleanUp(gSyncEnv.pTimerManager); + memset(&gSyncEnv, 0, sizeof(SSyncEnv)); + + if (gNodeRefId != -1) { + sDebug("sync rsetId:%d is closed", gNodeRefId); + taosCloseRef(gNodeRefId); + gNodeRefId = -1; + } } -static int32_t doSyncEnvStopTimer(SSyncEnv *pSyncEnv) { +int64_t syncNodeAdd(SSyncNode *pNode) { + pNode->rid = taosAddRef(gNodeRefId, pNode); + if (pNode->rid < 0) return -1; + + sDebug("vgId:%d, sync rid:%" PRId64 " is added to rsetId:%d", pNode->vgId, pNode->rid, gNodeRefId); + return pNode->rid; +} + +void syncNodeRemove(int64_t rid) { taosRemoveRef(gNodeRefId, rid); } + +SSyncNode *syncNodeAcquire(int64_t rid) { + SSyncNode *pNode = taosAcquireRef(gNodeRefId, rid); + if (pNode == NULL) { + sTrace("failed to acquire node from refId:%" PRId64, rid); + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; + } + + return pNode; +} + +void syncNodeRelease(SSyncNode *pNode) { taosReleaseRef(gNodeRefId, pNode->rid); } + +#if 0 +void syncEnvStartTimer() { + taosTmrReset(gSyncEnv.FpEnvTickTimer, gSyncEnv.envTickTimerMS, &gSyncEnv, gSyncEnv.pTimerManager, + &gSyncEnv.pEnvTickTimer); + atomic_store_64(&gSyncEnv.envTickTimerLogicClock, gSyncEnv.envTickTimerLogicClockUser); +} + +void syncEnvStopTimer() { int32_t ret = 0; - atomic_add_fetch_64(&pSyncEnv->envTickTimerLogicClockUser, 1); - taosTmrStop(pSyncEnv->pEnvTickTimer); - pSyncEnv->pEnvTickTimer = NULL; + atomic_add_fetch_64(&gSyncEnv.envTickTimerLogicClockUser, 1); + taosTmrStop(gSyncEnv.pEnvTickTimer); + gSyncEnv.pEnvTickTimer = NULL; return ret; } +#endif + +static void syncEnvTick(void *param, void *tmrId) { + SSyncEnv *pSyncEnv = param; + if (atomic_load_64(&gSyncEnv.envTickTimerLogicClockUser) <= atomic_load_64(&gSyncEnv.envTickTimerLogicClock)) { + gSyncEnv.envTickTimerCounter++; + sTrace("syncEnvTick do ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 + ", envTickTimerCounter:%" PRIu64 ", envTickTimerMS:%d, tmrId:%p", + gSyncEnv.envTickTimerLogicClockUser, gSyncEnv.envTickTimerLogicClock, gSyncEnv.envTickTimerCounter, + gSyncEnv.envTickTimerMS, tmrId); + + // do something, tick ... + taosTmrReset(syncEnvTick, gSyncEnv.envTickTimerMS, pSyncEnv, gSyncEnv.pTimerManager, &gSyncEnv.pEnvTickTimer); + } else { + sTrace("syncEnvTick pass ... envTickTimerLogicClockUser:%" PRIu64 ", envTickTimerLogicClock:%" PRIu64 + ", envTickTimerCounter:%" PRIu64 ", envTickTimerMS:%d, tmrId:%p", + gSyncEnv.envTickTimerLogicClockUser, gSyncEnv.envTickTimerLogicClock, gSyncEnv.envTickTimerCounter, + gSyncEnv.envTickTimerMS, tmrId); + } +} diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 59f39a697d..26770a17d7 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "sync.h" #include "syncAppendEntries.h" #include "syncAppendEntriesReply.h" @@ -33,14 +34,7 @@ #include "syncTimeout.h" #include "syncUtil.h" #include "syncVoteMgr.h" -#include "tref.h" -bool gRaftDetailLog = false; - -static int32_t tsNodeRefId = -1; - -// ------ local funciton --------- -// enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId); static void syncNodeEqElectTimer(void* param, void* tmrId); static void syncNodeEqHeartbeatTimer(void* param, void* tmrId); @@ -49,45 +43,6 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths); static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId); static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg); -// process message ---- -int32_t syncNodeOnPing(SSyncNode* ths, SyncPing* pMsg); -int32_t syncNodeOnPingReply(SSyncNode* ths, SyncPingReply* pMsg); - -// --------------------------------- -static void syncNodeFreeCb(void* param) { - syncNodeClose(param); - param = NULL; -} - -int32_t syncInit() { - int32_t ret = 0; - - if (!syncEnvIsStart()) { - tsNodeRefId = taosOpenRef(200, syncNodeFreeCb); - if (tsNodeRefId < 0) { - sError("failed to init node ref"); - syncCleanUp(); - ret = -1; - } else { - sDebug("sync rsetId:%d is open", tsNodeRefId); - ret = syncEnvStart(); - } - } - - return ret; -} - -void syncCleanUp() { - int32_t ret = syncEnvStop(); - ASSERT(ret == 0); - - if (tsNodeRefId != -1) { - sDebug("sync rsetId:%d is closed", tsNodeRefId); - taosCloseRef(tsNodeRefId); - tsNodeRefId = -1; - } -} - int64_t syncOpen(SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); if (pSyncNode == NULL) { @@ -95,215 +50,139 @@ int64_t syncOpen(SSyncInfo* pSyncInfo) { return -1; } - pSyncNode->rid = taosAddRef(tsNodeRefId, pSyncNode); + pSyncNode->rid = syncNodeAdd(pSyncNode); if (pSyncNode->rid < 0) { syncNodeClose(pSyncNode); - pSyncNode = NULL; return -1; } - sDebug("vgId:%d, sync rid:%" PRId64 " is added to rsetId:%d", pSyncInfo->vgId, pSyncNode->rid, tsNodeRefId); return pSyncNode->rid; } void syncStart(int64_t rid) { - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - return; - } - - if (pSyncNode->pRaftCfg->isStandBy) { - syncNodeStartStandBy(pSyncNode); - } else { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode != NULL) { syncNodeStart(pSyncNode); + syncNodeRelease(pSyncNode); } - - taosReleaseRef(tsNodeRefId, pSyncNode->rid); -} - -void syncStartNormal(int64_t rid) { - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - return; - } - syncNodeStart(pSyncNode); - - taosReleaseRef(tsNodeRefId, pSyncNode->rid); -} - -void syncStartStandBy(int64_t rid) { - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - return; - } - syncNodeStartStandBy(pSyncNode); - - taosReleaseRef(tsNodeRefId, pSyncNode->rid); } void syncStop(int64_t rid) { - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) return; - int32_t vgId = pSyncNode->vgId; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); - - taosRemoveRef(tsNodeRefId, rid); - sDebug("vgId:%d, sync rid:%" PRId64 " is removed from rsetId:%d", vgId, rid, tsNodeRefId); + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode != NULL) { + syncNodeRelease(pSyncNode); + syncNodeRemove(rid); + } } -int32_t syncSetStandby(int64_t rid) { - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - sError("failed to set standby since accquire ref error, rid:%" PRId64, rid); - return -1; - } - - if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { - if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { - terrno = TSDB_CODE_SYN_IS_LEADER; - } else { - terrno = TSDB_CODE_SYN_STANDBY_NOT_READY; - } - sError("failed to set standby since it is not follower, state:%s rid:%" PRId64, syncStr(pSyncNode->state), rid); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return -1; - } - - // state change - pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; - syncNodeStopHeartbeatTimer(pSyncNode); - - // reset elect timer, long enough - int32_t electMS = TIMER_MAX_MS; - int32_t ret = syncNodeRestartElectTimer(pSyncNode, electMS); - ASSERT(ret == 0); - - pSyncNode->pRaftCfg->isStandBy = 1; - raftCfgPersist(pSyncNode->pRaftCfg); - - taosReleaseRef(tsNodeRefId, pSyncNode->rid); - sInfo("vgId:%d, set to standby", pSyncNode->vgId); - return 0; -} - -bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pNewCfg) { - bool IamInNew = syncNodeInConfig(pSyncNode, pNewCfg); - if (!IamInNew) { - return false; - } - - if (pNewCfg->replicaNum > pSyncNode->replicaNum + 1) { - return false; - } - - if (pNewCfg->replicaNum < pSyncNode->replicaNum - 1) { - return false; - } - - return true; -} - -int32_t syncReconfigBuild(int64_t rid, const SSyncCfg* pNewCfg, SRpcMsg* pRpcMsg) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - ASSERT(rid == pSyncNode->rid); - int32_t ret = 0; - - if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { - taosReleaseRef(tsNodeRefId, pSyncNode->rid); - terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; - sError("invalid new config. vgId:%d", pSyncNode->vgId); - return -1; - } - - char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); - pRpcMsg->msgType = TDMT_SYNC_CONFIG_CHANGE; - pRpcMsg->info.noResp = 1; - pRpcMsg->contLen = strlen(newconfig) + 1; - pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); - snprintf(pRpcMsg->pCont, pRpcMsg->contLen, "%s", newconfig); - taosMemoryFree(newconfig); - - taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return ret; +static bool syncNodeCheckNewConfig(SSyncNode* pSyncNode, const SSyncCfg* pCfg) { + if (!syncNodeInConfig(pSyncNode, pCfg)) return false; + return abs(pCfg->replicaNum - pSyncNode->replicaNum) <= 1; } int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - ASSERT(rid == pSyncNode->rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) return -1; if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; - sError("invalid new config. vgId:%d", pSyncNode->vgId); + sError("vgId:%d, failed to reconfig since invalid new config", pSyncNode->vgId); return -1; } -#if 0 - char* newconfig = syncCfg2Str((SSyncCfg*)pNewCfg); - int32_t ret = 0; - - SRpcMsg rpcMsg = {0}; - rpcMsg.msgType = TDMT_SYNC_CONFIG_CHANGE; - rpcMsg.info.noResp = 1; - rpcMsg.contLen = strlen(newconfig) + 1; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", newconfig); - taosMemoryFree(newconfig); - ret = syncNodePropose(pSyncNode, &rpcMsg, false); - - taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return ret; -#else syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeStopHeartbeatTimer(pSyncNode); for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { - syncHbTimerInit(pSyncNode, &(pSyncNode->peerHeartbeatTimerArr[i]), (pSyncNode->replicasId)[i]); + syncHbTimerInit(pSyncNode, &pSyncNode->peerHeartbeatTimerArr[i], pSyncNode->replicasId[i]); } syncNodeStartHeartbeatTimer(pSyncNode); - syncNodeReplicate(pSyncNode); } - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + + syncNodeRelease(pSyncNode); return 0; -#endif +} + +int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg) { + int32_t code = -1; + if (!syncIsInit()) return code; + + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) return code; + + if (pMsg->msgType == TDMT_SYNC_HEARTBEAT) { + SyncHeartbeat* pSyncMsg = syncHeartbeatFromRpcMsg2(pMsg); + code = syncNodeOnHeartbeat(pSyncNode, pSyncMsg); + syncHeartbeatDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_HEARTBEAT_REPLY) { + SyncHeartbeatReply* pSyncMsg = syncHeartbeatReplyFromRpcMsg2(pMsg); + code = syncNodeOnHeartbeatReply(pSyncNode, pSyncMsg); + syncHeartbeatReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_TIMEOUT) { + SyncTimeout* pSyncMsg = syncTimeoutFromRpcMsg2(pMsg); + code = syncNodeOnTimer(pSyncNode, pSyncMsg); + syncTimeoutDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_PING) { + SyncPing* pSyncMsg = syncPingFromRpcMsg2(pMsg); + code = syncNodeOnPing(pSyncNode, pSyncMsg); + syncPingDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_PING_REPLY) { + SyncPingReply* pSyncMsg = syncPingReplyFromRpcMsg2(pMsg); + code = syncNodeOnPingReply(pSyncNode, pSyncMsg); + syncPingReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { + SyncClientRequest* pSyncMsg = syncClientRequestFromRpcMsg2(pMsg); + code = syncNodeOnClientRequest(pSyncNode, pSyncMsg, NULL); + syncClientRequestDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { + SyncRequestVote* pSyncMsg = syncRequestVoteFromRpcMsg2(pMsg); + code = syncNodeOnRequestVote(pSyncNode, pSyncMsg); + syncRequestVoteDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { + SyncRequestVoteReply* pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pMsg); + code = syncNodeOnRequestVoteReply(pSyncNode, pSyncMsg); + syncRequestVoteReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { + SyncAppendEntries* pSyncMsg = syncAppendEntriesFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntries(pSyncNode, pSyncMsg); + syncAppendEntriesDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { + SyncAppendEntriesReply* pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pMsg); + code = syncNodeOnAppendEntriesReply(pSyncNode, pSyncMsg); + syncAppendEntriesReplyDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { + SyncSnapshotSend* pSyncMsg = syncSnapshotSendFromRpcMsg2(pMsg); + code = syncNodeOnSnapshot(pSyncNode, pSyncMsg); + syncSnapshotSendDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { + SyncSnapshotRsp* pSyncMsg = syncSnapshotRspFromRpcMsg2(pMsg); + code = syncNodeOnSnapshotReply(pSyncNode, pSyncMsg); + syncSnapshotRspDestroy(pSyncMsg); + } else if (pMsg->msgType == TDMT_SYNC_LOCAL_CMD) { + SyncLocalCmd* pSyncMsg = syncLocalCmdFromRpcMsg2(pMsg); + code = syncNodeOnLocalCmd(pSyncNode, pSyncMsg); + syncLocalCmdDestroy(pSyncMsg); + } else { + sError("vgId:%d, failed to process msg:%p since invalid type:%s", pSyncNode->vgId, pMsg, TMSG_INFO(pMsg->msgType)); + code = -1; + } + + syncNodeRelease(pSyncNode); + return code; } int32_t syncLeaderTransfer(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - ASSERT(rid == pSyncNode->rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) return -1; int32_t ret = syncNodeLeaderTransfer(pSyncNode); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); - return ret; -} - -int32_t syncLeaderTransferTo(int64_t rid, SNodeInfo newLeader) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); - if (pSyncNode == NULL) { - terrno = TSDB_CODE_SYN_INTERNAL_ERROR; - return -1; - } - ASSERT(rid == pSyncNode->rid); - - int32_t ret = syncNodeLeaderTransferTo(pSyncNode, newLeader); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return ret; } @@ -359,7 +238,7 @@ char* syncNodePeerState2Str(const SSyncNode* pSyncNode) { } int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; @@ -383,7 +262,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { logNum, isEmpty); syncNodeEventLog(pSyncNode, logBuf); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } @@ -412,7 +291,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { syncNodeEventLog(pSyncNode, logBuf); } while (0); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } } @@ -425,7 +304,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { lastApplyIndex, pSyncNode->minMatchIndex); syncNodeEventLog(pSyncNode, logBuf); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } @@ -434,7 +313,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { snprintf(logBuf, sizeof(logBuf), "new-snapshot-index:%" PRId64 " candidate, do not delete wal", lastApplyIndex); syncNodeEventLog(pSyncNode, logBuf); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } else { @@ -443,7 +322,7 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { lastApplyIndex); syncNodeEventLog(pSyncNode, logBuf); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } @@ -492,12 +371,12 @@ _DEL_WAL: } } while (0); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return code; } int32_t syncEndSnapshot(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; @@ -509,9 +388,9 @@ int32_t syncEndSnapshot(int64_t rid) { SSyncLogStoreData* pData = pSyncNode->pLogStore->data; code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, wal snapshot end error since:%s", pSyncNode->vgId, terrstr(terrno)); + sError("vgId:%d, wal snapshot end error since:%s", pSyncNode->vgId, terrstr()); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return -1; } else { do { @@ -525,12 +404,12 @@ int32_t syncEndSnapshot(int64_t rid) { } } - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return code; } int32_t syncStepDown(int64_t rid, SyncTerm newTerm) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; @@ -539,7 +418,7 @@ int32_t syncStepDown(int64_t rid, SyncTerm newTerm) { syncNodeStepDown(pSyncNode, newTerm); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } @@ -584,19 +463,19 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { } bool syncCanLeaderTransfer(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return false; } ASSERT(rid == pSyncNode->rid); if (pSyncNode->replicaNum == 1) { - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return false; } if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return true; } @@ -611,7 +490,7 @@ bool syncCanLeaderTransfer(int64_t rid) { } } - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return matchOK; } @@ -621,25 +500,25 @@ int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak) { } ESyncState syncGetMyRole(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } ASSERT(rid == pSyncNode->rid); ESyncState state = pSyncNode->state; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return state; } bool syncIsReady(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return false; } ASSERT(rid == pSyncNode->rid); bool b = (pSyncNode->state == TAOS_SYNC_STATE_LEADER) && pSyncNode->restoreFinish; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); // if false, set error code if (false == b) { @@ -653,14 +532,14 @@ bool syncIsReady(int64_t rid) { } bool syncIsRestoreFinish(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return false; } ASSERT(rid == pSyncNode->rid); bool b = pSyncNode->restoreFinish; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return b; } @@ -669,7 +548,7 @@ int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapsho return -1; } - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return -1; } @@ -681,7 +560,7 @@ int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapsho if (pEntry != NULL) { syncEntryDestory(pEntry); } - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return -1; } ASSERT(pEntry != NULL); @@ -692,12 +571,12 @@ int32_t syncGetSnapshotByIndex(int64_t rid, SyncIndex index, SSnapshot* pSnapsho pSnapshot->lastConfigIndex = syncNodeGetSnapshotConfigIndex(pSyncNode, index); syncEntryDestory(pEntry); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return -1; } @@ -706,12 +585,12 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->pRaftCfg->lastConfigIndex); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct SSnapshotMeta* sMeta) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return -1; } @@ -730,7 +609,7 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct sTrace("vgId:%d, get snapshot meta by index:%" PRId64 " lcindex:%" PRId64, pSyncNode->vgId, snapshotIndex, sMeta->lastConfigIndex); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return 0; } @@ -756,67 +635,67 @@ const char* syncGetMyRoleStr(int64_t rid) { } bool syncRestoreFinish(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return false; } ASSERT(rid == pSyncNode->rid); bool restoreFinish = pSyncNode->restoreFinish; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return restoreFinish; } SyncTerm syncGetMyTerm(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } ASSERT(rid == pSyncNode->rid); SyncTerm term = pSyncNode->pRaftStore->currentTerm; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return term; } SyncIndex syncGetLastIndex(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return SYNC_INDEX_INVALID; } ASSERT(rid == pSyncNode->rid); SyncIndex lastIndex = syncNodeGetLastIndex(pSyncNode); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return lastIndex; } SyncIndex syncGetCommitIndex(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return SYNC_INDEX_INVALID; } ASSERT(rid == pSyncNode->rid); SyncIndex cmtIndex = pSyncNode->commitIndex; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return cmtIndex; } SyncGroupId syncGetVgId(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } ASSERT(rid == pSyncNode->rid); SyncGroupId vgId = pSyncNode->vgId; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return vgId; } void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { memset(pEpSet, 0, sizeof(*pEpSet)); return; @@ -832,11 +711,11 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { pEpSet->inUse = pSyncNode->pRaftCfg->cfg.myIndex; sInfo("vgId:%d, sync get epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); } void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { memset(pEpSet, 0, sizeof(*pEpSet)); return; @@ -855,11 +734,11 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { } sInfo("vgId:%d, sync get retry epset in-use:%d", pSyncNode->vgId, pEpSet->inUse); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); } int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } @@ -871,12 +750,12 @@ int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { memcpy(msg, &(stub.rpcMsg), sizeof(SRpcMsg)); } - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return ret; } int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } @@ -889,12 +768,12 @@ int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo) } sTrace("vgId:%d, get seq:%" PRIu64 " rpc handle:%p", pSyncNode->vgId, index, pInfo->handle); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return ret; } void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { sTrace("syncSetQ get pSyncNode is NULL, rid:%" PRId64, rid); return; @@ -902,24 +781,24 @@ void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb) { ASSERT(rid == pSyncNode->rid); pSyncNode->msgcb = msgcb; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); } char* sync2SimpleStr(int64_t rid) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { sTrace("syncSetRpc get pSyncNode is NULL, rid:%" PRId64, rid); return NULL; } ASSERT(rid == pSyncNode->rid); char* s = syncNode2SimpleStr(pSyncNode); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return s; } void setPingTimerMS(int64_t rid, int32_t pingTimerMS) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return; } @@ -927,22 +806,22 @@ void setPingTimerMS(int64_t rid, int32_t pingTimerMS) { pSyncNode->pingBaseLine = pingTimerMS; pSyncNode->pingTimerMS = pingTimerMS; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); } void setElectTimerMS(int64_t rid, int32_t electTimerMS) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return; } ASSERT(rid == pSyncNode->rid); pSyncNode->electBaseLine = electTimerMS; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); } void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { - SSyncNode* pSyncNode = (SSyncNode*)taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { return; } @@ -950,20 +829,20 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { pSyncNode->hbBaseLine = hbTimerMS; pSyncNode->heartbeatTimerMS = hbTimerMS; - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); } int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { - SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); + SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) { - taosReleaseRef(tsNodeRefId, rid); + syncNodeRelease(pSyncNode); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } ASSERT(rid == pSyncNode->rid); int32_t ret = syncNodePropose(pSyncNode, pMsg, isWeak); - taosReleaseRef(tsNodeRefId, pSyncNode->rid); + syncNodeRelease(pSyncNode); return ret; } @@ -1085,7 +964,7 @@ int32_t syncHbTimerInit(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer, SRaftId de int32_t syncHbTimerStart(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { int32_t ret = 0; - if (syncEnvIsStart()) { + if (syncIsInit()) { SSyncHbTimerData* pData = taosMemoryMalloc(sizeof(SSyncHbTimerData)); pData->pSyncNode = pSyncNode; pData->pTimer = pSyncTimer; @@ -1093,7 +972,7 @@ int32_t syncHbTimerStart(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { pData->logicClock = pSyncTimer->logicClock; pSyncTimer->pData = pData; - taosTmrReset(pSyncTimer->timerCb, pSyncTimer->timerMS, pData, gSyncEnv->pTimerManager, &pSyncTimer->pTimer); + taosTmrReset(pSyncTimer->timerCb, pSyncTimer->timerMS, pData, syncEnv()->pTimerManager, &pSyncTimer->pTimer); } else { sError("vgId:%d, start ctrl hb timer error, sync env is stop", pSyncNode->vgId); } @@ -1558,8 +1437,8 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode) { // timer control -------------- int32_t syncNodeStartPingTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - if (syncEnvIsStart()) { - taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, + if (syncIsInit()) { + taosTmrReset(pSyncNode->FpPingTimerCB, pSyncNode->pingTimerMS, pSyncNode, syncEnv()->pTimerManager, &pSyncNode->pPingTimer); atomic_store_64(&pSyncNode->pingTimerLogicClock, pSyncNode->pingTimerLogicClockUser); } else { @@ -1578,7 +1457,7 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { int32_t ret = 0; - if (syncEnvIsStart()) { + if (syncIsInit()) { pSyncNode->electTimerMS = ms; SElectTimer* pElectTimer = taosMemoryMalloc(sizeof(SElectTimer)); @@ -1586,7 +1465,7 @@ int32_t syncNodeStartElectTimer(SSyncNode* pSyncNode, int32_t ms) { pElectTimer->pSyncNode = pSyncNode; pElectTimer->pData = NULL; - taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pElectTimer, gSyncEnv->pTimerManager, + taosTmrReset(pSyncNode->FpElectTimerCB, pSyncNode->electTimerMS, pElectTimer, syncEnv()->pTimerManager, &pSyncNode->pElectTimer); } else { @@ -1634,8 +1513,8 @@ int32_t syncNodeResetElectTimer(SSyncNode* pSyncNode) { static int32_t syncNodeDoStartHeartbeatTimer(SSyncNode* pSyncNode) { int32_t ret = 0; - if (syncEnvIsStart()) { - taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, + if (syncIsInit()) { + taosTmrReset(pSyncNode->FpHeartbeatTimerCB, pSyncNode->heartbeatTimerMS, pSyncNode, syncEnv()->pTimerManager, &pSyncNode->pHeartbeatTimer); atomic_store_64(&pSyncNode->heartbeatTimerLogicClock, pSyncNode->heartbeatTimerLogicClockUser); } else { @@ -2326,17 +2205,6 @@ _END: return; } -SSyncNode* syncNodeAcquire(int64_t rid) { - SSyncNode* pNode = taosAcquireRef(tsNodeRefId, rid); - if (pNode == NULL) { - sTrace("failed to acquire node from refId:%" PRId64, rid); - } - - return pNode; -} - -void syncNodeRelease(SSyncNode* pNode) { taosReleaseRef(tsNodeRefId, pNode->rid); } - // raft state change -------------- void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term) { if (term > pSyncNode->pRaftStore->currentTerm) { @@ -2787,8 +2655,8 @@ static void syncNodeEqPingTimer(void* param, void* tmrId) { } syncTimeoutDestroy(pSyncMsg); - if (syncEnvIsStart()) { - taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, gSyncEnv->pTimerManager, + if (syncIsInit()) { + taosTmrReset(syncNodeEqPingTimer, pSyncNode->pingTimerMS, pSyncNode, syncEnv()->pTimerManager, &pSyncNode->pPingTimer); } else { sError("sync env is stop, syncNodeEqPingTimer"); @@ -2808,7 +2676,7 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { pSyncNode->vgId, pSyncNode); SRpcMsg rpcMsg; syncTimeout2RpcMsg(pSyncMsg, &rpcMsg); - if (pSyncNode->FpEqMsg != NULL) { + if (pSyncNode->FpEqMsg != NULL && pSyncNode->msgcb != NULL && pSyncNode->msgcb->putToQueueFp != NULL) { int32_t code = pSyncNode->FpEqMsg(pSyncNode->msgcb, &rpcMsg); if (code != 0) { sError("vgId:%d, sync enqueue elect msg error, code:%d", pSyncNode->vgId, code); @@ -2833,9 +2701,9 @@ static void syncNodeEqElectTimer(void* param, void* tmrId) { #if 0 // reset timer ms - if (syncEnvIsStart() && pSyncNode->electBaseLine > 0) { + if (syncIsInit() && pSyncNode->electBaseLine > 0) { pSyncNode->electTimerMS = syncUtilElectRandomMS(pSyncNode->electBaseLine, 2 * pSyncNode->electBaseLine); - taosTmrReset(syncNodeEqElectTimer, pSyncNode->electTimerMS, pSyncNode, gSyncEnv->pTimerManager, + taosTmrReset(syncNodeEqElectTimer, pSyncNode->electTimerMS, pSyncNode, syncEnv()->pTimerManager, &pSyncNode->pElectTimer); } else { sError("sync env is stop, syncNodeEqElectTimer"); @@ -2870,8 +2738,8 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { } syncTimeoutDestroy(pSyncMsg); - if (syncEnvIsStart()) { - taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, gSyncEnv->pTimerManager, + if (syncIsInit()) { + taosTmrReset(syncNodeEqHeartbeatTimer, pSyncNode->heartbeatTimerMS, pSyncNode, syncEnv()->pTimerManager, &pSyncNode->pHeartbeatTimer); } else { sError("sync env is stop, syncNodeEqHeartbeatTimer"); @@ -2931,8 +2799,8 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { syncHeartbeatDestroy(pSyncMsg); - if (syncEnvIsStart()) { - taosTmrReset(syncNodeEqPeerHeartbeatTimer, pSyncTimer->timerMS, pData, gSyncEnv->pTimerManager, + if (syncIsInit()) { + taosTmrReset(syncNodeEqPeerHeartbeatTimer, pSyncTimer->timerMS, pData, syncEnv()->pTimerManager, &pSyncTimer->pTimer); } else { sError("sync env is stop, syncNodeEqHeartbeatTimer"); @@ -3424,6 +3292,7 @@ int32_t syncNodeDoCommit(SSyncNode* ths, SyncIndex beginIndex, SyncIndex endInde // ASSERT(pEntry != NULL); if (code != 0 || pEntry == NULL) { syncNodeErrorLog(ths, "get log entry error"); + sFatal("vgId:%d, get log entry %" PRId64 " error when commit since %s", ths->vgId, i, terrstr()); continue; } } @@ -3793,4 +3662,4 @@ void syncLogRecvLocalCmd(SSyncNode* pSyncNode, const SyncLocalCmd* pMsg, const c snprintf(logBuf, sizeof(logBuf), "recv sync-local-cmd {cmd:%d-%s, sd-new-term:%" PRIu64 "}, %s", pMsg->cmd, syncLocalCmdGetStr(pMsg->cmd), pMsg->sdNewTerm, s); syncNodeEventLog(pSyncNode, logBuf); -} \ No newline at end of file +} diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index 5ff73a6406..1c08217099 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -76,7 +76,7 @@ int32_t syncNodeTimerRoutine(SSyncNode* ths) { SSyncLogStoreData* pData = ths->pLogStore->data; int32_t code = walEndSnapshot(pData->pWal); if (code != 0) { - sError("vgId:%d, wal snapshot end error since:%s", ths->vgId, terrstr(terrno)); + sError("vgId:%d, timer wal snapshot end error since:%s", ths->vgId, terrstr()); return -1; } else { do { diff --git a/source/libs/sync/test/syncElectTest.cpp b/source/libs/sync/test/syncElectTest.cpp index 862f7bd0ba..d09879a699 100644 --- a/source/libs/sync/test/syncElectTest.cpp +++ b/source/libs/sync/test/syncElectTest.cpp @@ -98,7 +98,7 @@ int main(int argc, char** argv) { init(); int32_t ret = syncIOStart((char*)"127.0.0.1", gPorts[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); char walPath[128]; diff --git a/source/libs/sync/test/syncEncodeTest.cpp b/source/libs/sync/test/syncEncodeTest.cpp index 9f1a81e7ed..fdb5cf7ac8 100644 --- a/source/libs/sync/test/syncEncodeTest.cpp +++ b/source/libs/sync/test/syncEncodeTest.cpp @@ -152,7 +152,7 @@ int main(int argc, char **argv) { int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); taosRemoveDir("./wal_test"); diff --git a/source/libs/sync/test/syncEnqTest.cpp b/source/libs/sync/test/syncEnqTest.cpp index 8461bfe9b7..191e245b1e 100644 --- a/source/libs/sync/test/syncEnqTest.cpp +++ b/source/libs/sync/test/syncEnqTest.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); diff --git a/source/libs/sync/test/syncEnvTest.cpp b/source/libs/sync/test/syncEnvTest.cpp index a7a819e046..404ab32d58 100644 --- a/source/libs/sync/test/syncEnvTest.cpp +++ b/source/libs/sync/test/syncEnvTest.cpp @@ -22,7 +22,7 @@ int main() { logTest(); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); for (int i = 0; i < 5; ++i) { @@ -37,8 +37,6 @@ int main() { taosMsleep(5000); } - ret = syncEnvStop(); - assert(ret == 0); - + syncCleanUp(); return 0; } diff --git a/source/libs/sync/test/syncIOSendMsgTest.cpp b/source/libs/sync/test/syncIOSendMsgTest.cpp index 630d96054b..4a457136c2 100644 --- a/source/libs/sync/test/syncIOSendMsgTest.cpp +++ b/source/libs/sync/test/syncIOSendMsgTest.cpp @@ -82,7 +82,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); diff --git a/source/libs/sync/test/syncInitTest.cpp b/source/libs/sync/test/syncInitTest.cpp index d0843151f4..d654ad06fe 100644 --- a/source/libs/sync/test/syncInitTest.cpp +++ b/source/libs/sync/test/syncInitTest.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); @@ -91,7 +91,7 @@ int main(int argc, char** argv) { initRaftId(pSyncNode); syncNodeClose(pSyncNode); - syncEnvStop(); + syncCleanUp(); // syncIOStop(); // taosCloseLog(); diff --git a/source/libs/sync/test/syncPingSelfTest.cpp b/source/libs/sync/test/syncPingSelfTest.cpp index 99287bf7b0..f44cbb04d5 100644 --- a/source/libs/sync/test/syncPingSelfTest.cpp +++ b/source/libs/sync/test/syncPingSelfTest.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); diff --git a/source/libs/sync/test/syncPingTimerTest.cpp b/source/libs/sync/test/syncPingTimerTest.cpp index cd9440e3e2..fd6342aa84 100644 --- a/source/libs/sync/test/syncPingTimerTest.cpp +++ b/source/libs/sync/test/syncPingTimerTest.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); diff --git a/source/libs/sync/test/syncPingTimerTest2.cpp b/source/libs/sync/test/syncPingTimerTest2.cpp index fa09d04368..295003dff3 100644 --- a/source/libs/sync/test/syncPingTimerTest2.cpp +++ b/source/libs/sync/test/syncPingTimerTest2.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); diff --git a/source/libs/sync/test/syncSnapshotTest.cpp b/source/libs/sync/test/syncSnapshotTest.cpp index 3dc8518072..50771ac476 100644 --- a/source/libs/sync/test/syncSnapshotTest.cpp +++ b/source/libs/sync/test/syncSnapshotTest.cpp @@ -179,7 +179,7 @@ int main(int argc, char **argv) { int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); // taosRemoveDir(pWalDir); diff --git a/source/libs/sync/test/syncVotesGrantedTest.cpp b/source/libs/sync/test/syncVotesGrantedTest.cpp index d4885d0316..e2e8748697 100644 --- a/source/libs/sync/test/syncVotesGrantedTest.cpp +++ b/source/libs/sync/test/syncVotesGrantedTest.cpp @@ -82,7 +82,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); diff --git a/source/libs/sync/test/syncVotesRespondTest.cpp b/source/libs/sync/test/syncVotesRespondTest.cpp index 77262dfc65..881a5331b1 100644 --- a/source/libs/sync/test/syncVotesRespondTest.cpp +++ b/source/libs/sync/test/syncVotesRespondTest.cpp @@ -82,7 +82,7 @@ int main(int argc, char** argv) { int32_t ret = syncIOStart((char*)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); SSyncNode* pSyncNode = syncInitTest(); diff --git a/source/libs/sync/test/syncWriteTest.cpp b/source/libs/sync/test/syncWriteTest.cpp index b185f52f75..fee98ddd52 100644 --- a/source/libs/sync/test/syncWriteTest.cpp +++ b/source/libs/sync/test/syncWriteTest.cpp @@ -154,7 +154,7 @@ int main(int argc, char **argv) { int32_t ret = syncIOStart((char *)"127.0.0.1", ports[myIndex]); assert(ret == 0); - ret = syncEnvStart(); + ret = syncInit(); assert(ret == 0); taosRemoveDir("./wal_test"); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index b83f84e3f2..2354f0f959 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -428,6 +428,7 @@ void transDestoryExHandle(void* handle); int32_t transGetRefMgt(); int32_t transGetInstMgt(); +void transHttpEnvDestroy(); #ifdef __cplusplus } #endif diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 92989a45f5..00854b5ee5 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -20,21 +20,52 @@ #include "thttp.h" #include "taoserror.h" #include "tlog.h" +#include "transComm.h" // clang-format on #define HTTP_RECV_BUF_SIZE 1024 +typedef struct SHttpModule { + uv_loop_t* loop; + SAsyncPool* asyncPool; + TdThread thread; +} SHttpModule; + +typedef struct SHttpMsg { + queue q; + char* server; + int32_t port; + char* cont; + int32_t len; + EHttpCompFlag flag; + int8_t quit; + SHttpModule* http; + +} SHttpMsg; + typedef struct SHttpClient { - uv_connect_t conn; - uv_tcp_t tcp; - uv_write_t req; - uv_buf_t* wbuf; - char* rbuf; - char* addr; - uint16_t port; + uv_connect_t conn; + uv_tcp_t tcp; + uv_write_t req; + uv_buf_t* wbuf; + char* rbuf; + char* addr; + uint16_t port; + struct sockaddr_in dest; } SHttpClient; +static TdThreadOnce transHttpInit = PTHREAD_ONCE_INIT; +static SHttpModule* thttp = NULL; +static void transHttpEnvInit(); + +static void httpHandleReq(SHttpMsg* msg); +static void httpHandleQuit(SHttpMsg* msg); +static int32_t httpSendQuit(); + +static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag); + static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pHead, int32_t headLen, EHttpCompFlag flag) { if (flag == HTTP_FLAT) { @@ -53,6 +84,7 @@ static int32_t taosBuildHttpHeader(const char* server, int32_t contLen, char* pH "Content-Length: %d\n\n", server, contLen); } else { + terrno = TSDB_CODE_INVALID_CFG; return -1; } } @@ -126,88 +158,10 @@ _OVER: return code; } -static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) { - taosMemoryFree(cli->wbuf); - taosMemoryFree(cli->rbuf); - taosMemoryFree(cli->addr); - taosMemoryFree(cli); -} -static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) { - SHttpClient* cli = handle->data; - destroyHttpClient(cli); -} -static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { - SHttpClient* cli = handle->data; - buf->base = cli->rbuf; - buf->len = HTTP_RECV_BUF_SIZE; -} -static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { - SHttpClient* cli = handle->data; - if (nread < 0) { - uError("http-report recv error:%s", uv_err_name(nread)); - } else { - uTrace("http-report succ to recv %d bytes", (int32_t)nread); - } - if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { - uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); - } -} -static void clientSentCb(uv_write_t* req, int32_t status) { - SHttpClient* cli = req->data; - if (status != 0) { - terrno = TAOS_SYSTEM_ERROR(status); - uError("http-report failed to send data %s", uv_strerror(status)); - if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { - uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); - } - return; - } else { - uTrace("http-report succ to send data"); - } - status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); - if (status != 0) { - terrno = TAOS_SYSTEM_ERROR(status); - uError("http-report failed to recv data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); - if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { - uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); - } - } -} -static void clientConnCb(uv_connect_t* req, int32_t status) { - SHttpClient* cli = req->data; - if (status != 0) { - terrno = TAOS_SYSTEM_ERROR(status); - uError("http-report failed to conn to server, reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); - if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { - uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); - } - return; - } - status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); - if (0 != status) { - terrno = TAOS_SYSTEM_ERROR(status); - uError("http-report failed to send data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); - if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { - uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); - } else { - destroyHttpClient(cli); - } - } -} - static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr()); + tError("http-report failed to get http server:%s since %s", server, errno == 0 ? "invalid http server" : terrstr()); return -1; } char buf[128] = {0}; @@ -215,27 +169,206 @@ static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, uv_ip4_addr(buf, port, dest); return 0; } -int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { - struct sockaddr_in dest = {0}; - if (taosBuildDstAddr(server, port, &dest) < 0) { - return -1; - } - if (flag == HTTP_GZIP) { - int32_t dstLen = taosCompressHttpRport(pCont, contLen); - if (dstLen > 0) { - contLen = dstLen; + +static void* httpThread(void* arg) { + SHttpModule* http = (SHttpModule*)arg; + setThreadName("http-cli-send-thread"); + uv_run(http->loop, UV_RUN_DEFAULT); + return NULL; +} + +static void httpDestroyMsg(SHttpMsg* msg) { + if (msg == NULL) return; + + taosMemoryFree(msg->server); + taosMemoryFree(msg->cont); + taosMemoryFree(msg); +} +static void httpAsyncCb(uv_async_t* handle) { + SAsyncItem* item = handle->data; + SHttpModule* http = item->pThrd; + + SHttpMsg *msg = NULL, *quitMsg = NULL; + + queue wq; + taosThreadMutexLock(&item->mtx); + QUEUE_MOVE(&item->qmsg, &wq); + taosThreadMutexUnlock(&item->mtx); + + int count = 0; + while (!QUEUE_IS_EMPTY(&wq)) { + queue* h = QUEUE_HEAD(&wq); + QUEUE_REMOVE(h); + msg = QUEUE_DATA(h, SHttpMsg, q); + if (msg->quit) { + quitMsg = msg; } else { - flag = HTTP_FLAT; + httpHandleReq(msg); } } - terrno = 0; + if (quitMsg) httpHandleQuit(quitMsg); +} - char header[2048] = {0}; - int32_t headLen = taosBuildHttpHeader(server, contLen, header, sizeof(header), flag); +static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) { + taosMemoryFree(cli->wbuf[0].base); + taosMemoryFree(cli->wbuf[1].base); + taosMemoryFree(cli->wbuf); + taosMemoryFree(cli->rbuf); + taosMemoryFree(cli->addr); + taosMemoryFree(cli); +} + +static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) { + SHttpClient* cli = handle->data; + destroyHttpClient(cli); +} + +static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + SHttpClient* cli = handle->data; + buf->base = cli->rbuf; + buf->len = HTTP_RECV_BUF_SIZE; +} + +static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { + SHttpClient* cli = handle->data; + if (nread < 0) { + tError("http-report recv error:%s", uv_err_name(nread)); + } else { + tTrace("http-report succ to recv %d bytes", (int32_t)nread); + } + if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + } +} +static void clientSentCb(uv_write_t* req, int32_t status) { + SHttpClient* cli = req->data; + if (status != 0) { + tError("http-report failed to send data, reason: %s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + } + return; + } else { + tTrace("http-report succ to send data"); + } + status = uv_read_start((uv_stream_t*)&cli->tcp, clientAllocBuffCb, clientRecvCb); + if (status != 0) { + tError("http-report failed to recv data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + } + } +} +static void clientConnCb(uv_connect_t* req, int32_t status) { + SHttpClient* cli = req->data; + if (status != 0) { + tError("http-report failed to conn to server, reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + } + return; + } + status = uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); + if (0 != status) { + tError("http-report failed to send data,reason:%s, dst:%s:%d", uv_strerror(status), cli->addr, cli->port); + if (!uv_is_closing((uv_handle_t*)&cli->tcp)) { + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + } + } +} + +int32_t httpSendQuit() { + SHttpMsg* msg = taosMemoryCalloc(1, sizeof(SHttpMsg)); + msg->quit = 1; + + SHttpModule* load = atomic_load_ptr(&thttp); + if (load == NULL) { + httpDestroyMsg(msg); + tError("http-report already released"); + return -1; + } else { + msg->http = load; + } + transAsyncSend(load->asyncPool, &(msg->q)); + return 0; +} + +static int32_t taosSendHttpReportImpl(const char* server, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag) { + SHttpMsg* msg = taosMemoryMalloc(sizeof(SHttpMsg)); + msg->server = strdup(server); + msg->port = port; + msg->cont = taosMemoryMalloc(contLen); + memcpy(msg->cont, pCont, contLen); + msg->len = contLen; + msg->flag = flag; + msg->quit = 0; + + SHttpModule* load = atomic_load_ptr(&thttp); + if (load == NULL) { + httpDestroyMsg(msg); + tError("http-report already released"); + return -1; + } else { + msg->http = load; + transAsyncSend(load->asyncPool, &(msg->q)); + } + + return 0; +} + +static void httpDestroyClientCb(uv_handle_t* handle) { + SHttpClient* http = handle->data; + destroyHttpClient(http); +} +static void httpWalkCb(uv_handle_t* handle, void* arg) { + // impl later + if (!uv_is_closing(handle)) { + uv_handle_type type = uv_handle_get_type(handle); + if (uv_handle_get_type(handle) == UV_TCP) { + uv_close(handle, httpDestroyClientCb); + } else { + uv_close(handle, NULL); + } + } + return; +} +static void httpHandleQuit(SHttpMsg* msg) { + SHttpModule* http = msg->http; + taosMemoryFree(msg); + + uv_walk(http->loop, httpWalkCb, NULL); +} +static void httpHandleReq(SHttpMsg* msg) { + SHttpModule* http = msg->http; + + struct sockaddr_in dest = {0}; + if (taosBuildDstAddr(msg->server, msg->port, &dest) < 0) { + goto END; + } + if (msg->flag == HTTP_GZIP) { + int32_t dstLen = taosCompressHttpRport(msg->cont, msg->len); + if (dstLen > 0) { + msg->len = dstLen; + } else { + msg->flag = HTTP_FLAT; + } + if (dstLen < 0) { + goto END; + } + } + + int32_t len = 2048; + char* header = taosMemoryCalloc(1, len); + int32_t headLen = taosBuildHttpHeader(msg->server, msg->len, header, len, msg->flag); + if (headLen < 0) { + taosMemoryFree(header); + goto END; + } uv_buf_t* wb = taosMemoryCalloc(2, sizeof(uv_buf_t)); - wb[0] = uv_buf_init((char*)header, headLen); // stack var - wb[1] = uv_buf_init((char*)pCont, contLen); // heap var + wb[0] = uv_buf_init((char*)header, strlen(header)); // heap var + wb[1] = uv_buf_init((char*)msg->cont, msg->len); // heap var SHttpClient* cli = taosMemoryCalloc(1, sizeof(SHttpClient)); cli->conn.data = cli; @@ -243,41 +376,71 @@ int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32 cli->req.data = cli; cli->wbuf = wb; cli->rbuf = taosMemoryCalloc(1, HTTP_RECV_BUF_SIZE); - cli->addr = tstrdup(server); - cli->port = port; + cli->addr = msg->server; + cli->port = msg->port; + cli->dest = dest; + + taosMemoryFree(msg); + + uv_tcp_init(http->loop, &cli->tcp); - uv_loop_t* loop = taosMemoryMalloc(sizeof(uv_loop_t)); - int err = uv_loop_init(loop); - if (err != 0) { - uError("http-report failed to init uv_loop, reason: %s", uv_strerror(err)); - taosMemoryFree(loop); - terrno = TAOS_SYSTEM_ERROR(err); - destroyHttpClient(cli); - return terrno; - } - uv_tcp_init(loop, &cli->tcp); // set up timeout to avoid stuck; int32_t fd = taosCreateSocketWithTimeout(5); - - int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); + int ret = uv_tcp_open((uv_tcp_t*)&cli->tcp, fd); if (ret != 0) { - uError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); + tError("http-report failed to open socket, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, cli->port); destroyHttpClient(cli); - uv_stop(loop); - terrno = TAOS_SYSTEM_ERROR(ret); - } else { - ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&dest, clientConnCb); - if (ret != 0) { - uError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, - cli->port); - destroyHttpClient(cli); - uv_stop(loop); - terrno = TAOS_SYSTEM_ERROR(ret); - } + return; } - uv_run(loop, UV_RUN_DEFAULT); - uv_loop_close(loop); - taosMemoryFree(loop); - return terrno; + ret = uv_tcp_connect(&cli->conn, &cli->tcp, (const struct sockaddr*)&cli->dest, clientConnCb); + if (ret != 0) { + tError("http-report failed to connect to http-server, reason:%s, dst:%s:%d", uv_strerror(ret), cli->addr, + cli->port); + destroyHttpClient(cli); + } + return; + +END: + tError("http-report failed to report, reason: %s, addr: %s:%d", terrstr(), msg->server, msg->port); + httpDestroyMsg(msg); +} + +int32_t taosSendHttpReport(const char* server, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { + taosThreadOnce(&transHttpInit, transHttpEnvInit); + return taosSendHttpReportImpl(server, port, pCont, contLen, flag); +} + +static void transHttpEnvInit() { + SHttpModule* http = taosMemoryMalloc(sizeof(SHttpModule)); + + http->loop = taosMemoryMalloc(sizeof(uv_loop_t)); + uv_loop_init(http->loop); + + http->asyncPool = transAsyncPoolCreate(http->loop, 1, http, httpAsyncCb); + + int err = taosThreadCreate(&http->thread, NULL, httpThread, (void*)http); + if (err != 0) { + taosMemoryFree(http->loop); + taosMemoryFree(http); + http = NULL; + } + atomic_store_ptr(&thttp, http); +} + +void transHttpEnvDestroy() { + SHttpModule* load = atomic_load_ptr(&thttp); + if (load == NULL) { + return; + } + httpSendQuit(); + taosThreadJoin(load->thread, NULL); + + TRANS_DESTROY_ASYNC_POOL_MSG(load->asyncPool, SHttpMsg, httpDestroyMsg); + transAsyncPoolDestroy(load->asyncPool); + uv_loop_close(load->loop); + taosMemoryFree(load->loop); + taosMemoryFree(load); + + atomic_store_ptr(&thttp, NULL); } diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 756a8ff2cf..d3db4879d1 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -172,6 +172,8 @@ int32_t rpcInit() { } void rpcCleanup(void) { transCleanup(); + transHttpEnvDestroy(); + return; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 28660934f8..97915e1ded 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -187,18 +187,8 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \ } while (0) -#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \ - do { \ - if (exh == NULL) { \ - idx = -1; \ - } else { \ - ASYNC_CHECK_HANDLE((exh), refId); \ - pThrd = (SCliThrd*)(exh)->pThrd; \ - } \ - } while (0) -#define CONN_PERSIST_TIME(para) ((para) <= 90000 ? 90000 : (para)) -#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_PERSIST_TIME(para) ((para) <= 90000 ? 90000 : (para)) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) #define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ do { \ @@ -217,6 +207,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { tDebug("msg found, %" PRIu64 "", ahandle); \ } \ } while (0) + #define CONN_GET_NEXT_SENDMSG(conn) \ do { \ int i = 0; \ @@ -231,21 +222,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ } while (0) -#define CONN_HANDLE_THREAD_QUIT(thrd) \ - do { \ - if (thrd->quit) { \ - return; \ - } \ - } while (0) - -#define CONN_HANDLE_BROKEN(conn) \ - do { \ - if (conn->broken) { \ - cliHandleExcept(conn); \ - return; \ - } \ - } while (0) - #define CONN_SET_PERSIST_BY_APP(conn) \ do { \ if (conn->status == ConnNormal) { \ diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index b489df2f4c..ad27a50f68 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -288,6 +288,33 @@ void walAlignVersions(SWal* pWal) { pWal->vers.appliedVer = TMIN(pWal->vers.commitVer, pWal->vers.appliedVer); } +bool walLogEntriesComplete(const SWal* pWal) { + int32_t sz = taosArrayGetSize(pWal->fileInfoSet); + bool complete = true; + int32_t fileIdx = -1; + int64_t index = pWal->vers.firstVer; + + while (++fileIdx < sz) { + SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); + if (pFileInfo->firstVer != index) { + break; + } + index = pFileInfo->lastVer + ((fileIdx + 1 < sz) ? 1 : 0); + } + // empty is regarded as complete + if (sz != 0) { + complete = (index == pWal->vers.lastVer); + } + + if (!complete) { + wError("vgId:%d, WAL log entries incomplete in range [%" PRId64 ", %" PRId64 "], aligned with snaphotVer:%" PRId64, + pWal->cfg.vgId, pWal->vers.firstVer, pWal->vers.lastVer, pWal->vers.snapshotVer); + terrno = TSDB_CODE_WAL_LOG_INCOMPLETE; + } + + return complete; +} + int walCheckAndRepairMeta(SWal* pWal) { // load log files, get first/snapshot/last version info const char* logPattern = "^[0-9]+.log$"; @@ -401,6 +428,11 @@ int walCheckAndRepairMeta(SWal* pWal) { if (updateMeta) { (void)walSaveMeta(pWal); } + + if (!walLogEntriesComplete(pWal)) { + return -1; + } + return 0; } diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 527ffa0056..e016e72471 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -101,7 +101,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { taosThreadMutexLock(&pWal->mutex); int64_t code; char fnameStr[WAL_FILE_LEN]; - if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { + if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer || ver <= pWal->vers.snapshotVer) { terrno = TSDB_CODE_WAL_INVALID_VER; taosThreadMutexUnlock(&pWal->mutex); return -1; diff --git a/source/libs/wal/test/walMetaTest.cpp b/source/libs/wal/test/walMetaTest.cpp index d98826a384..891e7dcdae 100644 --- a/source/libs/wal/test/walMetaTest.cpp +++ b/source/libs/wal/test/walMetaTest.cpp @@ -274,17 +274,17 @@ TEST_F(WalCleanEnv, rollbackMultiFile) { code = walRollback(pWal, 9); ASSERT_EQ(code, 0); ASSERT_EQ(pWal->vers.lastVer, 8); + code = walRollback(pWal, 6); + ASSERT_EQ(code, 0); + ASSERT_EQ(pWal->vers.lastVer, 5); code = walRollback(pWal, 5); - ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 4); - code = walRollback(pWal, 3); - ASSERT_EQ(code, 0); + ASSERT_EQ(code, -1); - ASSERT_EQ(pWal->vers.lastVer, 2); + ASSERT_EQ(pWal->vers.lastVer, 5); - code = walWrite(pWal, 3, 3, (void*)ranStr, ranStrLen); + code = walWrite(pWal, 6, 6, (void*)ranStr, ranStrLen); ASSERT_EQ(code, 0); - ASSERT_EQ(pWal->vers.lastVer, 3); + ASSERT_EQ(pWal->vers.lastVer, 6); code = walSaveMeta(pWal); ASSERT_EQ(code, 0); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 37f9734eab..bbdf43c85e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -452,6 +452,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid versi TAOS_DEFINE_ERROR(TSDB_CODE_WAL_OUT_OF_MEMORY, "WAL out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE, "WAL log incomplete") // tfs TAOS_DEFINE_ERROR(TSDB_CODE_FS_INVLD_CFG, "tfs invalid mount config") diff --git a/source/util/src/tref.c b/source/util/src/tref.c index c984ef3f34..aa741b909a 100644 --- a/source/util/src/tref.c +++ b/source/util/src/tref.c @@ -57,7 +57,7 @@ static void taosIncRsetCount(SRefSet *pSet); static void taosDecRsetCount(SRefSet *pSet); static int32_t taosDecRefCount(int32_t rsetId, int64_t rid, int32_t remove); -int32_t taosOpenRef(int32_t max, void (*fp)(void *)) { +int32_t taosOpenRef(int32_t max, RefFp fp) { SRefNode **nodeList; SRefSet *pSet; int64_t *lockedBy; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 6328124b0d..1a80aa0681 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -234,6 +234,7 @@ ./test.sh -f tsim/stream/basic1.sim ./test.sh -f tsim/stream/basic2.sim ./test.sh -f tsim/stream/drop_stream.sim +./test.sh -f tsim/stream/fillHistoryBasic1.sim ./test.sh -f tsim/stream/distributeInterval0.sim ./test.sh -f tsim/stream/distributeIntervalRetrive0.sim ./test.sh -f tsim/stream/distributeSession0.sim diff --git a/tests/script/tsim/stream/fillHistoryBasic1.sim b/tests/script/tsim/stream/fillHistoryBasic1.sim new file mode 100644 index 0000000000..5bbaf1b712 --- /dev/null +++ b/tests/script/tsim/stream/fillHistoryBasic1.sim @@ -0,0 +1,926 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print =============== create database +sql create database test vgroups 1; +sql select * from information_schema.ins_databases +if $rows != 3 then + return -1 +endi + +print $data00 $data01 $data02 + +sql use test; + +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream1 trigger at_once fill_history 1 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sql insert into t1 values(1648791243003,4,2,3,3.1); +sql insert into t1 values(1648791213004,4,2,3,4.1); + + +sleep 1000 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 2 then + print ======$data13 + return -1 +endi + +if $data14 != 2 then + print ======$data14 + return -1 +endi + +if $data15 != 3 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223001,12,14,13,11.1); +sleep 500 +sql select * from streamt; + +print count(*) , count(d) , sum(a) , max(b) , min(c) +print 0: $data00 , $data01 , $data02 , $data03 , $data04 , $data05 +print 1: $data10 , $data11 , $data12 , $data13 , $data14 , $data15 + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 12 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223002,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 2 then + print ======$data11 + return -1 +endi + +if $data12 != 2 then + print ======$data12 + return -1 +endi + +if $data13 != 24 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223003,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 36 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223001,1,1,1,1.1); +sql insert into t1 values(1648791223002,2,2,2,2.1); +sql insert into t1 values(1648791223003,3,3,3,3.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 6 then + print ======$data13 + return -1 +endi + +if $data14 != 3 then + print ======$data14 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791233003,3,2,3,2.1); +sql insert into t1 values(1648791233002,5,6,7,8.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 2 +if $data21 != 2 then + print ======$data21 + return -1 +endi + +if $data22 != 2 then + print ======$data22 + return -1 +endi + +if $data23 != 6 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 0 +if $data01 != 4 then + print ======$data01 + return -1 +endi + +if $data02 != 4 then + print ======$data02 + return -1 +endi + +if $data03 != 50 then + print ======$data03 != 50 + return -1 +endi + +if $data04 != 20 then + print ======$data04 != 20 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 4 then + print ======$data11 + return -1 +endi + +if $data12 != 4 then + print ======$data12 + return -1 +endi + +if $data13 != 46 then + print ======$data13 != 46 + return -1 +endi + +if $data14 != 20 then + print ======$data14 != 20 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 4 then + print ======$data21 + return -1 +endi + +if $data22 != 4 then + print ======$data22 + return -1 +endi + +if $data23 != 15 then + print ======$data23 + return -1 +endi + +if $data24 != 4 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + + + + + +sql create database test2 vgroups 1; +sql select * from information_schema.ins_databases; + +sql use test2; + +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sql insert into t1 values(1648791243003,4,2,3,3.1); +sql insert into t1 values(1648791213004,4,2,3,4.1); + +sql create stream stream2 trigger at_once fill_history 1 into streamt as select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s); + +sleep 1000 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 2 then + print ======$data13 + return -1 +endi + +if $data14 != 2 then + print ======$data14 + return -1 +endi + +if $data15 != 3 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223001,12,14,13,11.1); +sleep 500 +sql select * from streamt; + +print count(*) , count(d) , sum(a) , max(b) , min(c) +print 0: $data00 , $data01 , $data02 , $data03 , $data04 , $data05 +print 1: $data10 , $data11 , $data12 , $data13 , $data14 , $data15 + +if $rows != 4 then + print ======$rows + return -1 +endi + +# row 0 +if $data01 != 2 then + print ======$data01 + return -1 +endi + +if $data02 != 2 then + print ======$data02 + return -1 +endi + +if $data03 != 5 then + print ======$data03 + return -1 +endi + +if $data04 != 2 then + print ======$data04 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +# row 1 +if $data11 != 1 then + print ======$data11 + return -1 +endi + +if $data12 != 1 then + print ======$data12 + return -1 +endi + +if $data13 != 12 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 1 then + print ======$data21 + return -1 +endi + +if $data22 != 1 then + print ======$data22 + return -1 +endi + +if $data23 != 3 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +# row 3 +if $data31 != 1 then + print ======$data31 + return -1 +endi + +if $data32 != 1 then + print ======$data32 + return -1 +endi + +if $data33 != 4 then + print ======$data33 + return -1 +endi + +if $data34 != 2 then + print ======$data34 + return -1 +endi + +if $data35 != 3 then + print ======$data35 + return -1 +endi + +sql insert into t1 values(1648791223002,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 2 then + print ======$data11 + return -1 +endi + +if $data12 != 2 then + print ======$data12 + return -1 +endi + +if $data13 != 24 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223003,12,14,13,11.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 36 then + print ======$data13 + return -1 +endi + +if $data14 != 14 then + print ======$data14 + return -1 +endi + +if $data15 != 13 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791223001,1,1,1,1.1); +sql insert into t1 values(1648791223002,2,2,2,2.1); +sql insert into t1 values(1648791223003,3,3,3,3.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 3 then + print ======$data11 + return -1 +endi + +if $data12 != 3 then + print ======$data12 + return -1 +endi + +if $data13 != 6 then + print ======$data13 + return -1 +endi + +if $data14 != 3 then + print ======$data14 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +sql insert into t1 values(1648791233003,3,2,3,2.1); +sql insert into t1 values(1648791233002,5,6,7,8.1); +sql insert into t1 values(1648791233002,3,2,3,2.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 2 +if $data21 != 2 then + print ======$data21 + return -1 +endi + +if $data22 != 2 then + print ======$data22 + return -1 +endi + +if $data23 != 6 then + print ======$data23 + return -1 +endi + +if $data24 != 2 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + +sql insert into t1 values(1648791213004,4,2,3,4.1) (1648791213006,5,4,7,9.1) (1648791213004,40,20,30,40.1) (1648791213005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 0 +if $data01 != 4 then + print ======$data01 + return -1 +endi + +if $data02 != 4 then + print ======$data02 + return -1 +endi + +if $data03 != 50 then + print ======$data03 != 50 + return -1 +endi + +if $data04 != 20 then + print ======$data04 != 20 + return -1 +endi + +if $data05 != 3 then + print ======$data05 + return -1 +endi + +sql insert into t1 values(1648791223004,4,2,3,4.1) (1648791233006,5,4,7,9.1) (1648791223004,40,20,30,40.1) (1648791233005,4,2,3,4.1); +sleep 100 +sql select `_wstart`, c1, c2 ,c3 ,c4, c5 from streamt; + +# row 1 +if $data11 != 4 then + print ======$data11 + return -1 +endi + +if $data12 != 4 then + print ======$data12 + return -1 +endi + +if $data13 != 46 then + print ======$data13 != 46 + return -1 +endi + +if $data14 != 20 then + print ======$data14 != 20 + return -1 +endi + +if $data15 != 1 then + print ======$data15 + return -1 +endi + +# row 2 +if $data21 != 4 then + print ======$data21 + return -1 +endi + +if $data22 != 4 then + print ======$data22 + return -1 +endi + +if $data23 != 15 then + print ======$data23 + return -1 +endi + +if $data24 != 4 then + print ======$data24 + return -1 +endi + +if $data25 != 3 then + print ======$data25 + return -1 +endi + + diff --git a/tests/script/tsim/vnode/replica3_repeat.sim b/tests/script/tsim/vnode/replica3_repeat.sim index 4902cde7c5..ccff06819b 100644 --- a/tests/script/tsim/vnode/replica3_repeat.sim +++ b/tests/script/tsim/vnode/replica3_repeat.sim @@ -107,7 +107,16 @@ system sh/exec.sh -n dnode2 -s stop sleep 3000 print ======== step7 -sql select count(*) from db.tb +$x = 0 +step7: + $x = $x + 1 + sleep 1000 + if $x == 30 then + print ====> dnode not ready! + return -1 + endi +sql select count(*) from db.tb -x step7 + print select count(*) from db.tb ==> $data00 $lastRows if $data00 <= $lastRows then return -1 diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index 054afe4fef..714f10b362 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -21,7 +21,7 @@ class TDTestCase: self.ctbNum = 100 self.rowsPerTbl = 1000 - def init(self, conn, logSql): + def init(self, conn, logSql, replicaVar=1): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), False) diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index 7ee9e5d169..5c858d0dce 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -244,7 +244,7 @@ class TDTestCase: tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 9000000 # Forever loop + paraDict['pollDelay'] = 9000000 # Forever loop showMsg = 1 showRow = 1 #self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py index d0b46bf572..0a21680e78 100644 --- a/tests/system-test/7-tmq/tmqShow.py +++ b/tests/system-test/7-tmq/tmqShow.py @@ -138,8 +138,9 @@ class TDTestCase: tdSql.query("show subscriptions") # tdLog.debug(tdSql.queryResult) rows = tdSql.getRows() - tdLog.info("show subscriptions rows: %d"%rows) - if rows != paraDict['vgroups'] * len(topicNameList): + expectSubscriptions = paraDict['vgroups'] * len(topicNameList) + tdLog.info("show subscriptions rows: %d, expect Subscriptions: %d"%(rows,expectSubscriptions)) + if rows != expectSubscriptions: tdLog.exit("show subscriptions rows error") pThread.join() diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 31dcde036d..1e7d0ed140 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -2,6 +2,14 @@ aux_source_directory(src SHELL_SRC) add_executable(shell ${SHELL_SRC}) +IF (TD_LINUX_64 AND JEMALLOC_ENABLED) + ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) + SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") + ADD_DEPENDENCIES(shell jemalloc) +ELSE () + SET(LINK_JEMALLOC "") +ENDIF () + IF (TD_LINUX AND TD_WEBSOCKET) ADD_DEFINITIONS(-DWEBSOCKET -I${CMAKE_BINARY_DIR}/build/include -ltaosws) SET(LINK_WEBSOCKET "-L${CMAKE_BINARY_DIR}/build/lib -ltaosws") @@ -21,7 +29,7 @@ ENDIF () if(TD_WINDOWS) target_link_libraries(shell PUBLIC taos_static ${LINK_WEBSOCKET}) else() - target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET}) + target_link_libraries(shell PUBLIC taos ${LINK_WEBSOCKET} ${LINK_JEMALLOC}) endif () target_link_libraries( shell