From 2af2eb40e5cacb8ffa7f14ac4179d43e862e5bd6 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 26 Jun 2024 09:16:33 +0800 Subject: [PATCH 1/2] fix: adjust TSDB_CODE_PAR_TBNAME_ERROR error message --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 71812ee14d..2f29469458 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -673,7 +673,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_COL_PK_TYPE, "primary key column TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INVALID_PK_OP, "primary key column can not be added, modified, and dropped") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL, "Primary key column should not be null") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE, "Primary key column should not be none") -TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_ERROR, "Tbanme duplicated or not set") +TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_ERROR, "Pseudo tag tbname not set") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TBNAME_DUPLICATED, "Table name duplicated") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_TAG_NAME_DUPLICATED, "Tag name duplicated") TAOS_DEFINE_ERROR(TSDB_CODE_PAR_INTERNAL_ERROR, "Parser internal error") From b761bd2e3870a77ce21e695c88810a8a7f10933d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 26 Jun 2024 10:32:15 +0800 Subject: [PATCH 2/2] enh: port main change to 3.0 --- include/libs/stream/streammsg.h | 12 +- include/libs/stream/tstream.h | 115 ++++---- source/client/src/clientMain.c | 1 + source/client/src/clientStmt.c | 204 +++++++------- source/client/src/clientTmq.c | 116 ++++---- source/common/src/cos.c | 26 +- source/common/src/cos_cp.c | 9 +- source/common/src/rsync.c | 67 ++--- source/common/src/tdataformat.c | 2 +- source/common/src/tglobal.c | 4 +- source/common/src/tmsg.c | 5 +- source/common/src/tvariant.c | 21 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 9 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 1 + source/dnode/mnode/impl/src/mndStream.c | 103 ++++--- source/dnode/mnode/impl/src/mndStreamHb.c | 36 +-- source/dnode/mnode/impl/src/mndStreamTrans.c | 36 +-- source/dnode/mnode/impl/src/mndSubscribe.c | 265 +++++++++--------- source/dnode/snode/src/snode.c | 12 +- source/dnode/vnode/src/meta/metaCache.c | 10 +- source/dnode/vnode/src/meta/metaTable.c | 88 ++++-- source/dnode/vnode/src/tq/tq.c | 5 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 51 ++-- source/dnode/vnode/src/tsdb/tsdbCache.c | 1 + source/dnode/vnode/src/tsdb/tsdbRead2.c | 92 +++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 1 + source/libs/catalog/src/ctgDbg.c | 1 + source/libs/executor/src/mergejoin.c | 11 +- source/libs/executor/src/mergeoperator.c | 2 +- source/libs/executor/src/scanoperator.c | 6 +- source/libs/parser/src/parInsertStmt.c | 53 ++-- source/libs/parser/src/parInsertUtil.c | 79 +++--- source/libs/parser/src/parTranslater.c | 8 +- source/libs/qworker/src/qworker.c | 52 ++-- source/libs/stream/inc/streamInt.h | 8 +- source/libs/stream/src/streamBackendRocksdb.c | 39 +-- source/libs/stream/src/streamCheckStatus.c | 18 +- source/libs/stream/src/streamCheckpoint.c | 75 ++--- source/libs/stream/src/streamDispatch.c | 59 ++-- source/libs/stream/src/streamMeta.c | 25 +- source/libs/stream/src/streamSched.c | 5 +- source/libs/stream/src/streamStartHistory.c | 22 +- source/libs/stream/src/streamTask.c | 37 ++- source/libs/stream/src/streamTaskSm.c | 4 +- source/libs/stream/src/streammsg.c | 7 +- source/libs/tdb/src/db/tdbBtree.c | 20 +- source/libs/transport/src/transCli.c | 29 +- source/libs/transport/src/transComm.c | 6 +- source/libs/transport/src/transSvr.c | 1 + source/util/src/tdecompress.c | 160 ++++++----- 50 files changed, 1069 insertions(+), 950 deletions(-) diff --git a/include/libs/stream/streammsg.h b/include/libs/stream/streammsg.h index 723c9fd099..5f6f93a4ff 100644 --- a/include/libs/stream/streammsg.h +++ b/include/libs/stream/streammsg.h @@ -182,12 +182,12 @@ typedef struct SRetrieveChkptTriggerReq { } SRetrieveChkptTriggerReq; typedef struct SCheckpointTriggerRsp { - int64_t streamId; - int64_t checkpointId; - int32_t upstreamTaskId; - int32_t taskId; - int32_t transId; - int32_t rspCode; + int64_t streamId; + int64_t checkpointId; + int32_t upstreamTaskId; + int32_t taskId; + int32_t transId; + int32_t rspCode; } SCheckpointTriggerRsp; typedef struct SCheckpointReport { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index bf223e8c28..d35b17126d 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -18,13 +18,13 @@ #include "os.h" #include "streamState.h" +#include "streammsg.h" #include "tdatablock.h" #include "tdbInt.h" #include "tmsg.h" #include "tmsgcb.h" #include "tqueue.h" #include "ttimer.h" -#include "streammsg.h" #ifdef __cplusplus extern "C" { @@ -265,14 +265,14 @@ typedef struct SStreamTaskId { } SStreamTaskId; typedef struct SCheckpointInfo { - int64_t startTs; - int64_t checkpointId; // latest checkpoint id - int64_t checkpointVer; // latest checkpoint offset in wal - int64_t checkpointTime; // latest checkpoint time - int64_t processedVer; - int64_t nextProcessVer; // current offset in WAL, not serialize it + int64_t startTs; + int64_t checkpointId; // latest checkpoint id + int64_t checkpointVer; // latest checkpoint offset in wal + int64_t checkpointTime; // latest checkpoint time + int64_t processedVer; + int64_t nextProcessVer; // current offset in WAL, not serialize it SActiveCheckpointInfo* pActiveInfo; - int64_t msgVer; + int64_t msgVer; } SCheckpointInfo; typedef struct SStreamStatus { @@ -301,7 +301,7 @@ typedef struct SSTaskBasicInfo { int32_t selfChildId; int32_t totalLevel; int8_t taskLevel; - int8_t fillHistory; // is fill history task or not + int8_t fillHistory; // is fill history task or not int64_t delaySchedParam; // in msec } SSTaskBasicInfo; @@ -323,7 +323,7 @@ typedef struct SDispatchMsgInfo { void* pRetryTmr; // used to dispatch data after a given time duration TdThreadMutex lock; int8_t inMonitor; - SArray* pSendInfo; // SArray + SArray* pSendInfo; // SArray } SDispatchMsgInfo; typedef struct STaskQueue { @@ -345,11 +345,11 @@ typedef struct SSinkRecorder { } SSinkRecorder; typedef struct STaskExecStatisInfo { - int64_t created; - int64_t checkTs; - int64_t readyTs; - int64_t startCheckpointId; - int64_t startCheckpointVer; + int64_t created; + int64_t checkTs; + int64_t readyTs; + int64_t startCheckpointId; + int64_t startCheckpointVer; int64_t step1Start; double step1El; @@ -432,25 +432,25 @@ struct SStreamTask { SCheckpointInfo chkInfo; STaskExec exec; SDataRange dataRange; - SVersionRange step2Range; // version range used to scan wal, information in dataRange should not modified. + SVersionRange step2Range; // version range used to scan wal, information in dataRange should not modified. SHistoryTaskInfo hTaskInfo; STaskId streamTaskId; STaskExecStatisInfo execInfo; - TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ - SMsgCb* pMsgCb; // msg handle - SStreamState* pState; // state backend + TdThreadMutex lock; // secure the operation of set task status and puting data into inputQ + SMsgCb* pMsgCb; // msg handle + SStreamState* pState; // state backend SUpstreamInfo upstreamInfo; STaskCheckInfo taskCheckInfo; // the followings attributes don't be serialized SScanhistorySchedInfo schedHistoryInfo; - int32_t refCnt; - int32_t transferStateAlignCnt; - struct SStreamMeta* pMeta; - SSHashObj* pNameMap; - void* pBackend; - int8_t subtableWithoutMd5; - char reserve[256]; + int32_t refCnt; + int32_t transferStateAlignCnt; + struct SStreamMeta* pMeta; + SSHashObj* pNameMap; + void* pBackend; + int8_t subtableWithoutMd5; + char reserve[256]; }; typedef int32_t (*startComplete_fn_t)(struct SStreamMeta*); @@ -459,7 +459,7 @@ typedef struct STaskStartInfo { int64_t startTs; int64_t readyTs; int32_t tasksWillRestart; - int32_t startAllTasks; // restart flag, sentinel to guard the restart procedure. + int32_t startAllTasks; // restart flag, sentinel to guard the restart procedure. SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed int64_t elapsedTime; @@ -512,7 +512,7 @@ typedef struct SStreamMeta { SArray* chkpSaved; SArray* chkpInUse; SRWLatch chkpDirLock; - void* qHandle; // todo remove it + void* qHandle; // todo remove it void* bkdChkptMgt; } SStreamMeta; @@ -567,14 +567,14 @@ typedef struct { } SStreamScanHistoryReq; typedef struct STaskCkptInfo { - int64_t latestId; // saved checkpoint id - int64_t latestVer; // saved checkpoint ver - int64_t latestTime; // latest checkpoint time - int64_t latestSize; // latest checkpoint size - int8_t remoteBackup; // latest checkpoint backup done - int64_t activeId; // current active checkpoint id - int32_t activeTransId; // checkpoint trans id - int8_t failed; // denote if the checkpoint is failed or not + int64_t latestId; // saved checkpoint id + int64_t latestVer; // saved checkpoint ver + int64_t latestTime; // latest checkpoint time + int64_t latestSize; // latest checkpoint size + int8_t remoteBackup; // latest checkpoint backup done + int64_t activeId; // current active checkpoint id + int32_t activeTransId; // checkpoint trans id + int8_t failed; // denote if the checkpoint is failed or not } STaskCkptInfo; typedef struct STaskStatusEntry { @@ -589,12 +589,12 @@ typedef struct STaskStatusEntry { int64_t inputQUnchangeCounter; double inputQUsed; // in MiB double inputRate; - double procsThroughput; // duration between one element put into input queue and being processed. - double procsTotal; // duration between one element put into input queue and being processed. - double outputThroughput; // the size of dispatched result blocks in bytes - double outputTotal; // the size of dispatched result blocks in bytes - double sinkQuota; // existed quota size for sink task - double sinkDataSize; // sink to dst data size + double procsThroughput; // duration between one element put into input queue and being processed. + double procsTotal; // duration between one element put into input queue and being processed. + double outputThroughput; // the size of dispatched result blocks in bytes + double outputTotal; // the size of dispatched result blocks in bytes + double sinkQuota; // existed quota size for sink task + double sinkDataSize; // sink to dst data size int64_t startTime; int64_t startCheckpointId; int64_t startCheckpointVer; @@ -622,12 +622,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i SStreamUpstreamEpInfo* streamTaskGetUpstreamTaskEpInfo(SStreamTask* pTask, int32_t taskId); SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId); -void streamTaskInputFail(SStreamTask* pTask); +void streamTaskInputFail(SStreamTask* pTask); -bool streamTaskShouldStop(const SStreamTask* pStatus); -bool streamTaskShouldPause(const SStreamTask* pStatus); -bool streamTaskIsIdle(const SStreamTask* pTask); -bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); +bool streamTaskShouldStop(const SStreamTask* pStatus); +bool streamTaskShouldPause(const SStreamTask* pStatus); +bool streamTaskIsIdle(const SStreamTask* pTask); +bool streamTaskReadyToRun(const SStreamTask* pTask, char** pStatus); char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); SStreamTaskState* streamTaskGetStatus(const SStreamTask* pTask); @@ -636,8 +636,8 @@ void streamTaskResetStatus(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask); ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask); -bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); -void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); +bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); +void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); // stream task sched bool streamTaskIsAllUpstreamClosed(SStreamTask* pTask); @@ -737,10 +737,10 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int32_t streamMetaResetTaskStatus(SStreamMeta* pMeta); int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs); -void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, - int64_t startTs); -void streamMetaClearUpdateTaskList(SStreamMeta* pMeta); -void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId); +void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, + int64_t startTs); +void streamMetaClearUpdateTaskList(SStreamMeta* pMeta); +void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId); void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta); @@ -756,12 +756,13 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas bool streamMetaAllTasksReady(const SStreamMeta* pMeta); // timer -tmr_h streamTimerGetInstance(); +tmr_h streamTimerGetInstance(); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp); -int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, int32_t downstreamTaskId); +int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, + int32_t downstreamTaskId); int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId); int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); void streamTaskClearCheckInfo(SStreamTask* pTask, bool clearChkpReadyMsg); @@ -786,8 +787,8 @@ int32_t streamTaskRestoreStatus(SStreamTask* pTask); // stream task retrieve related API int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq); -int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq *req); -void streamTaskSendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp); +int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* req); +void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp); #ifdef __cplusplus } diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 9108cae14a..56f89ffba6 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -195,6 +195,7 @@ int32_t fetchWhiteListCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pEpSet); taosMemoryFree(pInfo); tFreeSGetUserWhiteListRsp(&wlRsp); + return TSDB_CODE_OUT_OF_MEMORY; } for (int i = 0; i < wlRsp.numWhiteLists; ++i) { diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 0b907b8661..c8307e4a13 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -17,11 +17,11 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void** *pBuf = pTblBuf->pCurBuff; pTblBuf->buffOffset = pTblBuf->buffUnit; } else { - void *buff = taosMemoryMalloc(pTblBuf->buffSize); + void* buff = taosMemoryMalloc(pTblBuf->buffSize); if (NULL == buff) { return TSDB_CODE_OUT_OF_MEMORY; } - + taosArrayPush(pTblBuf->pBufList, &buff); pTblBuf->buffIdx++; @@ -33,18 +33,18 @@ static FORCE_INLINE int32_t stmtAllocQNodeFromBuf(STableBufInfo* pTblBuf, void** return TSDB_CODE_SUCCESS; } -bool stmtDequeue(STscStmt* pStmt, SStmtQNode **param) { +bool stmtDequeue(STscStmt* pStmt, SStmtQNode** param) { while (0 == atomic_load_64(&pStmt->queue.qRemainNum)) { taosUsleep(1); return false; } - SStmtQNode *orig = pStmt->queue.head; + SStmtQNode* orig = pStmt->queue.head; - SStmtQNode *node = pStmt->queue.head->next; + SStmtQNode* node = pStmt->queue.head->next; pStmt->queue.head = pStmt->queue.head->next; - //taosMemoryFreeClear(orig); + // taosMemoryFreeClear(orig); *param = node; @@ -61,7 +61,6 @@ void stmtEnqueue(STscStmt* pStmt, SStmtQNode* param) { atomic_add_fetch_64(&pStmt->queue.qRemainNum, 1); } - static int32_t stmtCreateRequest(STscStmt* pStmt) { int32_t code = 0; @@ -93,7 +92,7 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { switch (newStatus) { case STMT_PREPARE: - pStmt->errCode = 0; + pStmt->errCode = 0; break; case STMT_SETTBNAME: if (STMT_STATUS_EQ(INIT)) { @@ -265,7 +264,7 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa *pVgHash = pStmt->sql.pVgHash; pStmt->sql.pVgHash = NULL; - + *pBlockHash = pStmt->exec.pBlockHash; pStmt->exec.pBlockHash = NULL; @@ -325,7 +324,7 @@ int32_t stmtParseSql(STscStmt* pStmt) { pStmt->stat.parseSqlNum++; STMT_ERR_RET(parseSql(pStmt->exec.pRequest, false, &pStmt->sql.pQuery, &stmtCb)); pStmt->sql.siInfo.pQuery = pStmt->sql.pQuery; - + pStmt->bInfo.needParse = false; if (pStmt->sql.pQuery->pRoot && 0 == pStmt->sql.type) { @@ -338,15 +337,16 @@ int32_t stmtParseSql(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - STableDataCxt** pSrc = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + STableDataCxt** pSrc = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pSrc || NULL == *pSrc) { return TSDB_CODE_OUT_OF_MEMORY; } STableDataCxt* pTableCtx = *pSrc; if (pStmt->sql.stbInterlaceMode) { - int16_t lastIdx = -1; - + int16_t lastIdx = -1; + for (int32_t i = 0; i < pTableCtx->boundColsInfo.numOfBound; ++i) { if (pTableCtx->boundColsInfo.pColIndex[i] < lastIdx) { pStmt->sql.stbInterlaceMode = false; @@ -381,7 +381,8 @@ int32_t stmtCleanBindInfo(STscStmt* pStmt) { qDestroyBoundColInfo(pStmt->bInfo.boundTags); taosMemoryFreeClear(pStmt->bInfo.boundTags); } - pStmt->bInfo.stbFName[0] = 0;; + pStmt->bInfo.stbFName[0] = 0; + ; return TSDB_CODE_SUCCESS; } @@ -457,12 +458,12 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { return TSDB_CODE_SUCCESS; } -void stmtFreeTbBuf(void *buf) { +void stmtFreeTbBuf(void* buf) { void* pBuf = *(void**)buf; taosMemoryFree(pBuf); } -void stmtFreeTbCols(void *buf) { +void stmtFreeTbCols(void* buf) { SArray* pCols = *(SArray**)buf; taosArrayDestroy(pCols); } @@ -505,19 +506,17 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { memset(&pStmt->sql, 0, sizeof(pStmt->sql)); pStmt->sql.siInfo.tableColsReady = true; - STMT_DLOG_E("end to free SQL info"); return TSDB_CODE_SUCCESS; } - int32_t stmtTryAddTableVgroupInfo(STscStmt* pStmt, int32_t* vgId) { if (*vgId >= 0 && taosHashGet(pStmt->sql.pVgHash, (const char*)vgId, sizeof(*vgId))) { return TSDB_CODE_SUCCESS; } - + SVgroupInfo vgInfo = {0}; SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, .requestId = pStmt->exec.pRequest->requestId, @@ -528,17 +527,17 @@ int32_t stmtTryAddTableVgroupInfo(STscStmt* pStmt, int32_t* vgId) { if (TSDB_CODE_SUCCESS != code) { return code; } - - code = taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo)); + + code = + taosHashPut(pStmt->sql.pVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo)); if (TSDB_CODE_SUCCESS != code) { return code; } *vgId = vgInfo.vgId; - - return TSDB_CODE_SUCCESS; -} + return TSDB_CODE_SUCCESS; +} int32_t stmtRebuildDataBlock(STscStmt* pStmt, STableDataCxt* pDataBlock, STableDataCxt** newBlock, uint64_t uid, uint64_t suid, int32_t vgId) { @@ -554,7 +553,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { if (pStmt->sql.stbInterlaceMode && pStmt->sql.siInfo.pDataCtx) { pStmt->bInfo.needParse = false; pStmt->bInfo.inExecCache = false; - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } pStmt->bInfo.needParse = true; @@ -589,7 +588,6 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - if (pStmt->sql.autoCreateTbl) { SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &pStmt->bInfo.tbSuid, sizeof(pStmt->bInfo.tbSuid)); if (pCache) { @@ -615,8 +613,8 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { } uint64_t uid, suid; - int32_t vgId; - int8_t tableType; + int32_t vgId; + int8_t tableType; STableMeta* pTableMeta = NULL; SRequestConnInfo conn = {.pTrans = pStmt->taos->pAppInfo->pTransporter, @@ -638,12 +636,12 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { uid = pTableMeta->uid; suid = pTableMeta->suid; - tableType = pTableMeta->tableType; + tableType = pTableMeta->tableType; pStmt->bInfo.tbVgId = pTableMeta->vgId; vgId = pTableMeta->vgId; - + taosMemoryFree(pTableMeta); - + uint64_t cacheUid = (TSDB_CHILD_TABLE == tableType) ? suid : uid; if (uid == pStmt->bInfo.tbUid) { @@ -675,7 +673,7 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - + SStmtTableCache* pCache = taosHashGet(pStmt->sql.pTableCache, &cacheUid, sizeof(cacheUid)); if (pCache) { pStmt->bInfo.needParse = false; @@ -720,7 +718,6 @@ int32_t stmtResetStmt(STscStmt* pStmt) { return TSDB_CODE_SUCCESS; } - int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) { SStmtQNode* pParam = (SStmtQNode*)param; @@ -729,20 +726,20 @@ int32_t stmtAsyncOutput(STscStmt* pStmt, void* param) { SArray** p = (SArray**)TARRAY_GET_ELEM(pStmt->sql.siInfo.pTableCols, i); *p = taosArrayInit(20, POINTER_BYTES); } - + atomic_store_8((int8_t*)&pStmt->sql.siInfo.tableColsReady, true); } else { - STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock, &pStmt->sql.siInfo)); + STMT_ERR_RET(qAppendStmtTableOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, &pParam->tblData, pStmt->exec.pCurrBlock, + &pStmt->sql.siInfo)); - //taosMemoryFree(pParam->pTbData); + // taosMemoryFree(pParam->pTbData); atomic_sub_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1); } return TSDB_CODE_SUCCESS; } - -void *stmtBindThreadFunc(void *param) { +void* stmtBindThreadFunc(void* param) { setThreadName("stmtBind"); qInfo("stmt bind thread started"); @@ -750,15 +747,15 @@ void *stmtBindThreadFunc(void *param) { STscStmt* pStmt = (STscStmt*)param; while (true) { - if (atomic_load_8((int8_t *)&pStmt->queue.stopQueue)) { + if (atomic_load_8((int8_t*)&pStmt->queue.stopQueue)) { break; } - SStmtQNode *asyncParam = NULL; + SStmtQNode* asyncParam = NULL; if (!stmtDequeue(pStmt, &asyncParam)) { continue; } - + stmtAsyncOutput(pStmt, asyncParam); } @@ -767,7 +764,6 @@ void *stmtBindThreadFunc(void *param) { return NULL; } - int32_t stmtStartBindThread(STscStmt* pStmt) { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -798,7 +794,7 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { if (NULL == pTblBuf->pBufList) { return TSDB_CODE_OUT_OF_MEMORY; } - void *buff = taosMemoryMalloc(pTblBuf->buffSize); + void* buff = taosMemoryMalloc(pTblBuf->buffSize); if (NULL == buff) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -815,7 +811,7 @@ int32_t stmtInitTableBuf(STableBufInfo* pTblBuf) { TAOS_STMT* stmtInit(STscObj* taos, int64_t reqid, TAOS_STMT_OPTIONS* pOptions) { STscObj* pObj = (STscObj*)taos; STscStmt* pStmt = NULL; - int32_t code = 0; + int32_t code = 0; pStmt = taosMemoryCalloc(1, sizeof(STscStmt)); if (NULL == pStmt) { @@ -909,22 +905,21 @@ int32_t stmtInitStbInterlaceTableInfo(STscStmt* pStmt) { return TSDB_CODE_OUT_OF_MEMORY; } STableDataCxt* pDst = NULL; - + STMT_ERR_RET(qCloneStmtDataBlock(&pDst, *pSrc, true)); pStmt->sql.siInfo.pDataCtx = pDst; - + SArray* pTblCols = NULL; for (int32_t i = 0; i < STMT_TABLE_COLS_NUM; i++) { pTblCols = taosArrayInit(20, POINTER_BYTES); if (NULL == pTblCols) { return TSDB_CODE_OUT_OF_MEMORY; } - + taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols); } - - pStmt->sql.siInfo.boundTags = pStmt->bInfo.boundTags; + pStmt->sql.siInfo.boundTags = pStmt->bInfo.boundTags; return TSDB_CODE_SUCCESS; } @@ -951,13 +946,13 @@ int stmtSetTbName(TAOS_STMT* stmt, const char* tbName) { STMT_ERR_RET(qCreateSName(&pStmt->bInfo.sname, tbName, pStmt->taos->acctId, pStmt->exec.pRequest->pDb, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen)); tNameExtractFullName(&pStmt->bInfo.sname, pStmt->bInfo.tbFName); - + STMT_ERR_RET(stmtGetFromCache(pStmt)); if (pStmt->bInfo.needParse) { strncpy(pStmt->bInfo.tbName, tbName, sizeof(pStmt->bInfo.tbName) - 1); pStmt->bInfo.tbName[sizeof(pStmt->bInfo.tbName) - 1] = 0; - + STMT_ERR_RET(stmtParseSql(pStmt)); } } else { @@ -1032,7 +1027,8 @@ int stmtFetchColFields(STscStmt* pStmt, int32_t* fieldNum, TAOS_FIELD_E** fields if (pStmt->sql.stbInterlaceMode) { pDataBlock = &pStmt->sql.siInfo.pDataCtx; } else { - pDataBlock = (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); + pDataBlock = + (STableDataCxt**)taosHashGet(pStmt->exec.pBlockHash, pStmt->bInfo.tbFName, strlen(pStmt->bInfo.tbFName)); if (NULL == pDataBlock) { tscError("table %s not found in exec blockHash", pStmt->bInfo.tbFName); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -1050,7 +1046,7 @@ SArray* stmtGetFreeCol(STscStmt* pStmt, int32_t* idx) { if (pStmt->exec.smInfo.pColIdx >= STMT_COL_BUF_SIZE) { pStmt->exec.smInfo.pColIdx = 0; } - + if ((pStmt->exec.smInfo.pColIdx + 1) == atomic_load_32(&pStmt->exec.smInfo.pColFreeIdx)) { taosUsleep(1); continue; @@ -1064,38 +1060,40 @@ SArray* stmtGetFreeCol(STscStmt* pStmt, int32_t* idx) { int32_t stmtAppendTablePostHandle(STscStmt* pStmt, SStmtQNode* param) { if (NULL == pStmt->sql.siInfo.pVgroupHash) { - pStmt->sql.siInfo.pVgroupHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + pStmt->sql.siInfo.pVgroupHash = + taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); } if (NULL == pStmt->sql.siInfo.pVgroupList) { pStmt->sql.siInfo.pVgroupList = taosArrayInit(64, POINTER_BYTES); } - + if (NULL == pStmt->sql.siInfo.pRequest) { - STMT_ERR_RET(buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, (SRequestObj**)&pStmt->sql.siInfo.pRequest, - pStmt->reqid)); - + STMT_ERR_RET(buildRequest(pStmt->taos->id, pStmt->sql.sqlStr, pStmt->sql.sqlLen, NULL, false, + (SRequestObj**)&pStmt->sql.siInfo.pRequest, pStmt->reqid)); + if (pStmt->reqid != 0) { pStmt->reqid++; } pStmt->exec.pRequest->syncQuery = true; - + pStmt->sql.siInfo.requestId = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->requestId; pStmt->sql.siInfo.requestSelf = ((SRequestObj*)pStmt->sql.siInfo.pRequest)->self; } - - if (!pStmt->sql.siInfo.tbFromHash && pStmt->sql.siInfo.firstName[0] && 0 == strcmp(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName)) { + + if (!pStmt->sql.siInfo.tbFromHash && pStmt->sql.siInfo.firstName[0] && + 0 == strcmp(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName)) { pStmt->sql.siInfo.tbFromHash = true; } - + if (0 == pStmt->sql.siInfo.firstName[0]) { strcpy(pStmt->sql.siInfo.firstName, pStmt->bInfo.tbName); } - + param->tblData.getFromHash = pStmt->sql.siInfo.tbFromHash; param->next = NULL; - + atomic_add_fetch_64(&pStmt->sql.siInfo.tbRemainNum, 1); - + stmtEnqueue(pStmt, param); return TSDB_CODE_SUCCESS; @@ -1113,7 +1111,7 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray** if (NULL == pTblCols) { return TSDB_CODE_OUT_OF_MEMORY; } - + taosArrayPush(pStmt->sql.siInfo.pTableCols, &pTblCols); } } @@ -1124,10 +1122,10 @@ static FORCE_INLINE int32_t stmtGetTableColsFromCache(STscStmt* pStmt, SArray** int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STscStmt* pStmt = (STscStmt*)stmt; - int32_t code = 0; + int32_t code = 0; int64_t startUs = taosGetTimestampUs(); - + STMT_DLOG("start to bind stmt data, colIdx: %d", colIdx); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_BIND)); @@ -1210,7 +1208,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } int64_t startUs2 = taosGetTimestampUs(); - pStmt->stat.bindDataUs1 += startUs2 - startUs; + pStmt->stat.bindDataUs1 += startUs2 - startUs; SStmtQNode* param = NULL; if (pStmt->sql.stbInterlaceMode) { @@ -1218,25 +1216,27 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { STMT_ERR_RET(stmtGetTableColsFromCache(pStmt, ¶m->tblData.aCol)); taosArrayClear(param->tblData.aCol); - //param->tblData.aCol = taosArrayInit(20, POINTER_BYTES); + // param->tblData.aCol = taosArrayInit(20, POINTER_BYTES); param->restoreTbCols = false; strcpy(param->tblData.tbName, pStmt->bInfo.tbName); } int64_t startUs3 = taosGetTimestampUs(); - pStmt->stat.bindDataUs2 += startUs3 - startUs2; + pStmt->stat.bindDataUs2 += startUs3 - startUs2; SArray* pCols = pStmt->sql.stbInterlaceMode ? param->tblData.aCol : (*pDataBlock)->pData->aCol; if (colIdx < 0) { if (pStmt->sql.stbInterlaceMode) { (*pDataBlock)->pData->flags = 0; - code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); + code = qBindStmtStbColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, + pStmt->exec.pRequest->msgBufLen, &pStmt->sql.siInfo.pTSchema, pStmt->sql.pBindInfo); } else { - code = qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); + code = + qBindStmtColsValue(*pDataBlock, pCols, bind, pStmt->exec.pRequest->msgBuf, pStmt->exec.pRequest->msgBufLen); } - + if (code) { tscError("qBindStmtColsValue failed, error:%s", tstrerror(code)); STMT_ERR_RET(code); @@ -1246,7 +1246,7 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { tscError("bind single column not allowed in stb insert mode"); STMT_ERR_RET(TSDB_CODE_TSC_STMT_API_ERROR); } - + if (colIdx != (pStmt->bInfo.sBindLastIdx + 1) && colIdx != 0) { tscError("bind column index not in sequence"); STMT_ERR_RET(TSDB_CODE_APP_ERROR); @@ -1267,13 +1267,13 @@ int stmtBindBatch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int32_t colIdx) { } int64_t startUs4 = taosGetTimestampUs(); - pStmt->stat.bindDataUs3 += startUs4 - startUs3; + pStmt->stat.bindDataUs3 += startUs4 - startUs3; if (pStmt->sql.stbInterlaceMode) { STMT_ERR_RET(stmtAppendTablePostHandle(pStmt, param)); } - pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4; + pStmt->stat.bindDataUs4 += taosGetTimestampUs() - startUs4; return TSDB_CODE_SUCCESS; } @@ -1292,12 +1292,12 @@ int stmtAddBatch(TAOS_STMT* stmt) { pStmt->stat.addBatchUs += startUs2 - startUs; pStmt->sql.siInfo.tableColsReady = false; - + SStmtQNode* param = NULL; STMT_ERR_RET(stmtAllocQNodeFromBuf(&pStmt->sql.siInfo.tbBuf, (void**)¶m)); param->restoreTbCols = true; param->next = NULL; - + stmtEnqueue(pStmt, param); return TSDB_CODE_SUCCESS; @@ -1372,7 +1372,7 @@ int stmtUpdateTableUid(STscStmt* pStmt, SSubmitRsp* pRsp) { int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta); pStmt->stat.ctgGetTbMetaNum++; - + taos_free_result(pStmt->exec.pRequest); pStmt->exec.pRequest = NULL; @@ -1402,13 +1402,14 @@ int stmtStaticModeExec(TAOS_STMT* stmt) { if (pStmt->sql.staticMode) { return TSDB_CODE_TSC_STMT_API_ERROR; } - + STMT_DLOG_E("start to exec"); STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_EXECUTE)); - STMT_ERR_RET(qBuildStmtOutputFromTbList(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pTbBlkList, pStmt->exec.pCurrBlock, pStmt->exec.tbBlkNum)); - + STMT_ERR_RET(qBuildStmtOutputFromTbList(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pTbBlkList, +pStmt->exec.pCurrBlock, pStmt->exec.tbBlkNum)); + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); if (pStmt->exec.pRequest->code && NEED_CLIENT_HANDLE_ERROR(pStmt->exec.pRequest->code)) { @@ -1430,7 +1431,7 @@ int stmtStaticModeExec(TAOS_STMT* stmt) { _return: stmtCleanExecInfo(pStmt, (code ? false : true), false); - + tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; @@ -1459,7 +1460,7 @@ int stmtExec(TAOS_STMT* stmt) { taosUsleep(1); } pStmt->stat.execWaitUs += taosGetTimestampUs() - startTs; - + STMT_ERR_RET(qBuildStmtFinOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->sql.siInfo.pVgroupList)); taosHashCleanup(pStmt->sql.siInfo.pVgroupHash); pStmt->sql.siInfo.pVgroupHash = NULL; @@ -1472,7 +1473,7 @@ int stmtExec(TAOS_STMT* stmt) { STMT_ERR_RET(qBuildStmtOutput(pStmt->sql.pQuery, pStmt->sql.pVgHash, pStmt->exec.pBlockHash)); } - + launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } @@ -1499,7 +1500,7 @@ _return: } stmtCleanExecInfo(pStmt, (code ? false : true), false); - + tFreeSSubmitRsp(pRsp); ++pStmt->sql.runTimes; @@ -1522,13 +1523,16 @@ int stmtClose(TAOS_STMT* stmt) { pStmt->bindThreadInUse = false; } - STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64 ", parseSqlNum=>%" PRId64 - ", pStmt->stat.bindDataNum=>%" PRId64 ", settbnameAPI:%u, bindAPI:%u, addbatchAPI:%u, execAPI:%u" - ", setTbNameUs:%" PRId64 ", bindDataUs:%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 " addBatchUs:%" PRId64 ", execWaitUs:%" PRId64 ", execUseUs:%" PRId64, - pStmt, pStmt->sql.stbInterlaceMode, pStmt->stat.ctgGetTbMetaNum, pStmt->stat.getCacheTbInfo, pStmt->stat.parseSqlNum, pStmt->stat.bindDataNum, - pStmt->seqIds[STMT_SETTBNAME], pStmt->seqIds[STMT_BIND], pStmt->seqIds[STMT_ADD_BATCH], pStmt->seqIds[STMT_EXECUTE], - pStmt->stat.setTbNameUs, pStmt->stat.bindDataUs1, pStmt->stat.bindDataUs2, pStmt->stat.bindDataUs3, pStmt->stat.bindDataUs4, - pStmt->stat.addBatchUs, pStmt->stat.execWaitUs, pStmt->stat.execUseUs); + STMT_DLOG("stmt %p closed, stbInterlaceMode: %d, statInfo: ctgGetTbMetaNum=>%" PRId64 ", getCacheTbInfo=>%" PRId64 + ", parseSqlNum=>%" PRId64 ", pStmt->stat.bindDataNum=>%" PRId64 + ", settbnameAPI:%u, bindAPI:%u, addbatchAPI:%u, execAPI:%u" + ", setTbNameUs:%" PRId64 ", bindDataUs:%" PRId64 ",%" PRId64 ",%" PRId64 ",%" PRId64 " addBatchUs:%" PRId64 + ", execWaitUs:%" PRId64 ", execUseUs:%" PRId64, + pStmt, pStmt->sql.stbInterlaceMode, pStmt->stat.ctgGetTbMetaNum, pStmt->stat.getCacheTbInfo, + pStmt->stat.parseSqlNum, pStmt->stat.bindDataNum, pStmt->seqIds[STMT_SETTBNAME], pStmt->seqIds[STMT_BIND], + pStmt->seqIds[STMT_ADD_BATCH], pStmt->seqIds[STMT_EXECUTE], pStmt->stat.setTbNameUs, + pStmt->stat.bindDataUs1, pStmt->stat.bindDataUs2, pStmt->stat.bindDataUs3, pStmt->stat.bindDataUs4, + pStmt->stat.addBatchUs, pStmt->stat.execWaitUs, pStmt->stat.execUseUs); stmtCleanSQLInfo(pStmt); taosMemoryFree(stmt); @@ -1567,9 +1571,9 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) { } int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { - int32_t code = 0; + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; - int32_t preCode = pStmt->errCode; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get tag fields"); @@ -1600,14 +1604,14 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { _return: pStmt->errCode = preCode; - + return code; } int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { - int32_t code = 0; + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; - int32_t preCode = pStmt->errCode; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get col fields"); @@ -1661,7 +1665,7 @@ int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { } STMT_ERR_RET(stmtCreateRequest(pStmt)); - + if (pStmt->bInfo.needParse) { STMT_ERR_RET(stmtParseSql(pStmt)); } diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index f5f083e5d8..7023754a26 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -107,7 +107,7 @@ struct tmq_t { STaosQueue* mqueue; // queue of rsp STaosQall* qall; STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit - tsem2_t rspSem; + tsem2_t rspSem; }; typedef struct SAskEpInfo { @@ -681,7 +681,7 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); tscDebug("consumer:0x%" PRIx64 " commit offset for topics:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, - numOfVgroups); + numOfVgroups); for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); @@ -703,19 +703,19 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us } tscDebug("consumer:0x%" PRIx64 - " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); + " topic:%s on vgId:%d send commit msg success, send offset:%s committed:%s, ordinal:%d/%d", + tmq->consumerId, pTopic->topicName, pVg->vgId, offsetBuf, commitBuf, j + 1, numOfVgroups); tOffsetCopy(&pVg->offsetInfo.committedOffset, &pVg->offsetInfo.endOffset); } else { tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.endOffset.version, j + 1, numOfVgroups); } } } taosRUnLockLatch(&tmq->lock); tscDebug("consumer:0x%" PRIx64 " total commit:%d for %d topics", tmq->consumerId, pParamSet->waitingRspNum - 1, - numOfTopics); + numOfTopics); // request is sent if (pParamSet->waitingRspNum != 1) { @@ -830,7 +830,7 @@ void tmqSendHbReq(void* param, void* tmrId) { char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &offRows->offset); tscDebug("consumer:0x%" PRIx64 ",report offset, group:%s vgId:%d, offset:%s/%" PRId64 ", rows:%" PRId64, - tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows); + tmq->consumerId, tmq->groupId, offRows->vgId, buf, offRows->ever, offRows->rows); } } taosRUnLockLatch(&tmq->lock); @@ -955,7 +955,7 @@ static void tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; taosMemoryFreeClear(pRsp->pEpset); tDeleteMqBatchMetaRsp(&pRsp->batchMetaRsp); - } + } } void tmqClearUnhandleMsg(tmq_t* tmq) { @@ -1021,17 +1021,16 @@ int32_t tmq_unsubscribe(tmq_t* tmq) { taosSsleep(2); // sleep 2s for hb to send offset and rows to server tmq_list_t* lst = tmq_list_new(); - int32_t rsp = tmq_subscribe(tmq, lst); + int32_t rsp = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); return rsp; } -static void freeClientVg(void* param){ +static void freeClientVg(void* param) { SMqClientVg* pVg = param; tOffsetDestroy(&pVg->offsetInfo.endOffset); tOffsetDestroy(&pVg->offsetInfo.beginOffset); tOffsetDestroy(&pVg->offsetInfo.committedOffset); - } static void freeClientVgImpl(void* param) { SMqClientTopic* pTopic = param; @@ -1282,8 +1281,9 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t retryCnt = 0; while ((code = syncAskEp(tmq)) != 0) { if (retryCnt++ > MAX_RETRY_COUNT || code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { - tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", tmq->consumerId, strerror(code)); - if(code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { + tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, retry more than 2 minutes, code:%s", + tmq->consumerId, strerror(code)); + if (code == TSDB_CODE_MND_CONSUMER_NOT_EXIST) { code = 0; } goto FAIL; @@ -1379,7 +1379,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { goto END; } - if(pMsg->pData == NULL){ + if (pMsg->pData == NULL) { tscError("consumer:0x%" PRIx64 " msg discard from vgId:%d, since msg is NULL", tmq->consumerId, vgId); code = TSDB_CODE_TSC_INTERNAL_ERROR; goto END; @@ -1407,7 +1407,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (rspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0){ + if (tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp) < 0) { tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1423,7 +1423,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0){ + if (tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp) < 0) { tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1434,7 +1434,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } else if (rspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0){ + if (tDecodeSTaosxRsp(&decoder, &pRspWrapper->taosxRsp) < 0) { tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1445,7 +1445,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } else if (rspType == TMQ_MSG_TYPE__POLL_BATCH_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - if(tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0){ + if (tSemiDecodeMqBatchMetaRsp(&decoder, &pRspWrapper->batchMetaRsp) < 0) { tDecoderClear(&decoder); taosReleaseRef(tmqMgmt.rsetId, refId); code = TSDB_CODE_OUT_OF_MEMORY; @@ -1529,11 +1529,11 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic clientVg.offsetInfo.walVerBegin = -1; clientVg.offsetInfo.walVerEnd = -1; clientVg.seekUpdated = false; - if(pInfo) { + if (pInfo) { tOffsetCopy(&clientVg.offsetInfo.endOffset, &pInfo->currentOffset); tOffsetCopy(&clientVg.offsetInfo.committedOffset, &pInfo->commitOffset); tOffsetCopy(&clientVg.offsetInfo.beginOffset, &pInfo->seekOffset); - }else{ + } else { clientVg.offsetInfo.endOffset = offsetNew; clientVg.offsetInfo.committedOffset = offsetNew; clientVg.offsetInfo.beginOffset = offsetNew; @@ -1548,7 +1548,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) int32_t topicNumGet = taosArrayGetSize(pRsp->topics); if (epoch < tmq->epoch || (epoch == tmq->epoch && topicNumGet == 0)) { tscDebug("consumer:0x%" PRIx64 " no update ep epoch from %d to epoch %d, incoming topics:%d", tmq->consumerId, - tmq->epoch, epoch, topicNumGet); + tmq->epoch, epoch, topicNumGet); if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__RECOVER) { atomic_store_8(&tmq->status, TMQ_CONSUMER_STATUS__READY); } @@ -1642,7 +1642,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); - if(pRspObj == NULL) { + if (pRspObj == NULL) { return NULL; } pRspObj->resType = RES_TYPE__TMQ_META; @@ -1656,7 +1656,7 @@ SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqBatchMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqBatchMetaRspObj)); - if(pRspObj == NULL) { + if (pRspObj == NULL) { return NULL; } pRspObj->common.resType = RES_TYPE__TMQ_BATCH_META; @@ -1669,10 +1669,11 @@ SMqBatchMetaRspObj* tmqBuildBatchMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) return pRspObj; } -void changeByteEndian(char* pData){ +void changeByteEndian(char* pData) { char* p = pData; - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | version: + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | version: int32_t blockVersion = *(int32_t*)p; ASSERT(blockVersion == BLOCK_VERSION_1); *(int32_t*)p = BLOCK_VERSION_2; @@ -1693,23 +1694,24 @@ void changeByteEndian(char* pData){ } } -static void tmqGetRawDataRowsPrecisionFromRes(void *pRetrieve, void** rawData, int64_t *rows, int32_t *precision){ - if(*(int64_t*)pRetrieve == 0){ +static void tmqGetRawDataRowsPrecisionFromRes(void* pRetrieve, void** rawData, int64_t* rows, int32_t* precision) { + if (*(int64_t*)pRetrieve == 0) { *rawData = ((SRetrieveTableRsp*)pRetrieve)->data; *rows = htobe64(((SRetrieveTableRsp*)pRetrieve)->numOfRows); - if(precision != NULL){ + if (precision != NULL) { *precision = ((SRetrieveTableRsp*)pRetrieve)->precision; } - }else if(*(int64_t*)pRetrieve == 1){ + } else if (*(int64_t*)pRetrieve == 1) { *rawData = ((SRetrieveTableRspForTmq*)pRetrieve)->data; *rows = htobe64(((SRetrieveTableRspForTmq*)pRetrieve)->numOfRows); - if(precision != NULL){ + if (precision != NULL) { *precision = ((SRetrieveTableRspForTmq*)pRetrieve)->precision; } } } -static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) { +static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows, + SMqRspObjCommon* pRspObj, SMqDataRspCommon* pDataRsp) { (*numOfRows) = 0; tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); @@ -1727,8 +1729,8 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg } // extract the rows in this data packet for (int32_t i = 0; i < pDataRsp->blockNum; ++i) { - void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i); - void* rawData = NULL; + void* pRetrieve = taosArrayGetP(pDataRsp->blockData, i); + void* rawData = NULL; int64_t rows = 0; // deal with compatibility tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, NULL); @@ -1736,9 +1738,9 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg pVg->numOfRows += rows; (*numOfRows) += rows; changeByteEndian(rawData); - if (needTransformSchema) { //withSchema is false if subscribe subquery, true if subscribe db or stable - SSchemaWrapper *schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); - if(schema){ + if (needTransformSchema) { // withSchema is false if subscribe subquery, true if subscribe db or stable + SSchemaWrapper* schema = tCloneSSchemaWrapper(&pWrapper->topicHandle->schema); + if (schema) { taosArrayPush(pDataRsp->blockSchema, &schema); } } @@ -1747,7 +1749,7 @@ static void tmqBuildRspFromWrapperInner(SMqPollRspWrapper* pWrapper, SMqClientVg SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); - if(pRspObj == NULL){ + if (pRspObj == NULL) { return NULL; } pRspObj->common.resType = RES_TYPE__TMQ; @@ -1758,7 +1760,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, SMqTaosxRspObj* tmqBuildTaosxRspFromWrapper(SMqPollRspWrapper* pWrapper, SMqClientVg* pVg, int64_t* numOfRows) { SMqTaosxRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqTaosxRspObj)); - if(pRspObj == NULL){ + if (pRspObj == NULL) { return NULL; } pRspObj->common.resType = RES_TYPE__TMQ_METADATA; @@ -1790,6 +1792,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { code = TSDB_CODE_INVALID_MSG; + taosMemoryFreeClear(msg); goto FAIL; } @@ -1858,7 +1861,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; + int64_t elapsed = taosGetTimestampMs() - pVg->emptyBlockReceiveTs; if (elapsed < EMPTY_BLOCK_POLL_IDLE_DURATION && elapsed >= 0) { // less than 10ms tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); @@ -1956,7 +1959,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; - int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + int32_t consumerEpoch = atomic_load_32(&tmq->epoch); SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->dataRsp; if (pDataRsp->head.epoch == consumerEpoch) { @@ -2096,7 +2099,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_DATA_META_RSP) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp; + SMqDataRspCommon* pDataRsp = (SMqDataRspCommon*)&pollRspWrapper->taosxRsp; if (pDataRsp->head.epoch == consumerEpoch) { taosWLockLatch(&tmq->lock); @@ -2112,9 +2115,8 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { return NULL; } - updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, - pDataRsp->head.walsver, pDataRsp->head.walever, tmq->consumerId, - pDataRsp->blockNum != 0); + updateVgInfo(pVg, &pDataRsp->reqOffset, &pDataRsp->rspOffset, pDataRsp->head.walsver, pDataRsp->head.walever, + tmq->consumerId, pDataRsp->blockNum != 0); if (pDataRsp->blockNum == 0) { tscDebug("consumer:0x%" PRIx64 " taosx empty block received, vgId:%d, vg total:%" PRId64 ", reqId:0x%" PRIx64, @@ -2130,15 +2132,15 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout) { // build rsp int64_t numOfRows = 0; - void* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); + void* pRsp = tmqBuildTaosxRspFromWrapper(pollRspWrapper, pVg, &numOfRows); tmq->totalRows += numOfRows; char buf[TSDB_OFFSET_LEN] = {0}; tFormatOffset(buf, TSDB_OFFSET_LEN, &pVg->offsetInfo.endOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 ", total:%" PRId64 ", reqId:0x%" PRIx64, - tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, - tmq->totalRows, pollRspWrapper->reqId); + tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, + pollRspWrapper->reqId); taosFreeQitem(pRspWrapper); taosWUnLockLatch(&tmq->lock); @@ -2266,7 +2268,7 @@ int32_t tmq_consumer_close(tmq_t* tmq) { taosSsleep(2); // sleep 2s for hb to send offset and rows to server tmq_list_t* lst = tmq_list_new(); - int32_t code = tmq_subscribe(tmq, lst); + int32_t code = tmq_subscribe(tmq, lst); tmq_list_destroy(lst); if (code != 0) { return code; @@ -2285,9 +2287,9 @@ const char* tmq_err2str(int32_t err) { } else if (err == -1) { return "fail"; } else { - if(*(taosGetErrMsg()) == 0){ + if (*(taosGetErrMsg()) == 0) { return tstrerror(err); - } else{ + } else { snprintf(taosGetErrMsgReturn(), ERR_MSG_LEN, "%s,detail:%s", tstrerror(err), taosGetErrMsg()); return (const char*)taosGetErrMsgReturn(); } @@ -2359,8 +2361,8 @@ int64_t tmq_get_vgroup_offset(TAOS_RES* res) { return TSDB_CODE_INVALID_PARA; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - STqOffsetVal* pOffset = &common->reqOffset; + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + STqOffsetVal* pOffset = &common->reqOffset; if (common->reqOffset.type == TMQ_OFFSET__LOG) { return common->reqOffset.version; } else { @@ -2389,7 +2391,7 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } if (TD_RES_TMQ(res) || TD_RES_TMQ_METADATA(res)) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; if (!common->withTbName || common->blockTbName == NULL || pRspObj->resIter < 0 || @@ -2734,8 +2736,8 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons } SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { - SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); - SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; + SMqDataRspCommon* common = (SMqDataRspCommon*)POINTER_SHIFT(res, sizeof(SMqRspObjCommon)); + SMqRspObjCommon* pRspObj = (SMqRspObjCommon*)res; pRspObj->resIter++; if (pRspObj->resIter < common->blockNum) { if (common->withSchema) { @@ -2744,8 +2746,8 @@ SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); } - void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); - void* rawData = NULL; + void* pRetrieve = taosArrayGetP(common->blockData, pRspObj->resIter); + void* rawData = NULL; int64_t rows = 0; int32_t precision = 0; tmqGetRawDataRowsPrecisionFromRes(pRetrieve, &rawData, &rows, &precision); diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 6c46f4ff42..fe6e20c47f 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -83,10 +83,10 @@ int32_t s3CheckCfg() { snprintf(path, PATH_MAX, "%s", tsTempDir); if (strncmp(tsTempDir + tmp_len - ds_len, TD_DIRSEP, ds_len) != 0) { - snprintf(path + tmp_len, PATH_MAX, "%s", TD_DIRSEP); - snprintf(path + tmp_len + ds_len, PATH_MAX, "%s", objectname[0]); + snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", TD_DIRSEP); + snprintf(path + tmp_len + ds_len, PATH_MAX - tmp_len - ds_len, "%s", objectname[0]); } else { - snprintf(path + tmp_len, PATH_MAX, "%s", objectname[0]); + snprintf(path + tmp_len, PATH_MAX - tmp_len, "%s", objectname[0]); } TdFilePtr fp = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); @@ -1075,32 +1075,32 @@ static SArray *getListByPrefix(const char *prefix) { S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &listBucketCallback}; - const char *marker = 0, *delimiter = 0; - int maxkeys = 0, allDetails = 0; - list_bucket_callback_data data = {0}; + const char /**marker = 0,*/ *delimiter = 0; + int /*maxkeys = 0, */ allDetails = 0; + list_bucket_callback_data data = {0}; data.objectArray = taosArrayInit(32, sizeof(void *)); if (!data.objectArray) { uError("%s: %s", __func__, "out of memoty"); return NULL; } - if (marker) { + /*if (marker) { snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); - } else { - data.nextMarker[0] = 0; - } + } else {*/ + data.nextMarker[0] = 0; + //} data.keyCount = 0; data.allDetails = allDetails; do { data.isTruncated = 0; do { - S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, - &data); + S3_list_bucket(&bucketContext, prefix, data.nextMarker, delimiter, 0 /*maxkeys*/, 0, timeoutMsG, + &listBucketHandler, &data); } while (S3_status_is_retryable(data.status) && should_retry()); if (data.status != S3StatusOK) { break; } - } while (data.isTruncated && (!maxkeys || (data.keyCount < maxkeys))); + } while (data.isTruncated /* && (!maxkeys || (data.keyCount < maxkeys))*/); if (data.status == S3StatusOK) { if (data.keyCount > 0) { diff --git a/source/common/src/cos_cp.c b/source/common/src/cos_cp.c index 6d37b4d4dc..509f0a3a1c 100644 --- a/source/common/src/cos_cp.c +++ b/source/common/src/cos_cp.c @@ -20,7 +20,7 @@ int32_t cos_cp_open(char const* cp_path, SCheckpoint* checkpoint) { } void cos_cp_close(TdFilePtr fd) { taosCloseFile(&fd); } -void cos_cp_remove(char const* filepath) { taosRemoveFile(filepath); } +void cos_cp_remove(char const* filepath) { (void)taosRemoveFile(filepath); } static int32_t cos_cp_parse_body(char* cp_body, SCheckpoint* cp) { int32_t code = 0; @@ -380,7 +380,7 @@ void cos_cp_get_undo_parts(SCheckpoint* checkpoint, int* part_num, SCheckpointPa void cos_cp_update(SCheckpoint* checkpoint, int32_t part_index, char const* etag, uint64_t crc64) { checkpoint->parts[part_index].completed = 1; - strncpy(checkpoint->parts[part_index].etag, etag, 128); + strncpy(checkpoint->parts[part_index].etag, etag, 127); checkpoint->parts[part_index].crc64 = crc64; } @@ -389,11 +389,12 @@ void cos_cp_build_upload(SCheckpoint* checkpoint, char const* filepath, int64_t int i = 0; checkpoint->cp_type = COS_CP_TYPE_UPLOAD; - strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN); + memset(checkpoint->file_path, 0, TSDB_FILENAME_LEN); + strncpy(checkpoint->file_path, filepath, TSDB_FILENAME_LEN - 1); checkpoint->file_size = size; checkpoint->file_last_modified = mtime; - strncpy(checkpoint->upload_id, upload_id, 128); + strncpy(checkpoint->upload_id, upload_id, 127); checkpoint->part_size = part_size; for (; i * part_size < size; i++) { diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index 7364ca3d6c..ac94625f8e 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -5,8 +5,8 @@ #include #include "tglobal.h" -#define ERRNO_ERR_FORMAT "errno:%d,msg:%s" -#define ERRNO_ERR_DATA errno,strerror(errno) +#define ERRNO_ERR_FORMAT "errno:%d,msg:%s" +#define ERRNO_ERR_DATA errno, strerror(errno) // deleteRsync function produce empty directories, traverse base directory to remove them static void removeEmptyDir() { @@ -24,14 +24,14 @@ static void removeEmptyDir() { char filename[PATH_MAX] = {0}; snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de)); - TdDirPtr pDirTmp = taosOpenDir(filename); + TdDirPtr pDirTmp = taosOpenDir(filename); TdDirEntryPtr deTmp = NULL; - bool empty = true; - while ((deTmp = taosReadDir(pDirTmp)) != NULL){ + bool empty = true; + while ((deTmp = taosReadDir(pDirTmp)) != NULL) { if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue; empty = false; } - if(empty) taosRemoveDir(filename); + if (empty) taosRemoveDir(filename); taosCloseDir(&pDirTmp); } @@ -40,10 +40,10 @@ static void removeEmptyDir() { #ifdef WINDOWS // C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/ -static void changeDirFromWindowsToLinux(char* from, char* to){ +static void changeDirFromWindowsToLinux(char* from, char* to) { to[0] = '/'; to[1] = from[0]; - for(int32_t i = 2; i < strlen(from); i++) { + for (int32_t i = 2; i < strlen(from); i++) { if (from[i] == '\\') { to[i] = '/'; } else { @@ -56,7 +56,7 @@ static void changeDirFromWindowsToLinux(char* from, char* to){ static int32_t generateConfigFile(char* confDir) { TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { - uError("[rsync] open conf file error, dir:%s,"ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); + uError("[rsync] open conf file error, dir:%s," ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); return -1; } @@ -65,8 +65,8 @@ static int32_t generateConfigFile(char* confDir) { changeDirFromWindowsToLinux(tsCheckpointBackupDir, path); #endif - char confContent[PATH_MAX*4] = {0}; - snprintf(confContent, PATH_MAX*4, + char confContent[PATH_MAX * 4] = {0}; + snprintf(confContent, PATH_MAX * 4, #ifndef WINDOWS "uid = root\n" "gid = root\n" @@ -80,16 +80,17 @@ static int32_t generateConfigFile(char* confDir) { "read only = false\n" "list = false\n" "[checkpoint]\n" - "path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir, + "path = %s", + tsCheckpointBackupDir, tsCheckpointBackupDir, #ifdef WINDOWS path #else tsCheckpointBackupDir #endif - ); + ); uDebug("[rsync] conf:%s", confContent); - if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){ - uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0) { + uError("[rsync] write conf file error," ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); taosCloseFile(&pFile); return -1; } @@ -98,10 +99,10 @@ static int32_t generateConfigFile(char* confDir) { return 0; } -static int32_t execCommand(char* command){ +static int32_t execCommand(char* command) { int32_t try = 3; int32_t code = 0; - while(try-- > 0) { + while (try-- > 0) { code = system(command); if (code == 0) { break; @@ -125,7 +126,7 @@ void stopRsync() { uDebug("[rsync] stop rsync server successful"); } - taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation. + taosMsleep(500); // sleep 500 ms to wait for the completion of kill operation. } void startRsync() { @@ -154,7 +155,6 @@ void startRsync() { } else { uDebug("[rsync] start server successful"); } - } int32_t uploadByRsync(const char* id, const char* path) { @@ -165,7 +165,7 @@ int32_t uploadByRsync(const char* id, const char* path) { char pathTransform[PATH_MAX] = {0}; changeDirFromWindowsToLinux(path, pathTransform); - if(pathTransform[strlen(pathTransform) - 1] != '/') { + if (pathTransform[strlen(pathTransform) - 1] != '/') { #else if (path[strlen(path) - 1] != '/') { #endif @@ -197,12 +197,12 @@ int32_t uploadByRsync(const char* id, const char* path) { // prepare the data directory int32_t code = execCommand(command); if (code != 0) { - uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, - tsSnodeAddress, code, ERRNO_ERR_DATA); + uError("[rsync] s-task:%s prepare checkpoint data in %s to %s failed, code:%d," ERRNO_ERR_FORMAT, id, path, + tsSnodeAddress, code, ERRNO_ERR_DATA); } else { - int64_t el = (taosGetTimestampMs() - st); - uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, - tsSnodeAddress, el); + int64_t el = (taosGetTimestampMs() - st); + uDebug("[rsync] s-task:%s prepare checkpoint data in:%s to %s successfully, elapsed time:%" PRId64 "ms", id, path, + tsSnodeAddress, el); } #ifdef WINDOWS @@ -264,19 +264,20 @@ int32_t downloadRsync(const char* id, const char* path) { #endif char command[PATH_MAX] = {0}; - snprintf(command, PATH_MAX, - "rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/data/ %s", - tsLogDir, tsSnodeAddress, id, + snprintf( + command, PATH_MAX, + "rsync -av --debug=all --log-file=%s/rsynclog --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/data/ %s", + tsLogDir, tsSnodeAddress, id, #ifdef WINDOWS - pathTransform + pathTransform #else - path + path #endif ); uDebug("[rsync] %s start to sync data from remote to:%s, %s", id, path, command); - while(times++ < MAX_RETRY) { + while (times++ < MAX_RETRY) { code = execCommand(command); if (code != TSDB_CODE_SUCCESS) { uError("[rsync] %s download checkpoint data:%s failed, retry after 1sec, times:%d, code:%d," ERRNO_ERR_FORMAT, id, @@ -302,8 +303,8 @@ int32_t deleteRsync(const char* id) { char command[PATH_MAX] = {0}; snprintf(command, PATH_MAX, - "rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/data/", tsLogDir, - tmp, tsSnodeAddress, id); + "rsync -av --debug=all --log-file=%s/rsynclog --delete --timeout=10 %s rsync://%s/checkpoint/%s/data/", + tsLogDir, tmp, tsSnodeAddress, id); code = execCommand(command); taosRemoveDir(tmp); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 698714da86..61af2f6aae 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2983,7 +2983,7 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt } } else { if (varDataTLen(data + offset) > bytes) { - uError("var data length invalid, varDataTLen(data + offset):%d >= bytes:%d", (int)varDataTLen(data + offset), + uError("var data length invalid, varDataTLen(data + offset):%d > bytes:%d", (int)varDataTLen(data + offset), bytes); code = TSDB_CODE_PAR_VALUE_TOO_LONG; goto _exit; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index cba852b48f..2016550b27 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -392,9 +392,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { return 0; } -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 65ac34c390..1d66a8c323 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -10522,7 +10522,10 @@ int32_t tCloneTbTSMAInfo(STableTSMAInfo *pInfo, STableTSMAInfo **pRes) { pRet->ast = taosStrdup(pInfo->ast); if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY; } - if (code) tFreeTableTSMAInfo(pRet); + if (code) { + tFreeTableTSMAInfo(pRet); + pRet = NULL; + } *pRes = pRet; return code; } diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index aad877312b..d8e7b83c37 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -161,8 +161,7 @@ int32_t toIntegerPure(const char *z, int32_t n, int32_t base, int64_t *value) { int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { errno = 0; char *endPtr = NULL; - switch (type) - { + switch (type) { case TK_NK_INTEGER: { *value = taosStr2Int64(z, &endPtr, 10); if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { @@ -172,7 +171,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(z, &endPtr)); - if(val < (double)INT64_MIN || val > (double)INT64_MAX){ + if (val < (double)INT64_MIN || val > (double)INT64_MAX) { return TSDB_CODE_FAILED; } if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { @@ -181,8 +180,8 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { *value = val; return TSDB_CODE_SUCCESS; } break; - default: - break; + default: + break; } if (n == 0) { @@ -201,7 +200,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { // pure decimal part const char *s = endPtr + 1; const char *end = z + n; - bool pure = true; + bool pure = true; while (s < end) { if (*s < '0' || *s > '9') { pure = false; @@ -210,17 +209,17 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { s++; } if (pure) { - if (endPtr+1 < end && endPtr[1] > '4') { + if (endPtr + 1 < end && endPtr[1] > '4') { const char *p = z; while (*p == ' ') { p++; } if (*p == '-') { - if ( *value > INT64_MIN) { + if (*value > INT64_MIN) { (*value)--; } } else { - if ( *value < INT64_MAX) { + if (*value < INT64_MAX) { (*value)++; } } @@ -229,7 +228,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } } - // 2. parse as other + // 2. parse as other bool is_neg = false; uint64_t uv = 0; int32_t code = parseSignAndUInteger(z, n, &is_neg, &uv, true); @@ -256,7 +255,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { errno = 0; - char *endPtr = NULL; + char *endPtr = NULL; const char *p = z; switch (type) { case TK_NK_INTEGER: { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3e1a633752..0360b54d6f 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -53,7 +53,8 @@ static void dmMayShouldUpdateIpWhiteList(SDnodeMgmt *pMgmt, int64_t ver) { .msgType = TDMT_MND_RETRIEVE_IP_WHITE, .info.ahandle = (void *)0x9527, .info.refId = 0, - .info.noResp = 0}; + .info.noResp = 0, + .info.handle = 0}; SEpSet epset = {0}; dmGetMnodeEpSet(pMgmt->pData, &epset); @@ -154,7 +155,8 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { .msgType = TDMT_MND_STATUS, .info.ahandle = (void *)0x9527, .info.refId = 0, - .info.noResp = 0}; + .info.noResp = 0, + .info.handle = 0}; SRpcMsg rpcRsp = {0}; dTrace("send status req to mnode, dnodeVer:%" PRId64 " statusSeq:%d", req.dnodeVer, req.statusSeq); @@ -186,7 +188,8 @@ void dmSendNotifyReq(SDnodeMgmt *pMgmt, SNotifyReq *pReq) { .msgType = TDMT_MND_NOTIFY, .info.ahandle = (void *)0x9527, .info.refId = 0, - .info.noResp = 1}; + .info.noResp = 1, + .info.handle = 0}; SEpSet epSet = {0}; dmGetMnodeEpSet(pMgmt->pData, &epSet); diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 6649d164a5..74bf1f964c 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -290,6 +290,7 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { dError("failed to send rpc msg:%s since %s, handle:%p", TMSG_INFO(pMsg->msgType), terrstr(), pMsg->info.handle); return -1; } else { + pMsg->info.handle = 0; rpcSendRequest(pDnode->trans.clientRpc, pEpSet, pMsg, NULL); return 0; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index af20617457..3291f6e3ed 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -56,7 +56,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger); static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); -static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList); +static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList); static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq); static int32_t mndProcessCheckpointReport(SRpcMsg *pReq); @@ -301,17 +301,17 @@ static int32_t mndCheckCreateStreamReq(SCMCreateStreamReq *pCreate) { return 0; } -static int32_t createSchemaByFields(const SArray* pFields, SSchemaWrapper* pWrapper) { +static int32_t createSchemaByFields(const SArray *pFields, SSchemaWrapper *pWrapper) { pWrapper->nCols = taosArrayGetSize(pFields); pWrapper->pSchema = taosMemoryCalloc(pWrapper->nCols, sizeof(SSchema)); if (NULL == pWrapper->pSchema) { return TSDB_CODE_OUT_OF_MEMORY; } - SNode* pNode; + SNode *pNode; int32_t index = 0; - for(int32_t i = 0; i < pWrapper->nCols; i++) { - SField* pField = (SField*)taosArrayGet(pFields, i); + for (int32_t i = 0; i < pWrapper->nCols; i++) { + SField *pField = (SField *)taosArrayGet(pFields, i); if (TSDB_DATA_TYPE_NULL == pField->type) { pWrapper->pSchema[index].type = TSDB_DATA_TYPE_VARCHAR; pWrapper->pSchema[index].bytes = VARSTR_HEADER_SIZE; @@ -328,14 +328,14 @@ static int32_t createSchemaByFields(const SArray* pFields, SSchemaWrapper* pWrap return TSDB_CODE_SUCCESS; } -static bool hasDestPrimaryKey(SSchemaWrapper* pWrapper) { +static bool hasDestPrimaryKey(SSchemaWrapper *pWrapper) { if (pWrapper->nCols < 2) { return false; } for (int32_t i = 1; i < pWrapper->nCols; i++) { - if(pWrapper->pSchema[i].flags & COL_IS_KEY) { + if (pWrapper->pSchema[i].flags & COL_IS_KEY) { return true; - } + } } return false; } @@ -445,7 +445,7 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, pObj->outputSchema.pSchema = pFullSchema; } - bool hasKey = hasDestPrimaryKey(&pObj->outputSchema); + bool hasKey = hasDestPrimaryKey(&pObj->outputSchema); SPlanContext cxt = { .pAstRoot = pAst, .topicQuery = false, @@ -492,7 +492,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { SEncoder encoder; tEncoderInit(&encoder, NULL, 0); - if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) { pTask->ver = SSTREAM_TASK_VER; } tEncodeStreamTask(&encoder, pTask); @@ -700,7 +700,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SStreamObj streamObj = {0}; char *sql = NULL; int32_t sqlLen = 0; - const char* pMsg = "create stream tasks on dnodes"; + const char *pMsg = "create stream tasks on dnodes"; terrno = TSDB_CODE_SUCCESS; @@ -959,7 +959,8 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask return -1; } - code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0); + code = + setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0); if (code != 0) { taosMemoryFree(buf); } @@ -1106,7 +1107,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); } - SArray* pInvalidList = taosArrayInit(4, sizeof(STaskId)); + SArray *pInvalidList = taosArrayInit(4, sizeof(STaskId)); for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { STaskId *p = taosArrayGet(execInfo.pTaskList, i); @@ -1116,8 +1117,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { } if (pEntry->status == TASK_STATUS__STOP) { - for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) { - STaskId* pId = taosArrayGet(pInvalidList, j); + for (int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) { + STaskId *pId = taosArrayGet(pInvalidList, j); if (pEntry->id.streamId == pId->streamId) { taosArrayPush(pInvalidList, &pEntry->id); break; @@ -1126,8 +1127,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { } if (pEntry->status != TASK_STATUS__READY) { - mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", - pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status)); + mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", pEntry->id.streamId, + (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status)); ready = false; } @@ -1153,14 +1154,14 @@ typedef struct { int64_t duration; } SCheckpointInterval; -static int32_t streamWaitComparFn(const void* p1, const void* p2) { - const SCheckpointInterval* pInt1 = p1; - const SCheckpointInterval* pInt2 = p2; +static int32_t streamWaitComparFn(const void *p1, const void *p2) { + const SCheckpointInterval *pInt1 = p1; + const SCheckpointInterval *pInt2 = p2; if (pInt1->duration == pInt2->duration) { return 0; } - return pInt1->duration > pInt2->duration? -1:1; + return pInt1->duration > pInt2->duration ? -1 : 1; } static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { @@ -1176,7 +1177,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { return -1; } - SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval)); + SArray *pList = taosArrayInit(4, sizeof(SCheckpointInterval)); int64_t now = taosGetTimestampMs(); while ((pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream)) != NULL) { @@ -1190,8 +1191,7 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { taosArrayPush(pList, &in); int32_t currentSize = taosArrayGetSize(pList); - mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 - "s) beyond threshold:%d", + mDebug("stream:%s (uid:0x%" PRIx64 ") checkpoint interval beyond threshold: %ds(%" PRId64 "s) beyond threshold:%d", pStream->name, pStream->uid, tsStreamCheckpointInterval, duration / 1000, currentSize); sdbRelease(pSdb, pStream); @@ -1278,11 +1278,11 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } if (pStream->smaId != 0) { - mDebug("stream:%s, uid:0x%"PRIx64" try to drop sma related stream", dropReq.name, pStream->uid); + mDebug("stream:%s, uid:0x%" PRIx64 " try to drop sma related stream", dropReq.name, pStream->uid); void *pIter = NULL; SSmaObj *pSma = NULL; - pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma); + pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma); while (pIter) { if (pSma && pSma->uid == pStream->smaId) { sdbRelease(pMnode->pSdb, pSma); @@ -1321,7 +1321,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { STrans *pTrans = doCreateTrans(pMnode, pStream, pReq, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); if (pTrans == NULL) { - mError("stream:%s uid:0x%"PRIx64" failed to drop since %s", dropReq.name, pStream->uid, terrstr()); + mError("stream:%s uid:0x%" PRIx64 " failed to drop since %s", dropReq.name, pStream->uid, terrstr()); sdbRelease(pMnode->pSdb, pStream); tFreeMDropStreamReq(&dropReq); return -1; @@ -1674,8 +1674,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); // input total - const char* formatTotalMb = "%7.2f MiB"; - const char* formatTotalGb = "%7.2f GiB"; + const char *formatTotalMb = "%7.2f MiB"; + const char *formatTotalGb = "%7.2f GiB"; if (pe->procsTotal < 1024) { snprintf(buf, tListLen(buf), formatTotalMb, pe->procsTotal); } else { @@ -1689,8 +1689,8 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); // process throughput - const char* formatKb = "%7.2f KiB/s"; - const char* formatMb = "%7.2f MiB/s"; + const char *formatKb = "%7.2f KiB/s"; + const char *formatMb = "%7.2f MiB/s"; if (pe->procsThroughput < 1024) { snprintf(buf, tListLen(buf), formatKb, pe->procsThroughput); } else { @@ -1760,15 +1760,15 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS // start_time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startTime, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startTime, false); // start id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointId, false); // start ver pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*)&pe->startCheckpointVer, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pe->startCheckpointVer, false); // checkpoint time pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1780,11 +1780,11 @@ static int32_t setTaskAttrInResBlock(SStreamObj *pStream, SStreamTask *pTask, SS // checkpoint_id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestId, false); // checkpoint version pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*)&pe->checkpointInfo.latestVer, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pe->checkpointInfo.latestVer, false); // checkpoint size pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1915,13 +1915,12 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { return -1; } - - { // check for tasks, if tasks are not ready, not allowed to pause + { // check for tasks, if tasks are not ready, not allowed to pause bool found = false; bool readyToPause = true; taosThreadMutexLock(&execInfo.lock); - for(int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); @@ -2237,7 +2236,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange return 0; } -static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) { +static int32_t extractNodeListFromStream(SMnode *pMnode, SArray *pNodeList) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; @@ -2359,7 +2358,7 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { return 0; } - int32_t size = sizeof(SMStreamNodeCheckMsg); + int32_t size = sizeof(SMStreamNodeCheckMsg); SMStreamNodeCheckMsg *pMsg = rpcMallocCont(size); SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_NODECHANGE_CHECK, .pCont = pMsg, .contLen = size}; @@ -2386,8 +2385,8 @@ void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) // add the new vgroups if not added yet bool exist = false; - for(int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) { - SNodeEntry* pEntry = taosArrayGet(pExecNode->pNodeList, j); + for (int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) { + SNodeEntry *pEntry = taosArrayGet(pExecNode->pNodeList, j); if (pEntry->nodeId == pTask->info.nodeId) { exist = true; break; @@ -2514,10 +2513,10 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { return 0; } -static void doAddTaskInfo(SArray* pList, SCheckpointReport* pReport) { +static void doAddTaskInfo(SArray *pList, SCheckpointReport *pReport) { bool existed = false; - for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { - STaskChkptInfo* p = taosArrayGet(pList ,i); + for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + STaskChkptInfo *p = taosArrayGet(pList, i); if (p->taskId == pReport->taskId) { existed = true; break; @@ -2563,8 +2562,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); if (pStream == NULL) { - mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", - req.streamId); + mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", req.streamId); // not in meta-store yet, try to acquire the task in exec buffer // the checkpoint req arrives too soon before the completion of the create stream trans. @@ -2671,7 +2669,7 @@ void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) { pExecInfo->initTaskList = true; } -void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { +void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; @@ -2687,13 +2685,14 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { } } -int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray* pChkptInfoList) { - STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, "update checkpoint-info"); +int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList) { + STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, + "update checkpoint-info"); if (pTrans == NULL) { return terrno; } - /*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); + /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); if (code != 0) { sdbRelease(pMnode->pSdb, pStream); diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index a79fe0cf0a..5c1dd3ab7a 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -39,10 +39,10 @@ static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { } } -static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pInfo) { +static void addIntoCheckpointList(SArray *pList, const SFailedCheckpointInfo *pInfo) { int32_t num = taosArrayGetSize(pList); - for(int32_t i = 0; i < num; ++i) { - SFailedCheckpointInfo* p = taosArrayGet(pList, i); + for (int32_t i = 0; i < num; ++i) { + SFailedCheckpointInfo *p = taosArrayGet(pList, i); if (p->transId == pInfo->transId) { return; } @@ -52,12 +52,13 @@ static void addIntoCheckpointList(SArray* pList, const SFailedCheckpointInfo* pI } int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { - STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, " reset from failed checkpoint"); + STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_TASK_RESET_NAME, + " reset from failed checkpoint"); if (pTrans == NULL) { return terrno; } - /*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); + /*int32_t code = */ mndStreamRegisterTrans(pTrans, MND_STREAM_TASK_RESET_NAME, pStream->uid); int32_t code = mndStreamSetResetTaskAction(pMnode, pTrans, pStream); if (code != 0) { sdbRelease(pMnode->pSdb, pStream); @@ -140,8 +141,8 @@ static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) { return TSDB_CODE_SUCCESS; } -static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) { - SOrphanTask* pTask = taosArrayGet(pList, 0); +static int32_t mndDropOrphanTasks(SMnode *pMnode, SArray *pList) { + SOrphanTask *pTask = taosArrayGet(pList, 0); // check if it is conflict with other trans in both sourceDb and targetDb. bool conflict = mndStreamTransConflictCheck(pMnode, pTask->streamId, MND_STREAM_DROP_NAME, false); @@ -150,7 +151,7 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) { } SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""}; - STrans* pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); + STrans *pTrans = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream"); if (pTrans == NULL) { mError("failed to create trans to drop orphan tasks since %s", terrstr()); return -1; @@ -180,21 +181,21 @@ static int32_t mndDropOrphanTasks(SMnode* pMnode, SArray* pList) { return 0; } -int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){ +int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo *info) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void* pIter = NULL; - while(1) { + void *pIter = NULL; + while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; - if(pStream->status != STREAM_STATUS__PAUSE){ + if (pStream->status != STREAM_STATUS__PAUSE) { SMPauseStreamReq reqPause = {0}; strcpy(reqPause.name, pStream->name); reqPause.igNotExists = 1; int32_t contLen = tSerializeSMPauseStreamReq(NULL, 0, &reqPause); - void * pHead = rpcMallocCont(contLen); + void *pHead = rpcMallocCont(contLen); tSerializeSMPauseStreamReq(pHead, contLen, &reqPause); SRpcMsg rpcMsg = { @@ -205,7 +206,8 @@ int32_t suspendAllStreams(SMnode *pMnode, SRpcHandleInfo* info){ }; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); - mInfo("receive pause stream:%s, %s, %"PRId64 ", because grant expired", pStream->name, reqPause.name, pStream->uid); + mInfo("receive pause stream:%s, %s, %" PRId64 ", because grant expired", pStream->name, reqPause.name, + pStream->uid); } sdbRelease(pSdb, pStream); @@ -311,7 +313,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { if (allReady || snodeChanged) { // if the execInfo.activeCheckpoint == 0, the checkpoint is restoring from wal - for(int32_t i = 0; i < taosArrayGetSize(pFailedChkpt); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pFailedChkpt); ++i) { SFailedCheckpointInfo *pInfo = taosArrayGet(pFailedChkpt, i); mInfo("checkpointId:%" PRId64 " transId:%d failed, issue task-reset trans to reset all tasks status", pInfo->checkpointId, pInfo->transId); @@ -341,11 +343,11 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { { SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamHbRspMsg)}; rsp.pCont = rpcMallocCont(rsp.contLen); - SMsgHead* pHead = rsp.pCont; + SMsgHead *pHead = rsp.pCont; pHead->vgId = htonl(req.vgId); tmsgSendRsp(&rsp); - pReq->info.handle = NULL; // disable auto rsp + pReq->info.handle = NULL; // disable auto rsp } return TSDB_CODE_SUCCESS; diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 0daa383d3e..f252791618 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -17,21 +17,21 @@ #include "mndTrans.h" typedef struct SKeyInfo { - void* pKey; + void *pKey; int32_t keyLen; } SKeyInfo; -int32_t mndStreamRegisterTrans(STrans* pTrans, const char* pTransName, int64_t streamId) { +int32_t mndStreamRegisterTrans(STrans *pTrans, const char *pTransName, int64_t streamId) { SStreamTransInfo info = { .transId = pTrans->id, .startTime = taosGetTimestampMs(), .name = pTransName, .streamId = streamId}; taosHashPut(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId), &info, sizeof(SStreamTransInfo)); return 0; } -int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) { +int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt) { size_t keyLen = 0; - void* pIter = NULL; - SArray* pList = taosArrayInit(4, sizeof(SKeyInfo)); + void *pIter = NULL; + SArray *pList = taosArrayInit(4, sizeof(SKeyInfo)); int32_t num = 0; while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { @@ -43,7 +43,8 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) void *pKey = taosHashGetKey(pEntry, &keyLen); // key is the name of src/dst db name SKeyInfo info = {.pKey = pKey, .keyLen = keyLen}; - mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, pEntry->startTime); + mDebug("transId:%d %s startTs:%" PRId64 " cleared since finished", pEntry->transId, pEntry->name, + pEntry->startTime); taosArrayPush(pList, &info); } else { if (strcmp(pEntry->name, MND_STREAM_CHECKPOINT_NAME) == 0) { @@ -55,7 +56,7 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) int32_t size = taosArrayGetSize(pList); for (int32_t i = 0; i < size; ++i) { - SKeyInfo* pKey = taosArrayGet(pList, i); + SKeyInfo *pKey = taosArrayGet(pList, i); taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); } @@ -76,7 +77,7 @@ int32_t mndStreamClearFinishedTrans(SMnode* pMnode, int32_t* pNumOfActiveChkpt) // For a given stream: // 1. checkpoint trans is conflict with any other trans except for the drop and reset trans. // 2. create/drop/reset/update trans are conflict with any other trans. -bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* pTransName, bool lock) { +bool mndStreamTransConflictCheck(SMnode *pMnode, int64_t streamId, const char *pTransName, bool lock) { if (lock) { taosThreadMutexLock(&execInfo.lock); } @@ -127,7 +128,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p return false; } -int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) { +int32_t mndStreamGetRelTrans(SMnode *pMnode, int64_t streamId) { taosThreadMutexLock(&execInfo.lock); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { @@ -136,7 +137,7 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) { } mndStreamClearFinishedTrans(pMnode, NULL); - SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); + SStreamTransInfo *pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); if (pEntry != NULL) { SStreamTransInfo tInfo = *pEntry; taosThreadMutexUnlock(&execInfo.lock); @@ -152,7 +153,8 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) { return 0; } -STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg) { +STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, + const char *pMsg) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); if (pTrans == NULL) { mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); @@ -208,7 +210,7 @@ SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { terrno = TSDB_CODE_SUCCESS; - STREAM_ENCODE_OVER: +STREAM_ENCODE_OVER: taosMemoryFreeClear(buf); if (terrno != TSDB_CODE_SUCCESS) { mError("stream:%s, failed to encode to raw:%p since %s", pStream->name, pRaw, terrstr()); @@ -248,12 +250,16 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status) int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, int32_t retryCode, int32_t acceptCode) { - STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode, + STransAction action = {.epSet = *pEpset, + .contLen = contLen, + .pCont = pCont, + .msgType = msgType, + .retryCode = retryCode, .acceptableCode = acceptCode}; return mndTransAppendRedoAction(pTrans, &action); } -static bool identicalName(const char* pDb, const char* pParam, int32_t len) { +static bool identicalName(const char *pDb, const char *pParam, int32_t len) { return (strlen(pDb) == len) && (strncmp(pDb, pParam, len) == 0); } @@ -300,5 +306,3 @@ void killAllCheckpointTrans(SMnode *pMnode, SVgroupChangeInfo *pChangeInfo) { mDebug("complete clear checkpoints in Dbs"); } - - diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 5f019d07f5..e2bedc258a 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -41,7 +41,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg); static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg); static int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSubscribe(SMnode *pMnode, void *pIter); -static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* hash); +static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *hash); static int32_t mndSetSubCommitLogs(STrans *pTrans, SMqSubscribeObj *pSub) { SSdbRaw *pCommitRaw = mndSubActionEncode(pSub); @@ -95,13 +95,13 @@ static SMqSubscribeObj *mndCreateSubscription(SMnode *pMnode, const SMqTopicObj return pSub; } -static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, - const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { +static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj *pSub, const SMqRebOutputVg *pRebVg, + SSubplan *pPlan) { SMqRebVgReq req = {0}; req.oldConsumerId = pRebVg->oldConsumerId; req.newConsumerId = pRebVg->newConsumerId; req.vgId = pRebVg->pVgEp->vgId; - if(pPlan){ + if (pPlan) { pPlan->execNode.epSet = pRebVg->pVgEp->epSet; pPlan->execNode.nodeId = pRebVg->pVgEp->vgId; int32_t msgLen; @@ -109,7 +109,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj terrno = TSDB_CODE_QRY_INVALID_INPUT; return -1; } - }else{ + } else { req.qmsg = taosStrdup(""); } req.subType = pSub->subType; @@ -126,7 +126,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj } tlen += sizeof(SMsgHead); - void *buf = taosMemoryMalloc(tlen); + void *buf = taosMemoryMalloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(req.qmsg); @@ -155,9 +155,9 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, SMqSubscribeObj } static int32_t mndPersistSubChangeVgReq(SMnode *pMnode, STrans *pTrans, SMqSubscribeObj *pSub, - const SMqRebOutputVg *pRebVg, SSubplan* pPlan) { + const SMqRebOutputVg *pRebVg, SSubplan *pPlan) { if (pRebVg->oldConsumerId == pRebVg->newConsumerId) { - if(pRebVg->oldConsumerId == -1) return 0; //drop stream, no consumer, while split vnode,all consumerId is -1 + if (pRebVg->oldConsumerId == -1) return 0; // drop stream, no consumer, while split vnode,all consumerId is -1 terrno = TSDB_CODE_MND_INVALID_SUB_OPTION; return -1; } @@ -223,19 +223,18 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebInfo; } -static void pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char* key){ +static void pushVgDataToHash(SArray *vgs, SHashObj *pHash, int64_t consumerId, char *key) { SMqVgEp *pVgEp = *(SMqVgEp **)taosArrayPop(vgs); SMqRebOutputVg outputVg = {consumerId, -1, pVgEp}; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); - mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64, - key, pVgEp->vgId, consumerId); + mInfo("[rebalance] sub:%s mq rebalance remove vgId:%d from consumer:0x%" PRIx64, key, pVgEp->vgId, consumerId); } static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t actualRemoved = 0; for (int32_t i = 0; i < numOfRemoved; i++) { - uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); + uint64_t consumerId = *(uint64_t *)taosArrayGet(pInput->pRebInfo->removedConsumers, i); SMqConsumerEp *pConsumerEp = taosHashGet(pOutput->pSub->consumerHash, &consumerId, sizeof(int64_t)); if (pConsumerEp == NULL) { continue; @@ -253,14 +252,15 @@ static void processRemovedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, c } if (numOfRemoved != actualRemoved) { - mError("[rebalance] sub:%s mq rebalance removedNum:%d not matched with actual:%d", pOutput->pSub->key, numOfRemoved, actualRemoved); + mError("[rebalance] sub:%s mq rebalance removedNum:%d not matched with actual:%d", pOutput->pSub->key, numOfRemoved, + actualRemoved); } else { mInfo("[rebalance] sub:%s removed %d consumers", pOutput->pSub->key, numOfRemoved); } } static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj *pInput) { - int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers); + int32_t numOfNewConsumers = taosArrayGetSize(pInput->pRebInfo->newConsumers); for (int32_t i = 0; i < numOfNewConsumers; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(pInput->pRebInfo->newConsumers, i); @@ -276,26 +276,26 @@ static void processNewConsumers(SMqRebOutputObj *pOutput, const SMqRebInputObj * } static void processUnassignedVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash) { - int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs); + int32_t numOfVgroups = taosArrayGetSize(pOutput->pSub->unassignedVgs); for (int32_t i = 0; i < numOfVgroups; i++) { pushVgDataToHash(pOutput->pSub->unassignedVgs, pHash, -1, pOutput->pSub->key); } } -//static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ -// for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ -// SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); -// SMqRebOutputVg outputVg = { -// .oldConsumerId = pConsumerEp->consumerId, -// .newConsumerId = pConsumerEp->consumerId, -// .pVgEp = pVgEp, -// }; -// taosArrayPush(pOutput->rebVgs, &outputVg); -// } -//} +// static void putNoTransferToOutput(SMqRebOutputObj *pOutput, SMqConsumerEp *pConsumerEp){ +// for(int i = 0; i < taosArrayGetSize(pConsumerEp->vgs); i++){ +// SMqVgEp *pVgEp = (SMqVgEp *)taosArrayGetP(pConsumerEp->vgs, i); +// SMqRebOutputVg outputVg = { +// .oldConsumerId = pConsumerEp->consumerId, +// .newConsumerId = pConsumerEp->consumerId, +// .pVgEp = pVgEp, +// }; +// taosArrayPush(pOutput->rebVgs, &outputVg); +// } +// } static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt, - int32_t remainderVgCnt) { + int32_t remainderVgCnt) { int32_t cnt = 0; void *pIter = NULL; @@ -311,7 +311,7 @@ static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, taosArrayPush(pOutput->modifyConsumers, &pConsumerEp->consumerId); if (consumerVgNum > minVgCnt) { if (cnt < remainderVgCnt) { - while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) { // pop until equal minVg + 1 + while (taosArrayGetSize(pConsumerEp->vgs) > minVgCnt + 1) { // pop until equal minVg + 1 pushVgDataToHash(pConsumerEp->vgs, pHash, pConsumerEp->consumerId, pOutput->pSub->key); } cnt++; @@ -321,17 +321,17 @@ static void processModifiedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, } } } -// putNoTransferToOutput(pOutput, pConsumerEp); + // putNoTransferToOutput(pOutput, pConsumerEp); } } -static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){ +static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput) { int32_t totalVgNum = 0; - SVgObj* pVgroup = NULL; - void* pIter = NULL; - SArray* newVgs = taosArrayInit(0, POINTER_BYTES); + SVgObj *pVgroup = NULL; + void *pIter = NULL; + SArray *newVgs = taosArrayInit(0, POINTER_BYTES); while (1) { - pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgroup); + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) { break; } @@ -342,7 +342,7 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){ } totalVgNum++; - SMqVgEp* pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); + SMqVgEp *pVgEp = taosMemoryMalloc(sizeof(SMqVgEp)); pVgEp->epSet = mndGetVgroupEpset(pMnode, pVgroup); pVgEp->vgId = pVgroup->vgId; taosArrayPush(newVgs, &pVgEp); @@ -358,17 +358,17 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){ int32_t j = 0; while (j < taosArrayGetSize(pConsumerEp->vgs)) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - bool find = false; - for(int32_t k = 0; k < taosArrayGetSize(newVgs); k++){ + bool find = false; + for (int32_t k = 0; k < taosArrayGetSize(newVgs); k++) { SMqVgEp *pnewVgEp = taosArrayGetP(newVgs, k); - if(pVgEp->vgId == pnewVgEp->vgId){ + if (pVgEp->vgId == pnewVgEp->vgId) { tDeleteSMqVgEp(pnewVgEp); taosArrayRemove(newVgs, k); find = true; break; } } - if(!find){ + if (!find) { mInfo("[rebalance] processRemoveAddVgs old vgId:%d", pVgEp->vgId); tDeleteSMqVgEp(pVgEp); taosArrayRemove(pConsumerEp->vgs, j); @@ -378,17 +378,17 @@ static int32_t processRemoveAddVgs(SMnode *pMnode, SMqRebOutputObj *pOutput){ } } - if(taosArrayGetSize(pOutput->pSub->unassignedVgs) == 0 && taosArrayGetSize(newVgs) != 0){ + if (taosArrayGetSize(pOutput->pSub->unassignedVgs) == 0 && taosArrayGetSize(newVgs) != 0) { taosArrayAddAll(pOutput->pSub->unassignedVgs, newVgs); mInfo("[rebalance] processRemoveAddVgs add new vg num:%d", (int)taosArrayGetSize(newVgs)); taosArrayDestroy(newVgs); - }else{ + } else { taosArrayDestroyP(newVgs, (FDelete)tDeleteSMqVgEp); } return totalVgNum; } -static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput){ +static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pInput->pRebInfo->key); // put all offset rows if (pSub == NULL) { return; @@ -406,16 +406,17 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S for (int j = 0; j < taosArrayGetSize(pConsumerEp->offsetRows); j++) { OffsetRows *d1 = taosArrayGet(pConsumerEp->offsetRows, j); - bool jump = false; - for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++){ + bool jump = false; + for (int i = 0; pConsumerEpNew && i < taosArrayGetSize(pConsumerEpNew->vgs); i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEpNew->vgs, i); - if(pVgEp->vgId == d1->vgId){ + if (pVgEp->vgId == d1->vgId) { jump = true; - mInfo("pSub->offsetRows jump, because consumer id:0x%"PRIx64 " and vgId:%d not change", pConsumerEp->consumerId, pVgEp->vgId); + mInfo("pSub->offsetRows jump, because consumer id:0x%" PRIx64 " and vgId:%d not change", + pConsumerEp->consumerId, pVgEp->vgId); break; } } - if(jump) continue; + if (jump) continue; bool find = false; for (int i = 0; i < taosArrayGetSize(pOutput->pSub->offsetRows); i++) { OffsetRows *d2 = taosArrayGet(pOutput->pSub->offsetRows, i); @@ -424,11 +425,11 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S d2->offset = d1->offset; d2->ever = d1->ever; find = true; - mInfo("pSub->offsetRows add vgId:%d, after:%"PRId64", before:%"PRId64, d2->vgId, d2->rows, d1->rows); + mInfo("pSub->offsetRows add vgId:%d, after:%" PRId64 ", before:%" PRId64, d2->vgId, d2->rows, d1->rows); break; } } - if(!find){ + if (!find) { taosArrayPush(pOutput->pSub->offsetRows, d1); } } @@ -437,7 +438,7 @@ static void processSubOffsetRows(SMnode *pMnode, const SMqRebInputObj *pInput, S mndReleaseSubscribe(pMnode, pSub); } -static void printRebalanceLog(SMqRebOutputObj *pOutput){ +static void printRebalanceLog(SMqRebOutputObj *pOutput) { mInfo("sub:%s mq rebalance calculation completed, re-balanced vg", pOutput->pSub->key); for (int32_t i = 0; i < taosArrayGetSize(pOutput->rebVgs); i++) { SMqRebOutputVg *pOutputRebVg = taosArrayGet(pOutput->rebVgs, i); @@ -451,7 +452,8 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput){ if (pIter == NULL) break; SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; int32_t sz = taosArrayGetSize(pConsumerEp->vgs); - mInfo("sub:%s mq rebalance final cfg: consumer:0x%" PRIx64 " has %d vg", pOutput->pSub->key, pConsumerEp->consumerId, sz); + mInfo("sub:%s mq rebalance final cfg: consumer:0x%" PRIx64 " has %d vg", pOutput->pSub->key, + pConsumerEp->consumerId, sz); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, i); mInfo("sub:%s mq rebalance final cfg: vg %d to consumer:0x%" PRIx64, pOutput->pSub->key, pVgEp->vgId, @@ -460,8 +462,8 @@ static void printRebalanceLog(SMqRebOutputObj *pOutput){ } } -static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, const char *pSubKey, - int32_t *minVgCnt, int32_t *remainderVgCnt){ +static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, const char *pSubKey, int32_t *minVgCnt, + int32_t *remainderVgCnt) { int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); int32_t numOfAdded = taosArrayGetSize(pInput->pRebInfo->newConsumers); int32_t numOfFinal = pInput->oldConsumerNum + numOfAdded - numOfRemoved; @@ -473,11 +475,12 @@ static void calcVgroupsCnt(const SMqRebInputObj *pInput, int32_t totalVgNum, con } else { mInfo("[rebalance] sub:%s no consumer subscribe this topic", pSubKey); } - mInfo("[rebalance] sub:%s mq rebalance %d vgroups, existed consumers:%d, added:%d, removed:%d, minVg:%d remainderVg:%d", - pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved, *minVgCnt, *remainderVgCnt); + mInfo( + "[rebalance] sub:%s mq rebalance %d vgroups, existed consumers:%d, added:%d, removed:%d, minVg:%d remainderVg:%d", + pSubKey, totalVgNum, pInput->oldConsumerNum, numOfAdded, numOfRemoved, *minVgCnt, *remainderVgCnt); } -static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt){ +static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t minVgCnt) { SMqRebOutputVg *pRebVg = NULL; void *pAssignIter = NULL; void *pIter = NULL; @@ -498,8 +501,8 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min pRebVg = (SMqRebOutputVg *)pAssignIter; pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); - mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", - pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } @@ -519,13 +522,13 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min pRebVg = (SMqRebOutputVg *)pAssignIter; pRebVg->newConsumerId = pConsumerEp->consumerId; taosArrayPush(pConsumerEp->vgs, &pRebVg->pVgEp); - mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", - pRebVg->pVgEp->vgId, pConsumerEp->consumerId); + mInfo("[rebalance] mq rebalance: add vgId:%d to consumer:0x%" PRIx64 " for average + 1", pRebVg->pVgEp->vgId, + pConsumerEp->consumerId); } } taosHashCancelIterate(pOutput->pSub->consumerHash, pIter); - if(pAssignIter != NULL){ + if (pAssignIter != NULL) { mError("[rebalance]sub:%s assign iter is not NULL, never should reach here", pOutput->pSub->key); } while (1) { @@ -534,19 +537,19 @@ static void assignVgroups(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t min break; } - SMqRebOutputVg* pRebOutput = (SMqRebOutputVg *)pAssignIter; + SMqRebOutputVg *pRebOutput = (SMqRebOutputVg *)pAssignIter; taosArrayPush(pOutput->rebVgs, pRebOutput); - if(taosHashGetSize(pOutput->pSub->consumerHash) == 0){ // if all consumer is removed + if (taosHashGetSize(pOutput->pSub->consumerHash) == 0) { // if all consumer is removed taosArrayPush(pOutput->pSub->unassignedVgs, &pRebOutput->pVgEp); // put all vg into unassigned } } } static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebOutputObj *pOutput) { - int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput); + int32_t totalVgNum = processRemoveAddVgs(pMnode, pOutput); const char *pSubKey = pOutput->pSub->key; - int32_t minVgCnt = 0; - int32_t remainderVgCnt = 0; + int32_t minVgCnt = 0; + int32_t remainderVgCnt = 0; SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -561,14 +564,14 @@ static void mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqRebO taosHashCleanup(pHash); } -static int32_t presistConsumerByType(STrans *pTrans, SArray* consumers, int8_t type, char *cgroup, char *topic){ - int32_t code = 0; - SMqConsumerObj *pConsumerNew = NULL; - int32_t consumerNum = taosArrayGetSize(consumers); +static int32_t presistConsumerByType(STrans *pTrans, SArray *consumers, int8_t type, char *cgroup, char *topic) { + int32_t code = 0; + SMqConsumerObj *pConsumerNew = NULL; + int32_t consumerNum = taosArrayGetSize(consumers); for (int32_t i = 0; i < consumerNum; i++) { int64_t consumerId = *(int64_t *)taosArrayGet(consumers, i); pConsumerNew = tNewSMqConsumerObj(consumerId, cgroup, type, topic, NULL); - if (pConsumerNew == NULL){ + if (pConsumerNew == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto END; } @@ -587,7 +590,7 @@ END: return code; } -static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic){ +static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput, char *cgroup, char *topic) { int32_t code = presistConsumerByType(pTrans, pOutput->modifyConsumers, CONSUMER_UPDATE_REB, cgroup, NULL); if (code != 0) { return code; @@ -603,10 +606,10 @@ static int32_t mndPresistConsumer(STrans *pTrans, const SMqRebOutputObj *pOutput static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOutputObj *pOutput) { struct SSubplan *pPlan = NULL; - int32_t code = 0; + int32_t code = 0; STrans *pTrans = NULL; - if(strcmp(pOutput->pSub->qmsg, "") != 0){ + if (strcmp(pOutput->pSub->qmsg, "") != 0) { code = qStringToSubplan(pOutput->pSub->qmsg, &pPlan); if (code != 0) { terrno = code; @@ -660,7 +663,7 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu code = mndTransPrepare(pMnode, pTrans); END: - nodesDestroyNode((SNode*)pPlan); + nodesDestroyNode((SNode *)pPlan); mndTransDrop(pTrans); return code; } @@ -672,24 +675,24 @@ static void freeRebalanceItem(void *param) { } // type = 0 remove type = 1 add -static void buildRebInfo(SHashObj* rebSubHash, SArray* topicList, int8_t type, char *group, int64_t consumerId){ +static void buildRebInfo(SHashObj *rebSubHash, SArray *topicList, int8_t type, char *group, int64_t consumerId) { int32_t topicNum = taosArrayGetSize(topicList); for (int32_t i = 0; i < topicNum; i++) { char key[TSDB_SUBSCRIBE_KEY_LEN]; char *removedTopic = taosArrayGetP(topicList, i); mndMakeSubscribeKey(key, group, removedTopic); SMqRebInfo *pRebSub = mndGetOrCreateRebSub(rebSubHash, key); - if(type == 0) + if (type == 0) taosArrayPush(pRebSub->removedConsumers, &consumerId); - else if(type == 1) + else if (type == 1) taosArrayPush(pRebSub->newConsumers, &consumerId); } } -static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj* rebSubHash){ +static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHashObj *rebSubHash) { int32_t newTopicNum = taosArrayGetSize(pConsumer->currentTopics); for (int32_t i = 0; i < newTopicNum; i++) { - char * topic = taosArrayGetP(pConsumer->currentTopics, i); + char *topic = taosArrayGetP(pConsumer->currentTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topic); if (pSub == NULL) { continue; @@ -702,7 +705,7 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash for (int32_t j = 0; j < vgNum; j++) { SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - SVgObj * pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); + SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVgEp->vgId); if (!pVgroup) { char key[TSDB_SUBSCRIBE_KEY_LEN]; mndMakeSubscribeKey(key, pConsumer->cgroup, topic); @@ -716,7 +719,7 @@ static void checkForVgroupSplit(SMnode *pMnode, SMqConsumerObj *pConsumer, SHash } } -static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { +static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj *rebSubHash) { SMnode *pMnode = pMsg->info.node; SSdb *pSdb = pMnode->pSdb; SMqConsumerObj *pConsumer = NULL; @@ -732,17 +735,19 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { int32_t hbStatus = atomic_add_fetch_32(&pConsumer->hbStatus, 1); int32_t status = atomic_load_32(&pConsumer->status); - mDebug("[rebalance] check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 ", hbstatus:%d", - pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, pConsumer->createTime, hbStatus); + mDebug("[rebalance] check for consumer:0x%" PRIx64 " status:%d(%s), sub-time:%" PRId64 ", createTime:%" PRId64 + ", hbstatus:%d", + pConsumer->consumerId, status, mndConsumerStatusName(status), pConsumer->subscribeTime, + pConsumer->createTime, hbStatus); if (status == MQ_CONSUMER_STATUS_READY) { - if (taosArrayGetSize(pConsumer->assignedTopics) == 0) {// unsubscribe or close + if (taosArrayGetSize(pConsumer->assignedTopics) == 0) { // unsubscribe or close mndSendConsumerMsg(pMnode, pConsumer->consumerId, TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, &pMsg->info); } else if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { taosRLockLatch(&pConsumer->lock); buildRebInfo(rebSubHash, pConsumer->currentTopics, 0, pConsumer->cgroup, pConsumer->consumerId); taosRUnLockLatch(&pConsumer->lock); - }else{ + } else { checkForVgroupSplit(pMnode, pConsumer, rebSubHash); } } else if (status == MQ_CONSUMER_STATUS_LOST) { @@ -762,8 +767,7 @@ static void mndCheckConsumer(SRpcMsg *pMsg, SHashObj* rebSubHash) { bool mndRebTryStart() { int32_t old = atomic_val_compare_exchange_32(&mqRebInExecCnt, 0, 1); - if (old > 0) mInfo("[rebalance] counter old val:%d", old) - return old == 0; + if (old > 0) mInfo("[rebalance] counter old val:%d", old) return old == 0; } void mndRebCntInc() { @@ -776,7 +780,7 @@ void mndRebCntDec() { if (val > 0) mInfo("[rebalance] cnt sub, value:%d", val) } -static void clearRebOutput(SMqRebOutputObj *rebOutput){ +static void clearRebOutput(SMqRebOutputObj *rebOutput) { taosArrayDestroy(rebOutput->newConsumers); taosArrayDestroy(rebOutput->modifyConsumers); taosArrayDestroy(rebOutput->removedConsumers); @@ -800,15 +804,15 @@ static int32_t initRebOutput(SMqRebOutputObj *rebOutput) { } // This function only works when there are dirty consumers -static void checkConsumer(SMnode *pMnode, SMqSubscribeObj* pSub){ - void *pIter = NULL; +static void checkConsumer(SMnode *pMnode, SMqSubscribeObj *pSub) { + void *pIter = NULL; while (1) { pIter = taosHashIterate(pSub->consumerHash, pIter); if (pIter == NULL) { break; } - SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; + SMqConsumerEp *pConsumerEp = (SMqConsumerEp *)pIter; SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pConsumerEp->consumerId); if (pConsumer != NULL) { mndReleaseConsumer(pMnode, pConsumer); @@ -822,8 +826,8 @@ static void checkConsumer(SMnode *pMnode, SMqSubscribeObj* pSub){ } } -static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput){ - const char *key = rebInput->pRebInfo->key; +static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOutputObj *rebOutput) { + const char *key = rebInput->pRebInfo->key; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, key); if (pSub == NULL) { @@ -869,14 +873,13 @@ static int32_t buildRebOutput(SMnode *pMnode, SMqRebInputObj *rebInput, SMqRebOu } static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { - int code = 0; - void *pIter = NULL; - SMnode *pMnode = pMsg->info.node; + int code = 0; + void *pIter = NULL; + SMnode *pMnode = pMsg->info.node; mDebug("[rebalance] start to process mq timer") - if (!mndRebTryStart()) { - mInfo("[rebalance] mq rebalance already in progress, do nothing") - return code; + if (!mndRebTryStart()) { + mInfo("[rebalance] mq rebalance already in progress, do nothing") return code; } SHashObj *rebSubHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); @@ -900,13 +903,13 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; code = initRebOutput(&rebOutput); - if (code != 0){ + if (code != 0) { goto END; } - rebInput.pRebInfo = (SMqRebInfo*)pIter; + rebInput.pRebInfo = (SMqRebInfo *)pIter; - if (buildRebOutput(pMnode, &rebInput, &rebOutput) != 0){ + if (buildRebOutput(pMnode, &rebInput, &rebOutput) != 0) { continue; } @@ -931,12 +934,12 @@ END: return code; } -static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){ - void* pIter = NULL; - SVgObj* pVgObj = NULL; +static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans) { + void *pIter = NULL; + SVgObj *pVgObj = NULL; int32_t ret = 0; while (1) { - pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgObj); + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgObj); if (pIter == NULL) { break; } @@ -946,7 +949,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran continue; } SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); - if(pReq == NULL){ + if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; ret = -1; goto END; @@ -957,7 +960,8 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); STransAction action = {0}; - action.epSet = mndGetVgroupEpset(pMnode, pVgObj);; + action.epSet = mndGetVgroupEpset(pMnode, pVgObj); + ; action.pCont = pReq; action.contLen = sizeof(SMqVDeleteReq); action.msgType = TDMT_VND_TMQ_DELETE_SUB; @@ -969,13 +973,13 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran goto END; } } - END: +END: sdbRelease(pMnode->pSdb, pVgObj); sdbCancelFetch(pMnode->pSdb, pIter); return ret; } -static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic){ +static int32_t mndDropConsumerByGroup(SMnode *pMnode, STrans *pTrans, char *cgroup, char *topic) { void *pIter = NULL; SMqConsumerObj *pConsumer = NULL; int ret = 0; @@ -1012,10 +1016,10 @@ END: } static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; - SMDropCgroupReq dropReq = {0}; - STrans *pTrans = NULL; - int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; + SMnode *pMnode = pMsg->info.node; + SMDropCgroupReq dropReq = {0}; + STrans *pTrans = NULL; + int32_t code = TSDB_CODE_ACTION_IN_PROGRESS; if (tDeserializeSMDropCgroupReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -1262,7 +1266,6 @@ int32_t mndGetGroupNumByTopic(SMnode *pMnode, const char *topicName) { pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pSub); if (pIter == NULL) break; - char topic[TSDB_TOPIC_FNAME_LEN] = {0}; char cgroup[TSDB_CGROUP_LEN] = {0}; mndSplitSubscribeKey(pSub->key, topic, cgroup, true); @@ -1340,7 +1343,8 @@ END: return code; } -static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t consumerId, const char* topic, const char* cgroup, SArray* vgs, SArray *offsetRows){ +static int32_t buildResult(SSDataBlock *pBlock, int32_t *numOfRows, int64_t consumerId, const char *topic, + const char *cgroup, SArray *vgs, SArray *offsetRows) { int32_t sz = taosArrayGetSize(vgs); for (int32_t j = 0; j < sz; j++) { SMqVgEp *pVgEp = taosArrayGetP(vgs, j); @@ -1359,37 +1363,37 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t cons colDataSetVal(pColInfo, *numOfRows, (const char *)&pVgEp->vgId, false); // consumer id - char consumerIdHex[32] = {0}; - sprintf(varDataVal(consumerIdHex), "0x%"PRIx64, consumerId); + char consumerIdHex[32] = {0}; + sprintf(varDataVal(consumerIdHex), "0x%" PRIx64, consumerId); varDataSetLen(consumerIdHex, strlen(varDataVal(consumerIdHex))); - + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)consumerIdHex, consumerId == -1); - - mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), - consumerId, varDataVal(cgroup), pVgEp->vgId); + + mInfo("mnd show subscriptions: topic %s, consumer:0x%" PRIx64 " cgroup %s vgid %d", varDataVal(topic), consumerId, + varDataVal(cgroup), pVgEp->vgId); // offset OffsetRows *data = NULL; - for(int i = 0; i < taosArrayGetSize(offsetRows); i++){ + for (int i = 0; i < taosArrayGetSize(offsetRows); i++) { OffsetRows *tmp = taosArrayGet(offsetRows, i); - if(tmp->vgId != pVgEp->vgId){ + if (tmp->vgId != pVgEp->vgId) { mError("mnd show subscriptions: do not find vgId:%d, %d in offsetRows", tmp->vgId, pVgEp->vgId); continue; } data = tmp; } - if(data){ + if (data) { // vg id - char buf[TSDB_OFFSET_LEN*2 + VARSTR_HEADER_SIZE] = {0}; + char buf[TSDB_OFFSET_LEN * 2 + VARSTR_HEADER_SIZE] = {0}; tFormatOffset(varDataVal(buf), TSDB_OFFSET_LEN, &data->offset); - sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%"PRId64, data->ever); + sprintf(varDataVal(buf) + strlen(varDataVal(buf)), "/%" PRId64, data->ever); varDataSetLen(buf, strlen(varDataVal(buf))); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)buf, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, *numOfRows, (const char *)&data->rows, false); - }else{ + } else { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetNULL(pColInfo, *numOfRows); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); @@ -1435,7 +1439,8 @@ int32_t mndRetrieveSubscribe(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock if (pIter == NULL) break; pConsumerEp = (SMqConsumerEp *)pIter; - buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, pConsumerEp->offsetRows); + buildResult(pBlock, &numOfRows, pConsumerEp->consumerId, topic, cgroup, pConsumerEp->vgs, + pConsumerEp->offsetRows); } buildResult(pBlock, &numOfRows, -1, topic, cgroup, pSub->unassignedVgs, pSub->offsetRows); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 0eb0db002f..3f7d3d3e8c 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -41,7 +41,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer SCheckpointInfo *pChkInfo = &pTask->chkInfo; tqSetRestoreVersionInfo(pTask); - char* p = streamTaskGetStatus(pTask)->name; + char *p = streamTaskGetStatus(pTask)->name; if (pTask->info.fillHistory) { sndInfo("vgId:%d expand stream task, s-task:%s, checkpointId:%" PRId64 " checkpointVer:%" PRId64 " nextProcessVer:%" PRId64 @@ -71,7 +71,8 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { startRsync(); pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE, + taosGetTimestampMs(), tqStartTaskCompleteCallback); if (pSnode->pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; @@ -138,9 +139,10 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_DEPLOY: { - void * pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pReq = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t len = pMsg->contLen - sizeof(SMsgHead); - return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb,pMsg->info.conn.applyIndex, pReq, len, true, true); + return tqStreamTaskProcessDeployReq(pSnode->pMeta, &pSnode->msgCb, pMsg->info.conn.applyIndex, pReq, len, true, + true); } case TDMT_STREAM_TASK_DROP: @@ -159,4 +161,4 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg *pRsp) { ASSERT(0); } return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index ceb72aa14d..64cee3513c 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -686,8 +686,8 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int } // add to cache. - taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL, - TAOS_LRU_PRIORITY_LOW, NULL); + (void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL, + TAOS_LRU_PRIORITY_LOW, NULL); _end: taosThreadMutexUnlock(pLock); metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid, @@ -909,7 +909,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) { return TSDB_CODE_SUCCESS; } -bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) { +bool metaTbInFilterCache(SMeta* pMeta, const void* key, int8_t type) { if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) { return true; } @@ -921,7 +921,7 @@ bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) { return false; } -int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) { +int32_t metaPutTbToFilterCache(SMeta* pMeta, const void* key, int8_t type) { if (type == 0) { return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0); } @@ -933,7 +933,7 @@ int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) { return 0; } -int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type) { +int32_t metaSizeOfTbFilterCache(SMeta* pMeta, int8_t type) { if (type == 0) { return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb); } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 58eef3caa6..e40478eec2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -647,13 +647,18 @@ int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SCtbIdxKey *table = (SCtbIdxKey *)pKey; STagVal tagVal = {.cid = pCol->colId}; - tTagGet((const STag *)pVal, &tagVal); - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if (tTagGet((const STag *)pVal, &tagVal)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; + if (!IS_VAR_DATA_TYPE(pCol->type)) { + nTagData = tDataTypes[pCol->type].bytes; + } } rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); tdbFree(pKey); @@ -779,13 +784,18 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) SCtbIdxKey *table = (SCtbIdxKey *)pKey; STagVal tagVal = {.cid = pCol->colId}; - tTagGet((const STag *)pVal, &tagVal); - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if (tTagGet((const STag *)pVal, &tagVal)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; + if (!IS_VAR_DATA_TYPE(pCol->type)) { + nTagData = tDataTypes[pCol->type].bytes; + } } rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); tdbFree(pKey); @@ -1272,13 +1282,18 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p int32_t nTagData = 0; STagVal tagVal = {.cid = pTagColumn->colId}; - tTagGet((const STag *)e.ctbEntry.pTags, &tagVal); - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if (tTagGet((const STag *)e.ctbEntry.pTags, &tagVal)) { + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; + if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { + nTagData = tDataTypes[pTagColumn->type].bytes; + } } if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, @@ -2036,13 +2051,18 @@ static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTb int32_t nTagData = 0; STagVal tagVal = {.cid = pCol->colId}; - tTagGet((const STag *)pVal, &tagVal); - if (IS_VAR_DATA_TYPE(pCol->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if (tTagGet((const STag *)pVal, &tagVal)) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pCol->type].bytes; + if (!IS_VAR_DATA_TYPE(pCol->type)) { + nTagData = tDataTypes[pCol->type].bytes; + } } if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { tdbFree(pKey); @@ -2497,19 +2517,25 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { goto end; } else { for (int i = 0; i < pTagSchema->nCols; i++) { + pTagData = NULL; + nTagData = 0; pTagColumn = &pTagSchema->pSchema[i]; if (!IS_IDX_ON(pTagColumn)) continue; STagVal tagVal = {.cid = pTagColumn->colId}; - tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if (tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal)) { + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; + if (!IS_VAR_DATA_TYPE(pTagColumn->type)) { + nTagData = tDataTypes[pTagColumn->type].bytes; + } } - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { ret = -1; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 07d77905cb..4be5983be8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -697,6 +697,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); + atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_store_32(&pHandle->epoch, 0); tqUnregisterPushHandle(pTq, pHandle); @@ -963,8 +964,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { tqError("failed to find s-task:0x%" PRIx64 ", it may have been destroyed, drop related fill-history task:%s", pTask->streamTaskId.taskId, pTask->id.idStr); - tqDebug("s-task:%s fill-history task set status to be dropping and drop it", id); - streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); + tqDebug("s-task:%s fill-history task set status to be dropping", id); + streamBuildAndSendDropTaskMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, 0); atomic_store_32(&pTask->status.inScanHistorySentinel, 0); streamMetaReleaseTask(pMeta, pTask); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 3ea7ad0718..624257d4c6 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -19,13 +19,13 @@ typedef struct SMStreamCheckpointReadyRspMsg { SMsgHead head; - int64_t streamId; - int32_t upstreamTaskId; - int32_t upstreamNodeId; - int32_t downstreamTaskId; - int32_t downstreamNodeId; - int64_t checkpointId; - int32_t transId; + int64_t streamId; + int32_t upstreamTaskId; + int32_t upstreamNodeId; + int32_t downstreamTaskId; + int32_t downstreamNodeId; + int64_t checkpointId; + int32_t transId; } SMStreamCheckpointReadyRspMsg; static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); @@ -160,7 +160,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL || *ppTask == NULL) { - tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId, req.taskId); + tqError("vgId:%d failed to acquire task:0x%x when handling update task epset, it may have been dropped", vgId, + req.taskId); rsp.code = TSDB_CODE_SUCCESS; streamMetaWUnLock(pMeta); taosArrayDestroy(req.pNodeList); @@ -194,7 +195,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM void* pReqTask = taosHashGet(pMeta->updateInfo.pTasks, &entry, sizeof(STaskUpdateEntry)); if (pReqTask != NULL) { - tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId, req.transId); + tqDebug("s-task:%s (vgId:%d) already update in transId:%d, discard the nodeEp update msg", idstr, vgId, + req.transId); rsp.code = TSDB_CODE_SUCCESS; streamMetaWUnLock(pMeta); taosArrayDestroy(req.pNodeList); @@ -838,10 +840,11 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { tqDebug("s-task:%s receive task-reset msg from mnode, reset status and ready for data processing", pTask->id.idStr); taosThreadMutexLock(&pTask->lock); + streamTaskClearCheckInfo(pTask, true); // clear flag set during do checkpoint, and open inputQ for all upstream tasks - SStreamTaskState *pState = streamTaskGetStatus(pTask); + SStreamTaskState* pState = streamTaskGetStatus(pTask); if (pState->state == TASK_STATUS__CK) { int32_t tranId = 0; int64_t activeChkId = 0; @@ -866,7 +869,7 @@ int32_t tqStreamTaskProcessTaskResetReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { } int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { - SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*) pMsg->pCont; + SRetrieveChkptTriggerReq* pReq = (SRetrieveChkptTriggerReq*)pMsg->pCont; SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, pReq->upstreamTaskId); if (pTask == NULL) { @@ -883,7 +886,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) tqError("s-task:%s not ready for checkpoint-trigger retrieve from 0x%x, since downstream not ready", pTask->id.idStr, (int32_t)pReq->downstreamTaskId); - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_STREAM_TASK_IVLD_STATUS); + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + TSDB_CODE_STREAM_TASK_IVLD_STATUS); streamMetaReleaseTask(pMeta, pTask); return TSDB_CODE_SUCCESS; @@ -895,14 +899,15 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) int64_t checkpointId = 0; streamTaskGetActiveCheckpointInfo(pTask, &transId, &checkpointId); - ASSERT (checkpointId == pReq->checkpointId); + ASSERT(checkpointId == pReq->checkpointId); if (streamTaskAlreadySendTrigger(pTask, pReq->downstreamNodeId)) { // re-send the lost checkpoint-trigger msg to downstream task tqDebug("s-task:%s re-send checkpoint-trigger to:0x%x, checkpointId:%" PRId64 ", transId:%d", pTask->id.idStr, (int32_t)pReq->downstreamTaskId, checkpointId, transId); - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_SUCCESS); - } else { // not send checkpoint-trigger yet, wait + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + TSDB_CODE_SUCCESS); + } else { // not send checkpoint-trigger yet, wait int32_t recv = 0, total = 0; streamTaskGetTriggerRecvStatus(pTask, &recv, &total); @@ -914,7 +919,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) "sending checkpoint-source/trigger", pTask->id.idStr, recv, total); } - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + TSDB_CODE_ACTION_IN_PROGRESS); } } else { // upstream not recv the checkpoint-source/trigger till now ASSERT(pState->state == TASK_STATUS__READY || pState->state == TASK_STATUS__HALT); @@ -922,7 +928,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerReq(SStreamMeta* pMeta, SRpcMsg* pMsg) "s-task:%s not recv checkpoint-source from mnode or checkpoint-trigger from upstream yet, wait for all " "upstream sending checkpoint-source/trigger", pTask->id.idStr); - streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, TSDB_CODE_ACTION_IN_PROGRESS); + streamTaskSendCheckpointTriggerMsg(pTask, pReq->downstreamTaskId, pReq->downstreamNodeId, &pMsg->info, + TSDB_CODE_ACTION_IN_PROGRESS); } streamMetaReleaseTask(pMeta, pTask); @@ -940,8 +947,8 @@ int32_t tqStreamTaskProcessRetrieveTriggerRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) return TSDB_CODE_STREAM_TASK_NOT_EXIST; } - tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%"PRId64", transId:%d", - pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); + tqDebug("s-task:%s recv re-send checkpoint-trigger msg from upstream:0x%x, checkpointId:%" PRId64 ", transId:%d", + pTask->id.idStr, pRsp->upstreamTaskId, pRsp->checkpointId, pRsp->transId); streamTaskProcessCheckpointTriggerRsp(pTask, pRsp); streamMetaReleaseTask(pMeta, pTask); @@ -998,7 +1005,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t int32_t level = pTask->info.taskLevel; if (level == TASK_LEVEL__SINK) { - ASSERT (status != TASK_STATUS__UNINIT); + ASSERT(status != TASK_STATUS__UNINIT); streamMetaReleaseTask(pMeta, pTask); return 0; } @@ -1025,7 +1032,7 @@ static int32_t tqProcessTaskResumeImpl(void* handle, SStreamTask* pTask, int64_t streamTrySchedExec(pTask); } } else { - ASSERT (status != TASK_STATUS__UNINIT); + ASSERT(status != TASK_STATUS__UNINIT); } streamMetaReleaseTask(pMeta, pTask); @@ -1063,7 +1070,7 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } -int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {return doProcessDummyRspMsg(pMeta, pMsg);} +int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 62b29db76e..bee5a1d990 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -771,6 +771,7 @@ int32_t tsdbCacheNewTable(STsdb *pTsdb, tb_uid_t uid, tb_uid_t suid, SSchemaWrap STSchema *pTSchema = NULL; code = metaGetTbTSchemaEx(pTsdb->pVnode->pMeta, suid, uid, -1, &pTSchema); if (code != TSDB_CODE_SUCCESS) { + taosThreadMutexUnlock(&pTsdb->lruMutex); terrno = code; return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c6f13bc0a8..ae807b10e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -49,7 +49,8 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); static int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); -static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader); +static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, + STsdbReader* pReader); static int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, SRowMerger* pMerger, int32_t pkSrcSlot, SVersionRange* pVerRange, const char* id); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, @@ -68,7 +69,8 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRo STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, SRowKey* pKey, STsdbReader* pReader); -static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); +static int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, + STsdbReader* pReader); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SReadCostSummary* pCost); static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, @@ -126,7 +128,7 @@ static void tColRowGetPriamyKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_ } // for test purpose, todo remove it -static int32_t tGetPrimaryKeyIndex(uint8_t *p, SPrimaryKeyIndex *index) { +static int32_t tGetPrimaryKeyIndex(uint8_t* p, SPrimaryKeyIndex* index) { int32_t n = 0; n += tGetI8(p + n, &index->type); n += tGetU32v(p + n, &index->offset); @@ -137,7 +139,7 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { SPrimaryKeyIndex indices[TD_MAX_PK_COLS]; ASSERT(pKey->numOfPKs <= TD_MAX_PK_COLS); - uint8_t *data = pRow->data; + uint8_t* data = pRow->data; for (int32_t i = 0; i < pRow->numOfPKs; i++) { data += tGetPrimaryKeyIndex(data, &indices[i]); } @@ -146,7 +148,7 @@ static void tRowGetPrimaryKeyDeepCopy(SRow* pRow, SRowKey* pKey) { for (int32_t i = 0; i < pRow->numOfPKs; i++) { pKey->pks[i].type = indices[i].type; - uint8_t *tdata = data + indices[i].offset; + uint8_t* tdata = data + indices[i].offset; if (pRow->flag >> 4) { tdata += tGetI16v(tdata, NULL); } @@ -363,9 +365,7 @@ _err: return code; } -bool shouldFreePkBuf(SBlockLoadSuppInfo *pSupp) { - return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); -} +bool shouldFreePkBuf(SBlockLoadSuppInfo* pSupp) { return (pSupp->numOfPks > 0) && IS_VAR_DATA_TYPE(pSupp->pk.type); } void resetDataBlockIterator(SDataBlockIter* pIter, int32_t order, bool needFree) { pIter->order = order; @@ -1077,10 +1077,10 @@ static void blockInfoToRecord(SBrinRecord* record, SFileDataBlockInfo* pBlockInf pLast->type = pSupp->pk.type; if (IS_VAR_DATA_TYPE(pFirst->type)) { - pFirst->pData = (uint8_t*) varDataVal(pBlockInfo->firstPk.pData); + pFirst->pData = (uint8_t*)varDataVal(pBlockInfo->firstPk.pData); pFirst->nData = varDataLen(pBlockInfo->firstPk.pData); - pLast->pData = (uint8_t*) varDataVal(pBlockInfo->lastPk.pData); + pLast->pData = (uint8_t*)varDataVal(pBlockInfo->lastPk.pData); pLast->nData = varDataLen(pBlockInfo->lastPk.pData); } else { pFirst->val = pBlockInfo->firstPk.val; @@ -1417,7 +1417,8 @@ static int32_t setFileBlockActiveInBlockIter(STsdbReader* pReader, SDataBlockIte } // todo: this attribute could be acquired during extractin the global ordered block list. -static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType, int32_t numOfPk) { +static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* pRec, int32_t order, int32_t pkType, + int32_t numOfPk) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { if (pBlock->lastKey == pRec->firstKey.key.ts) { @@ -1429,7 +1430,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p v1.val = pBlock->lastPk.val; } return (tValueCompare(&v1, &pRec->firstKey.key.pks[0]) == 0); - } else { // no pk + } else { // no pk return true; } } else { @@ -1445,7 +1446,7 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p v1.val = pBlock->firstPk.val; } return (tValueCompare(&v1, &pRec->lastKey.key.pks[0]) == 0); - } else { // no pk + } else { // no pk return true; } } else { @@ -1585,7 +1586,7 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB *copied = false; bool asc = (pReader->info.order == TSDB_ORDER_ASC); if ((pDumpInfo->rowIndex < pDumpInfo->totalRows - 1 && asc) || (pDumpInfo->rowIndex > 0 && (!asc))) { - int32_t step = ASCENDING_TRAVERSE(pReader->info.order)? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pReader->info.order) ? 1 : -1; SRowKey nextRowKey; tColRowGetKey(pBlockData, pDumpInfo->rowIndex + step, &nextRowKey); @@ -2077,7 +2078,8 @@ int32_t doInitMemDataIter(STsdbReader* pReader, STbData** pData, STableBlockScan return code; } -static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { +static void doForwardDataIter(SRowKey* pKey, SIterInfo* pIter, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader) { SRowKey rowKey = {0}; while (1) { @@ -2119,7 +2121,7 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea startKey.key = pBlockScanInfo->lastProcKey; startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer; if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) { - startKey.key.ts = asc? pWindow->skey:pWindow->ekey; + startKey.key.ts = asc ? pWindow->skey : pWindow->ekey; forward = false; } @@ -2168,8 +2170,8 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STable return false; } - if (ts == pBlockScanInfo->lastProcKey.ts) { // todo opt perf - SRowKey nextRowKey; // lazy eval + if (ts == pBlockScanInfo->lastProcKey.ts) { // todo opt perf + SRowKey nextRowKey; // lazy eval tColRowGetKey(pBlockData, rowIndex, &nextRowKey); if (pkCompEx(&pBlockScanInfo->lastProcKey, &nextRowKey) == 0) { return false; @@ -2268,12 +2270,12 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pKeyRangeList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA; - SRowKey* p = asc? &pScanInfo->sttRange.skey:&pScanInfo->sttRange.ekey; + SRowKey* p = asc ? &pScanInfo->sttRange.skey : &pScanInfo->sttRange.ekey; tRowKeyAssign(&pScanInfo->sttKeyInfo.nextProcKey, p); hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); - } else { // not clean stt blocks - INIT_KEYRANGE(&pScanInfo->sttRange); //reset the time window + } else { // not clean stt blocks + INIT_KEYRANGE(&pScanInfo->sttRange); // reset the time window pScanInfo->sttBlockReturned = false; hasData = nextRowFromSttBlocks(pSttBlockReader, pScanInfo, pReader->suppInfo.pkSrcSlot, &pReader->info.verRange); } @@ -2362,8 +2364,9 @@ int32_t mergeRowsInSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanIn TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; if (IS_VAR_DATA_TYPE(pScanInfo->lastProcKey.pks[0].type)) { - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow, pSttBlockReader->uid, - fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData, pReader->idStr); + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " pk:%s %s", pRow->pBlockData, pRow->iRow, + pSttBlockReader->uid, fRow.pBlockData->aTSKEY[fRow.iRow], pScanInfo->lastProcKey.pks[0].pData, + pReader->idStr); } int32_t code = @@ -2513,7 +2516,8 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { return code; } - STableBlockScanInfo* pBlockScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); + STableBlockScanInfo* pBlockScanInfo = + getTableBlockScanInfo(pReader->status.pTableMap, pBlockInfo->uid, pReader->idStr); if (pBlockScanInfo == NULL) { goto _end; } @@ -3560,7 +3564,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S int32_t order = pReader->info.order; TSDBROW* pRow = tsdbTbDataIterGet(pIter->iter); - TSDBKEY key; + TSDBKEY key; TSDBROW_INIT_KEY(pRow, key); if (outOfTimeWindow(key.ts, &pReader->info.window)) { pIter->hasVal = false; @@ -3572,7 +3576,8 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { return pRow; } else { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0); + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, + pReader->suppInfo.numOfPks > 0); if (!dropped) { return pRow; } @@ -3597,7 +3602,8 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S if (pDelList == NULL || TARRAY_SIZE(pDelList) == 0) { return pRow; } else { - bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, pReader->suppInfo.numOfPks > 0); + bool dropped = hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, order, &pReader->info.verRange, + pReader->suppInfo.numOfPks > 0); if (!dropped) { return pRow; } @@ -3606,7 +3612,7 @@ FORCE_INLINE TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, S } } -int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey *pCurKey, SArray* pDelList, STsdbReader* pReader) { +int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, SRowKey* pCurKey, SArray* pDelList, STsdbReader* pReader) { SRowMerger* pMerger = &pReader->status.merger; while (1) { @@ -3698,7 +3704,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn return code; } -int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, STsdbReader* pReader) { +int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, SRowKey* pKey, + STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SRowMerger* pMerger = &pReader->status.merger; bool asc = ASCENDING_TRAVERSE(pReader->info.order); @@ -3707,8 +3714,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc pDumpInfo->rowIndex += step; if ((pDumpInfo->rowIndex <= pBlockData->nRow - 1 && asc) || (pDumpInfo->rowIndex >= 0 && !asc)) { - pDumpInfo->rowIndex = - doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pRange, step); + pDumpInfo->rowIndex = doMergeRowsInFileBlockImpl(pBlockData, pDumpInfo->rowIndex, pKey, pMerger, pRange, step); } // all rows are consumed, let's try next file block @@ -3912,8 +3918,8 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, SRowKey* pRowKey, TSDBROW* piRow, SRow static int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, TSDBROW* pResRow, int64_t endKey, bool* freeTSRow) { - TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); - TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); SArray* pDelList = pBlockScanInfo->delSkyline; uint64_t uid = pBlockScanInfo->uid; @@ -4484,8 +4490,8 @@ static int32_t doSuspendCurrentReader(STsdbReader* pCurrentReader) { while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; clearBlockScanInfo(pInfo); -// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey.ts + step; -// pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step; + // pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey.ts + step; + // pInfo->sttKeyInfo.nextProcKey = pInfo->lastProcKey + step; } pStatus->uidList.currentIndex = 0; @@ -4815,7 +4821,6 @@ int32_t tsdbNextDataBlock2(STsdbReader* pReader, bool* hasNext) { return code; } - if (pReader->step == EXTERNAL_ROWS_MAIN && pReader->innerReader[1] != NULL) { // prepare for the next row scan int32_t step = -1; @@ -4956,7 +4961,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, if (pResBlock->pBlockAgg == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - for(int i = 0; i < num; ++i) { + for (int i = 0; i < num; ++i) { pResBlock->pBlockAgg[i].colId = -1; } } @@ -4976,6 +4981,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, } else if (pAgg->colId < pSup->colId[j]) { i += 1; } else if (pSup->colId[j] < pAgg->colId) { + pResBlock->pBlockAgg[pSup->slotId[j]].colId = -1; *allHave = false; j += 1; } @@ -5034,9 +5040,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { SReaderStatus* pStatus = &pTReader->status; if (pStatus->composedDataBlock || pReader->info.execMode == READER_EXEC_ROWS) { - -// tsdbReaderSuspend2(pReader); -// tsdbReaderResume2(pReader); + // tsdbReaderSuspend2(pReader); + // tsdbReaderResume2(pReader); return pTReader->resBlockInfo.pResBlock; } @@ -5046,8 +5051,8 @@ SSDataBlock* tsdbRetrieveDataBlock2(STsdbReader* pReader, SArray* pIdList) { qTrace("tsdb/read-retrieve: %p, unlock read mutex", pReader); tsdbReleaseReader(pReader); -// tsdbReaderSuspend2(pReader); -// tsdbReaderResume2(pReader); + // tsdbReaderSuspend2(pReader); + // tsdbReaderResume2(pReader); return ret; } @@ -5466,8 +5471,7 @@ void tsdbReaderSetId2(STsdbReader* pReader, const char* idstr) { pReader->status.fileIter.pSttBlockReader->mergeTree.idStr = pReader->idStr; } -void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ -} +void tsdbReaderSetCloseFlag(STsdbReader* pReader) { /*pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED;*/ } void tsdbSetFilesetDelimited(STsdbReader* pReader) { pReader->bFilesetDelimited = true; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f9bb636be3..2264e2779b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1103,6 +1103,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, if (vnodeValidateTableHash(pVnode, tbName) < 0) { cRsp.code = TSDB_CODE_VND_HASH_MISMATCH; taosArrayPush(rsp.pArray, &cRsp); + vError("vgId:%d create-table:%s failed due to hash value mismatch", TD_VID(pVnode), tbName); continue; } diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 8590080fad..a95df06307 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -426,6 +426,7 @@ int32_t ctgdGetClusterCacheNum(SCatalog *pCtg, int32_t type) { break; case CTG_DBG_VIEW_NUM: num += ctgdGetViewNum(dbCache); + break; default: ctgError("invalid type:%d", type); break; diff --git a/source/libs/executor/src/mergejoin.c b/source/libs/executor/src/mergejoin.c index df8c6e2faf..5f0a2eadfb 100755 --- a/source/libs/executor/src/mergejoin.c +++ b/source/libs/executor/src/mergejoin.c @@ -99,7 +99,7 @@ int32_t mWinJoinDumpGrpCache(SMJoinWindowCtx* pCtx) { int32_t buildEndIdx = buildGrp->endIdx; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; - mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp); + MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; rowsLeft = 0; @@ -225,7 +225,7 @@ static int32_t mOuterJoinMergeFullCart(SMJoinMergeCtx* pCtx) { int32_t buildEndIdx = buildGrp->endIdx; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; - mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp); + MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; rowsLeft = 0; @@ -291,7 +291,6 @@ static int32_t mOuterJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; - rowsLeft = 0; break; } @@ -682,7 +681,7 @@ static int32_t mInnerJoinMergeCart(SMJoinMergeCtx* pCtx) { int32_t buildEndIdx = buildGrp->endIdx; buildGrp->endIdx = buildGrp->readIdx + rowsLeft - 1; - mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp); + MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; rowsLeft = 0; @@ -1402,7 +1401,6 @@ static int32_t mSemiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; - rowsLeft = 0; break; } @@ -1712,7 +1710,6 @@ static int32_t mAntiJoinMergeSeqCart(SMJoinMergeCtx* pCtx) { MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->midBlk, true, probeGrp, buildGrp)); buildGrp->readIdx += rowsLeft; buildGrp->endIdx = buildEndIdx; - rowsLeft = 0; break; } @@ -2004,7 +2001,7 @@ int32_t mAsofBackwardDumpGrpCache(SMJoinWindowCtx* pCtx) { } buildGrp.endIdx = buildGrp.readIdx + rowsLeft - 1; - mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, &buildGrp); + MJ_ERR_RET(mJoinMergeGrpCart(pCtx->pJoin, pCtx->finBlk, true, probeGrp, &buildGrp)); pCtx->cache.outRowIdx += rowsLeft; break; } diff --git a/source/libs/executor/src/mergeoperator.c b/source/libs/executor/src/mergeoperator.c index b1fba588ab..993e8c72fd 100755 --- a/source/libs/executor/src/mergeoperator.c +++ b/source/libs/executor/src/mergeoperator.c @@ -182,7 +182,7 @@ SSDataBlock* doSortMerge(SOperatorInfo* pOperator) { resetLimitInfoForNextGroup(&pInfo->limitInfo); } - applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); + (void)applyLimitOffset(&pInfo->limitInfo, p, pTaskInfo); if (p->info.rows > 0) { break; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index cda4b67f97..f7c51c43b5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -3902,17 +3902,19 @@ static int32_t openSubTablesMergeSort(STmsSubTablesMergeInfo* pSubTblsInfo) { if (pInput->rowIdx == -1) { continue; } + if (pInput->type == SUB_TABLE_MEM_BLOCK) { pInput->rowIdx = 0; pInput->pageIdx = -1; } + pInput->pInputBlock = (pInput->type == SUB_TABLE_MEM_BLOCK) ? pInput->pReaderBlock : pInput->pPageBlock; SColumnInfoData* col = taosArrayGet(pInput->pInputBlock->pDataBlock, pSubTblsInfo->pTsOrderInfo->slotId); pInput->aTs = (int64_t*)col->pData; } + __merge_compare_fn_t mergeCompareFn = (!pSubTblsInfo->pPkOrderInfo) ? subTblRowCompareTsFn : subTblRowCompareTsPkFn; - tMergeTreeCreate(&pSubTblsInfo->pTree, pSubTblsInfo->numSubTables, pSubTblsInfo, mergeCompareFn); - return TSDB_CODE_SUCCESS; + return tMergeTreeCreate(&pSubTblsInfo->pTree, pSubTblsInfo->numSubTables, pSubTblsInfo, mergeCompareFn); } static int32_t initSubTablesMergeInfo(STableMergeScanInfo* pInfo) { diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 44f8bd90b0..2bcc7501fb 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -51,7 +51,8 @@ int32_t qCloneCurrentTbData(STableDataCxt* pDataBlock, SSubmitTbData** pData) { return TSDB_CODE_SUCCESS; } -int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo) { +int32_t qAppendStmtTableOutput(SQuery* pQuery, SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, + SStbInterlaceInfo* pBuildInfo) { // merge according to vgId return insAppendStmtTableDataCxt(pAllVgHash, pTbData, pTbCtx, pBuildInfo); } @@ -70,18 +71,16 @@ int32_t qBuildStmtFinOutput(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDat return code; } - /* -int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t tbNum) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* pVgDataBlocks = NULL; - SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; +int32_t qBuildStmtOutputFromTbList(SQuery* pQuery, SHashObj* pVgHash, SArray* pBlockList, STableDataCxt* pTbCtx, int32_t +tbNum) { int32_t code = TSDB_CODE_SUCCESS; SArray* pVgDataBlocks = NULL; SVnodeModifyOpStmt* +pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; // merge according to vgId if (tbNum > 0) { code = insMergeStmtTableDataCxt(pTbCtx, pBlockList, &pVgDataBlocks, true, tbNum); } - + if (TSDB_CODE_SUCCESS == code) { code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks); } @@ -102,7 +101,7 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash if (taosHashGetSize(pBlockHash) > 0) { code = insMergeTableDataCxt(pBlockHash, &pVgDataBlocks, true); } - + if (TSDB_CODE_SUCCESS == code) { code = insBuildVgDataBlocks(pVgHash, pVgDataBlocks, &pStmt->pDataBlocks, false); } @@ -217,6 +216,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, const ch insBuildCreateTbReq(pDataBlock->pData->pCreateTbReq, tName, pTag, suid, sTableName, tagName, pDataBlock->pMeta->tableInfo.numOfTags, TSDB_DEFAULT_TABLE_TTL); + pTag = NULL; end: for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { @@ -227,6 +227,7 @@ end: } taosArrayDestroy(pTagArray); taosArrayDestroy(tagName); + taosMemoryFree(pTag); return code; } @@ -243,7 +244,7 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* if (NULL == dst->length) { dst->length = taosMemoryRealloc(dst->length, sizeof(int32_t) * src->num); - if (NULL == dst->buffer) { + if (NULL == dst->length) { taosMemoryFreeClear(dst->buffer); return TSDB_CODE_OUT_OF_MEMORY; } @@ -276,7 +277,8 @@ int32_t convertStmtNcharCol(SMsgBuf* pMsgBuf, SSchema* pSchema, TAOS_MULTI_BIND* return TSDB_CODE_SUCCESS; } -int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, STSchema** pTSchema, SBindInfo* pBindInfos) { +int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, + STSchema** pTSchema, SBindInfo* pBindInfos) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -293,20 +295,21 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind } for (int c = 0; c < boundInfo->numOfBound; ++c) { - SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; + SSchema* pColSchema = &pSchema[boundInfo->pColIndex[c]]; if (pColSchema->colId <= lastColId) { colInOrder = false; } else { lastColId = pColSchema->colId; } - //SColData* pCol = taosArrayGet(pCols, c); + // SColData* pCol = taosArrayGet(pCols, c); if (bind[c].num != rowNum) { code = buildInvalidOperationMsg(&pBuf, "row number in each bind param should be the same"); goto _return; } - if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type + if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && + bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); goto _return; } @@ -325,10 +328,10 @@ int32_t qBindStmtStbColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind pBindInfos[c].bind = pBind; pBindInfos[c].type = pColSchema->type; - //code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); - //if (code) { - // goto _return; - //} + // code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - + // VARSTR_HEADER_SIZE: -1); if (code) { + // goto _return; + // } } code = tRowBuildFromBind(pBindInfos, boundInfo->numOfBound, colInOrder, *pTSchema, pCols); @@ -362,7 +365,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c goto _return; } - if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type + if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && + bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); goto _return; } @@ -377,7 +381,8 @@ int32_t qBindStmtColsValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, c pBind = bind + c; } - code = tColDataAddValueByBind(pCol, pBind, IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE: -1); + code = tColDataAddValueByBind(pCol, pBind, + IS_VAR_DATA_TYPE(pColSchema->type) ? pColSchema->bytes - VARSTR_HEADER_SIZE : -1); if (code) { goto _return; } @@ -393,9 +398,8 @@ _return: return code; } - -int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, - int32_t rowNum) { +int32_t qBindStmtSingleColValue(void* pBlock, SArray* pCols, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, + int32_t colIdx, int32_t rowNum) { STableDataCxt* pDataBlock = (STableDataCxt*)pBlock; SSchema* pSchema = getTableColumnSchema(pDataBlock->pMeta); SBoundColInfo* boundInfo = &pDataBlock->boundColsInfo; @@ -510,7 +514,7 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD_E** fiel } int32_t qResetStmtColumns(SArray* pCols, bool deepClear) { - int32_t colNum = taosArrayGetSize(pCols); + int32_t colNum = taosArrayGetSize(pCols); for (int32_t i = 0; i < colNum; ++i) { SColData* pCol = (SColData*)taosArrayGet(pCols, i); @@ -524,7 +528,6 @@ int32_t qResetStmtColumns(SArray* pCols, bool deepClear) { return TSDB_CODE_SUCCESS; } - int32_t qResetStmtDataBlock(STableDataCxt* block, bool deepClear) { STableDataCxt* pBlock = (STableDataCxt*)block; int32_t colNum = taosArrayGetSize(pBlock->pData->aCol); @@ -589,7 +592,7 @@ int32_t qCloneStmtDataBlock(STableDataCxt** pDst, STableDataCxt* pSrc, bool rese pNewTb->pCreateTbReq = NULL; pNewTb->aCol = taosArrayDup(pCxt->pData->aCol, NULL); - if (NULL == pNewTb) { + if (NULL == pNewTb->aCol) { insDestroyTableDataCxt(*pDst); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 028866064d..17b5733ff7 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -115,8 +115,7 @@ int32_t insCreateSName(SName* pName, SToken* pTableName, int32_t acctId, const c if (dbName == NULL) { return buildInvalidOperationMsg(pMsgBuf, msg3); } - if (name[0] == '\0') - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4); + if (name[0] == '\0') return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_INVALID_IDENTIFIER_NAME, msg4); code = tNameSetDbName(pName, acctId, dbName, strlen(dbName)); if (code != TSDB_CODE_SUCCESS) { @@ -487,12 +486,12 @@ int insColDataComp(const void* lp, const void* rp) { return 0; } - -int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, int32_t* vgId, STableColsData* pTbData, SName* sname) { +int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, int32_t* vgId, + STableColsData* pTbData, SName* sname) { if (*vgId >= 0 && taosHashGet(pAllVgHash, (const char*)vgId, sizeof(*vgId))) { return TSDB_CODE_SUCCESS; } - + SVgroupInfo vgInfo = {0}; SRequestConnInfo conn = {.pTrans = pBuildInfo->transport, .requestId = pBuildInfo->requestId, @@ -503,23 +502,23 @@ int32_t insTryAddTableVgroupInfo(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuild if (TSDB_CODE_SUCCESS != code) { return code; } - + code = taosHashPut(pAllVgHash, (const char*)&vgInfo.vgId, sizeof(vgInfo.vgId), (char*)&vgInfo, sizeof(vgInfo)); if (TSDB_CODE_SUCCESS != code) { return code; } - - return TSDB_CODE_SUCCESS; + + return TSDB_CODE_SUCCESS; } - -int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, STableColsData* pTbData, uint64_t* uid, int32_t* vgId) { +int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo, STableColsData* pTbData, + uint64_t* uid, int32_t* vgId) { STableVgUid* pTbInfo = NULL; - int32_t code = 0; + int32_t code = 0; if (pTbData->getFromHash) { pTbInfo = (STableVgUid*)tSimpleHashGet(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName)); - } + } if (NULL == pTbInfo) { SName sname; @@ -540,7 +539,7 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo if (TSDB_CODE_SUCCESS != code) { return code; } - + *uid = pTableMeta->uid; *vgId = pTableMeta->vgId; @@ -548,7 +547,7 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo tSimpleHashPut(pBuildInfo->pTableHash, pTbData->tbName, strlen(pTbData->tbName), &tbInfo, sizeof(tbInfo)); code = insTryAddTableVgroupInfo(pAllVgHash, pBuildInfo, vgId, pTbData, &sname); - + taosMemoryFree(pTableMeta); } else { *uid = pTbInfo->uid; @@ -558,7 +557,6 @@ int32_t insGetStmtTableVgUid(SHashObj* pAllVgHash, SStbInterlaceInfo* pBuildInfo return code; } - int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDataBlocks) { int32_t code = TSDB_CODE_SUCCESS; SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pQuery->pRoot; @@ -570,20 +568,19 @@ int32_t qBuildStmtFinOutput1(SQuery* pQuery, SHashObj* pAllVgHash, SArray* pVgDa return code; } - - -int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, SStbInterlaceInfo* pBuildInfo) { - int32_t code = TSDB_CODE_SUCCESS; +int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, STableDataCxt* pTbCtx, + SStbInterlaceInfo* pBuildInfo) { + int32_t code = TSDB_CODE_SUCCESS; uint64_t uid; int32_t vgId; - + pTbCtx->pData->aRowP = pTbData->aCol; - + code = insGetStmtTableVgUid(pAllVgHash, pBuildInfo, pTbData, &uid, &vgId); if (TSDB_CODE_SUCCESS != code) { return code; } - + pTbCtx->pMeta->vgId = vgId; pTbCtx->pMeta->uid = uid; pTbCtx->pData->uid = uid; @@ -611,24 +608,24 @@ int32_t insAppendStmtTableDataCxt(SHashObj* pAllVgHash, STableColsData* pTbData, } else { pVgCxt = *(SVgroupDataCxt**)pp; } - + if (TSDB_CODE_SUCCESS == code) { code = fillVgroupDataCxt(pTbCtx, pVgCxt, false, false); } if (taosArrayGetSize(pVgCxt->pData->aSubmitTbData) >= 20000) { code = qBuildStmtFinOutput1((SQuery*)pBuildInfo->pQuery, pAllVgHash, pBuildInfo->pVgroupList); - //taosArrayClear(pVgCxt->pData->aSubmitTbData); + // taosArrayClear(pVgCxt->pData->aSubmitTbData); tDestroySubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); - //insDestroyVgroupDataCxt(pVgCxt); + // insDestroyVgroupDataCxt(pVgCxt); } return code; } /* -int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, SArray** pVgDataBlocks, bool isRebuild, int32_t tbNum) { - SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); +int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, SArray** pVgDataBlocks, bool isRebuild, +int32_t tbNum) { SHashObj* pVgroupHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); SArray* pVgroupList = taosArrayInit(8, POINTER_BYTES); if (NULL == pVgroupHash || NULL == pVgroupList) { taosHashCleanup(pVgroupHash); @@ -644,7 +641,7 @@ int32_t insMergeStmtTableDataCxt(STableDataCxt* pTableCxt, SArray* pTableList, S pTableCxt->pMeta->uid = pTableCols->uid; pTableCxt->pData->uid = pTableCols->uid; pTableCxt->pData->aCol = pTableCols->aCol; - + SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, 0); if (pCol->nVal <= 0) { continue; @@ -810,7 +807,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, if (taosArrayGetSize(src->pData->aSubmitTbData) <= 0) { continue; } - SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); + SVgDataBlocks* dst = taosMemoryCalloc(1, sizeof(SVgDataBlocks)); if (NULL == dst) { code = TSDB_CODE_OUT_OF_MEMORY; } @@ -875,7 +872,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } char* p = (char*)data; - // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column length | + // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each + // column length | int32_t version = *(int32_t*)data; p += sizeof(int32_t); p += sizeof(int32_t); @@ -890,7 +888,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate p += sizeof(uint64_t); int8_t* fields = p; - if(*fields >= TSDB_DATA_TYPE_MAX || *fields < 0){ + if (*fields >= TSDB_DATA_TYPE_MAX || *fields < 0) { uError("fields type error:%d", *fields); ret = TSDB_CODE_INVALID_PARA; goto end; @@ -911,7 +909,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate goto end; } if (tFields != NULL && numFields > boundInfo->numOfBound) { - if (errstr != NULL) snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound); + if (errstr != NULL) + snprintf(errstr, errstrLen, "numFields:%d bigger than num of bound cols:%d", numFields, boundInfo->numOfBound); ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -920,8 +919,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pColSchema = &pSchema[j]; SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", - pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t))); + if (errstr != NULL) + snprintf(errstr, errstrLen, + "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", + pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, + *(int32_t*)(fields + sizeof(int8_t))); ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -951,8 +953,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SSchema* pColSchema = &pSchema[j]; if (strcmp(pColSchema->name, tFields[i].name) == 0) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { - if (errstr != NULL) snprintf(errstr, errstrLen, "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", - pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, *(int32_t*)(fields + sizeof(int8_t))); + if (errstr != NULL) + snprintf(errstr, errstrLen, + "column type or bytes not equal, name:%s, schema type:%s, bytes:%d, data type:%s, bytes:%d", + pColSchema->name, tDataTypes[pColSchema->type].name, pColSchema->bytes, tDataTypes[*fields].name, + *(int32_t*)(fields + sizeof(int8_t))); ret = TSDB_CODE_INVALID_PARA; goto end; } @@ -986,7 +991,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } } - if(!hasTs){ + if (!hasTs) { if (errstr != NULL) snprintf(errstr, errstrLen, "timestamp column(primary key) not found in raw data"); ret = TSDB_CODE_INVALID_PARA; goto end; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c3075dec78..2f932d8f27 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -8112,9 +8112,11 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, for (int32_t i = 1; i < num; ++i) { SRetention* pRetension = taosArrayGet(dbCfg.pRetensions, i); STranslateContext cxt = {0}; - initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt); - code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2, - 1 == i ? &pReq->ast1Len : &pReq->ast2Len); + code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt); + if (TSDB_CODE_SUCCESS == code) { + code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2, + 1 == i ? &pReq->ast1Len : &pReq->ast2Len); + } destroyTranslateContext(&cxt); if (TSDB_CODE_SUCCESS != code) { break; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index c181997ad2..c2cd07e9ed 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -90,7 +90,6 @@ int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *re QW_RET(TSDB_CODE_SUCCESS); } - int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { qTaskInfo_t taskHandle = ctx->taskHandle; @@ -192,12 +191,12 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { } else { QW_TASK_DLOG("dyn task qExecTask done, useconds:%" PRIu64, useconds); } - + ctx->queryExecDone = true; } dsEndPut(sinkHandle, useconds); - + if (queryStop) { *queryStop = true; } @@ -240,7 +239,6 @@ bool qwTaskNotInExec(SQWTaskCtx *ctx) { return false; } - int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) { int32_t taskNum = 0; @@ -324,7 +322,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, if (!ctx->dynamicTask) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask); } - + if (NULL == pRsp) { QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &pRsp)); *pOutput = output; @@ -351,8 +349,8 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, // set the serialize start position output.pData = pRsp->data + *dataLen - (len + PAYLOAD_PREFIX_LEN); - ((int32_t*) output.pData)[0] = len; - ((int32_t*) output.pData)[1] = rawLen; + ((int32_t *)output.pData)[0] = len; + ((int32_t *)output.pData)[1] = rawLen; output.pData += sizeof(int32_t) * 2; code = dsGetDataBlock(ctx->sinkHandle, &output); @@ -439,8 +437,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes return TSDB_CODE_SUCCESS; } - -int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, int32_t code) { +int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg, int32_t code) { if (QUERY_RSP_POLICY_QUICK == tsQueryRspPolicy && ctx != NULL) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { void *rsp = NULL; @@ -466,13 +463,13 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, i qwMsg->connInfo = ctx->dataConnInfo; QW_SET_EVENT_PROCESSED(ctx, QW_EVENT_FETCH); - + qwBuildAndSendFetchRsp(ctx->fetchMsgType + 1, &qwMsg->connInfo, rsp, dataLen, code); rsp = NULL; - + QW_TASK_DLOG("fetch rsp send, handle:%p, code:%x - %s, dataLen:%d", qwMsg->connInfo.handle, code, tstrerror(code), dataLen); - } + } } return TSDB_CODE_SUCCESS; @@ -487,14 +484,14 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg if (!atomic_val_compare_exchange_8((int8_t*)&ctx->queryEnd, true, false)) { QW_TASK_ELOG("dynamic task prev exec not finished, queryEnd:%d", ctx->queryEnd); return TSDB_CODE_ACTION_IN_PROGRESS; - } + } #else ctx->queryExecDone = false; ctx->queryEnd = false; #endif dsReset(ctx->sinkHandle); - + qUpdateOperatorParam(ctx->taskHandle, qwMsg->msg); QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); @@ -511,7 +508,6 @@ int32_t qwStartDynamicTaskNewExec(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SQWMsg *qwMsg return TSDB_CODE_SUCCESS; } - int32_t qwHandlePrePhaseEvents(QW_FPARAMS_DEF, int8_t phase, SQWPhaseInput *input, SQWPhaseOutput *output) { int32_t code = 0; SQWTaskCtx *ctx = NULL; @@ -747,7 +743,8 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(code); } - code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql, OPTR_EXEC_MODEL_BATCH); + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql, + OPTR_EXEC_MODEL_BATCH); sql = NULL; if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -772,7 +769,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { ctx->queryExecDone = true; ctx->queryEnd = true; } - + _return: taosMemoryFree(sql); @@ -809,7 +806,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (!queryStop) { QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); } - + if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput)); @@ -878,9 +875,9 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { - int32_t code = 0; - int32_t dataLen = 0; - int32_t rawDataLen = 0; + int32_t code = 0; + int32_t dataLen = 0; + int32_t rawDataLen = 0; bool locked = false; SQWTaskCtx *ctx = NULL; @@ -962,8 +959,8 @@ _return: qwFreeFetchRsp(rsp); rsp = NULL; } - } else { - //qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); + } else { + // qwQuickRspFetchReq(QW_FPARAMS(), ctx, qwMsg, code); } QW_RET(TSDB_CODE_SUCCESS); @@ -1005,6 +1002,8 @@ _return: if (ctx) { QW_UPDATE_RSP_CODE(ctx, code); qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_FAIL, ctx->dynamicTask); + } else { + tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER); } } @@ -1013,6 +1012,10 @@ _return: } if (ctx) { + if (qwMsg->connInfo.handle != ctx->ctrlConnInfo.handle) { + tmsgReleaseHandle(&qwMsg->connInfo, TAOS_CONN_SERVER); + } + qwReleaseTaskCtx(mgmt, ctx); } @@ -1040,7 +1043,7 @@ int32_t qwProcessNotify(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (ctx->explain && !ctx->explainRsped) { QW_ERR_RET(qwSendExplainResponse(QW_FPARAMS(), ctx)); } - break; + break; default: QW_ELOG("Invalid task notify type %d", qwMsg->msgType); QW_ERR_JRET(TSDB_CODE_INVALID_MSG); @@ -1067,7 +1070,6 @@ _return: QW_RET(TSDB_CODE_SUCCESS); } - int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 08d0a5e486..4741745419 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -55,10 +55,10 @@ struct SActiveCheckpointInfo { int64_t activeId; // current active checkpoint id int64_t failedId; bool dispatchTrigger; - SArray* pDispatchTriggerList; // SArray - SArray* pReadyMsgList; // SArray + SArray* pDispatchTriggerList; // SArray + SArray* pReadyMsgList; // SArray int8_t allUpstreamTriggerRecv; - SArray* pCheckpointReadyRecvList; // SArray + SArray* pCheckpointReadyRecvList; // SArray int32_t checkCounter; tmr_h pChkptTriggerTmr; int32_t sendReadyCheckCounter; @@ -100,7 +100,7 @@ typedef struct { int32_t upstreamNodeId; int32_t transId; int32_t childId; - SRpcMsg msg; // for mnode checkpoint-source rsp + SRpcMsg msg; // for mnode checkpoint-source rsp int64_t checkpointId; int64_t recvTs; int32_t sendCompleted; diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 1986cc43e9..06773c79e3 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -140,7 +140,7 @@ int32_t valueDecode(void* value, int32_t vlen, int64_t* ttl, char** dest); int32_t valueToString(void* k, char* buf); int32_t valueIsStale(void* k, int64_t ts); -void destroyCompare(void* arg); +void destroyCompare(void* arg); static void cleanDir(const char* pPath, const char* id); static bool streamStateIterSeekAndValid(rocksdb_iterator_t* iter, char* buf, size_t len); @@ -194,9 +194,7 @@ int32_t getCfIdx(const char* cfName) { return idx; } -bool isValidCheckpoint(const char* dir) { - return true; -} +bool isValidCheckpoint(const char* dir) { return true; } int32_t rebuildDirFromCheckpoint(const char* path, int64_t chkpId, char** dst) { // impl later @@ -486,9 +484,7 @@ _ERROR: return code; } -int32_t backendCopyFiles(const char* src, const char* dst) { - return backendFileCopyFilesImpl(src, dst); -} +int32_t backendCopyFiles(const char* src, const char* dst) { return backendFileCopyFilesImpl(src, dst); } static int32_t rebuildFromLocalCheckpoint(const char* pTaskIdStr, const char* checkpointPath, int64_t checkpointId, const char* defaultPath) { @@ -540,7 +536,8 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId char* chkptPath = taosMemoryCalloc(1, pathLen); if (chkptId > 0) { - snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", chkptId); + snprintf(chkptPath, pathLen, "%s%s%s%s%s%" PRId64 "", prefixPath, TD_DIRSEP, "checkpoints", TD_DIRSEP, "checkpoint", + chkptId); code = rebuildFromLocalCheckpoint(key, chkptPath, chkptId, defaultPath); if (code != 0) { @@ -549,11 +546,12 @@ int32_t restoreCheckpointData(const char* path, const char* key, int64_t chkptId if (code != 0) { stError("failed to start stream backend at %s, reason: %s, restart from default defaultPath:%s", chkptPath, - tstrerror(code), defaultPath); - code = 0; // reset the error code + tstrerror(code), defaultPath); + code = 0; // reset the error code } } else { // no valid checkpoint id - stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data, clean defaultPath:%s", key, defaultPath); + stInfo("%s no valid checkpoint ever generated, no need to copy checkpoint data, clean defaultPath:%s", key, + defaultPath); cleanDir(defaultPath, key); } @@ -2143,13 +2141,18 @@ void taskDbDestroy(void* pDb, bool flush) { rocksdb_flushoptions_destroy(flushOpt); } } - for (int i = 0; i < nCf; i++) { - if (wrapper->pCf[i] != NULL) { - rocksdb_column_family_handle_destroy(wrapper->pCf[i]); + + if (wrapper->pCf != NULL) { + for (int i = 0; i < nCf; i++) { + if (wrapper->pCf[i] != NULL) { + rocksdb_column_family_handle_destroy(wrapper->pCf[i]); + } } } - if (wrapper->db) rocksdb_close(wrapper->db); + if (wrapper->db) { + rocksdb_close(wrapper->db); + } rocksdb_options_destroy(wrapper->dbOpt); rocksdb_readoptions_destroy(wrapper->readOpt); @@ -2205,7 +2208,8 @@ int32_t taskDbGenChkpUploadData__rsync(STaskDbWrapper* pDb, int64_t chkpId, char return code; } -int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list, const char* idStr) { +int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64_t chkpId, char** path, SArray* list, + const char* idStr) { int32_t code = 0; SBkdMgt* p = (SBkdMgt*)bkdChkpMgt; @@ -2224,7 +2228,8 @@ int32_t taskDbGenChkpUploadData__s3(STaskDbWrapper* pDb, void* bkdChkpMgt, int64 return code; } -int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list, const char* idStr) { +int32_t taskDbGenChkpUploadData(void* arg, void* mgt, int64_t chkpId, int8_t type, char** path, SArray* list, + const char* idStr) { int32_t code = -1; STaskDbWrapper* pDb = arg; ECHECKPOINT_BACKUP_TYPE utype = type; diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 1728147c11..8778e3314a 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -58,7 +58,8 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ } if (pInfo->stage < stage) { - stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64 ", prev:%" PRId64, + stError("s-task:%s receive check msg from upstream task:0x%x(vgId:%d), new stage received:%" PRId64 + ", prev:%" PRId64, id, upstreamTaskId, vgId, stage, pInfo->stage); // record the checkpoint failure id and sent to mnode taosThreadMutexLock(&pTask->lock); @@ -70,7 +71,6 @@ int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_ } if (pInfo->stage != stage) { - taosThreadMutexLock(&pTask->lock); ETaskStatus status = streamTaskGetStatus(pTask)->state; if (status == TASK_STATUS__CK) { @@ -168,7 +168,8 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS } else { SStreamTask* pTask = streamMetaAcquireTask(pMeta, pReq->streamId, taskId); if (pTask != NULL) { - pRsp->status = streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); + pRsp->status = + streamTaskCheckStatus(pTask, pReq->upstreamTaskId, pReq->upstreamNodeId, pReq->stage, &pRsp->oldStage); SStreamTaskState* pState = streamTaskGetStatus(pTask); stDebug("s-task:%s status:%s, stage:%" PRId64 " recv task check req(reqId:0x%" PRIx64 @@ -183,7 +184,6 @@ void streamTaskProcessCheckMsg(SStreamMeta* pMeta, SStreamTaskCheckReq* pReq, SS pReq->streamId, taskId, pRsp->reqId, pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->status); } } - } int32_t streamTaskProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { @@ -402,7 +402,8 @@ int32_t streamTaskUpdateCheckInfo(STaskCheckInfo* pInfo, int32_t taskId, int32_t SDownstreamStatusInfo* p = findCheckRspStatus(pInfo, taskId); if (p != NULL) { if (reqId != p->reqId) { - stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 " expired check-rsp recv from downstream task:0x%x, discarded", + stError("s-task:%s reqId:0x%" PRIx64 " expected:0x%" PRIx64 + " expired check-rsp recv from downstream task:0x%x, discarded", id, reqId, p->reqId, taskId); taosThreadMutexUnlock(&pInfo->checkInfoLock); return TSDB_CODE_FAILED; @@ -435,7 +436,7 @@ int32_t streamTaskStartCheckDownstream(STaskCheckInfo* pInfo, const char* id) { ASSERT(pInfo->startTs > 0); stError("s-task:%s already in check procedure, checkTs:%" PRId64 ", start monitor check rsp failed", id, pInfo->startTs); - pInfo->stopCheckProcess = 0; // disable auto stop of check process + pInfo->stopCheckProcess = 0; // disable auto stop of check process return TSDB_CODE_FAILED; } @@ -509,8 +510,8 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher; setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId); - stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, - id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); + stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id, + pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); streamSendCheckMsg(pTask, &req, pOutputInfo->fixedDispatcher.nodeId, &pOutputInfo->fixedDispatcher.epSet); } else if (pOutputInfo->type == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -760,4 +761,3 @@ void rspMonitorFn(void* param, void* tmrId) { taosArrayDestroy(pNotReadyList); taosArrayDestroy(pTimeoutList); } - diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index ec8510d9f0..52265963bb 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -24,11 +24,13 @@ static int32_t streamTaskUploadCheckpoint(const char* id, const char* path); static int32_t deleteCheckpoint(const char* id); static int32_t downloadCheckpointByNameS3(const char* id, const char* fname, const char* dstName); static int32_t continueDispatchCheckpointTriggerBlock(SStreamDataBlock* pBlock, SStreamTask* pTask); -static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId); +static int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, + int32_t transId); static int32_t doSendRetrieveTriggerMsg(SStreamTask* pTask, SArray* pNotSendList); static void checkpointTriggerMonitorFn(void* param, void* tmrId); -static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId); +static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, + int32_t transId); SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId) { @@ -96,7 +98,7 @@ int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTri if (pRsp->rspCode != TSDB_CODE_SUCCESS) { stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr, - pRsp->upstreamTaskId, tstrerror(pRsp->rspCode)); + pRsp->upstreamTaskId, tstrerror(pRsp->rspCode)); return TSDB_CODE_SUCCESS; } @@ -108,7 +110,7 @@ int32_t streamTaskSendCheckpointTriggerMsg(SStreamTask* pTask, int32_t dstTaskId SRpcHandleInfo* pRpcInfo, int32_t code) { int32_t size = sizeof(SMsgHead) + sizeof(SCheckpointTriggerRsp); - void* pBuf = rpcMallocCont(size); + void* pBuf = rpcMallocCont(size); SCheckpointTriggerRsp* pRsp = POINTER_SHIFT(pBuf, sizeof(SMsgHead)); ((SMsgHead*)pBuf)->vgId = htonl(downstreamNodeId); @@ -162,15 +164,15 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock taosThreadMutexLock(&pTask->lock); if (pTask->chkInfo.checkpointId > checkpointId) { stError("s-task:%s vgId:%d current checkpointId:%" PRId64 - " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", - id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); + " recv expired checkpoint-trigger block, checkpointId:%" PRId64 " transId:%d, discard", + id, vgId, pTask->chkInfo.checkpointId, checkpointId, transId); taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_SUCCESS; } if (pTask->chkInfo.checkpointId == checkpointId) { { // send checkpoint-ready msg to upstream - SRpcMsg msg ={0}; + SRpcMsg msg = {0}; SStreamUpstreamEpInfo* pInfo = streamTaskGetUpstreamTaskEpInfo(pTask, pBlock->srcTaskId); initCheckpointReadyMsg(pTask, pInfo->nodeId, pBlock->srcTaskId, pInfo->childId, checkpointId, &msg); @@ -362,7 +364,8 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId taosThreadMutexUnlock(&pInfo->lock); if (notReady == 0) { - stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task", id); + stDebug("s-task:%s all downstream task(s) have completed build checkpoint, start to do checkpoint for current task", + id); appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId); } @@ -371,11 +374,11 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId) { SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; - int64_t now = taosGetTimestampMs(); - int32_t numOfConfirmed = 0; + int64_t now = taosGetTimestampMs(); + int32_t numOfConfirmed = 0; taosThreadMutexLock(&pInfo->lock); - for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); if (pReadyInfo->upstreamTaskId == upstreamTaskId && pReadyInfo->checkpointId == checkpointId) { pReadyInfo->sendCompleted = 1; @@ -385,7 +388,7 @@ int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstream } } - for(int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pReadyMsgList); ++i) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pInfo->pReadyMsgList, i); if (pReadyInfo->sendCompleted == 1) { numOfConfirmed += 1; @@ -443,13 +446,14 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, bool restored, SV SStreamTaskState* pStatus = streamTaskGetStatus(pTask); -// if (restored && (pStatus->state != TASK_STATUS__CK)) { -// stDebug("s-task:0x%x vgId:%d restored:%d status:%s not update checkpoint-info, checkpointId:%" PRId64 "->%" PRId64 -// " failed", -// pReq->taskId, vgId, restored, pStatus->name, pInfo->checkpointId, pReq->checkpointId); -// taosThreadMutexUnlock(&pTask->lock); -// return TSDB_CODE_STREAM_TASK_IVLD_STATUS; -// } + // if (restored && (pStatus->state != TASK_STATUS__CK)) { + // stDebug("s-task:0x%x vgId:%d restored:%d status:%s not update checkpoint-info, checkpointId:%" PRId64 "->%" + // PRId64 + // " failed", + // pReq->taskId, vgId, restored, pStatus->name, pInfo->checkpointId, pReq->checkpointId); + // taosThreadMutexUnlock(&pTask->lock); + // return TSDB_CODE_STREAM_TASK_IVLD_STATUS; + // } if (!restored) { // during restore procedure, do update checkpoint-info stDebug("s-task:%s vgId:%d status:%s update the checkpoint-info during restore, checkpointId:%" PRId64 "->%" PRId64 @@ -568,12 +572,12 @@ static int32_t getCheckpointDataMeta(const char* id, const char* path, SArray* l } int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t dbRefId, ECHECKPOINT_BACKUP_TYPE type) { - char* path = NULL; - int32_t code = 0; - SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); - int64_t now = taosGetTimestampMs(); - SStreamMeta* pMeta = pTask->pMeta; - const char* idStr = pTask->id.idStr; + char* path = NULL; + int32_t code = 0; + SArray* toDelFiles = taosArrayInit(4, POINTER_BYTES); + int64_t now = taosGetTimestampMs(); + SStreamMeta* pMeta = pTask->pMeta; + const char* idStr = pTask->id.idStr; if ((code = taskDbGenChkpUploadData(pTask->pBackend, pMeta->bkdChkptMgt, checkpointId, type, &path, toDelFiles, pTask->id.idStr)) != 0) { @@ -619,8 +623,8 @@ int32_t uploadCheckpointData(SStreamTask* pTask, int64_t checkpointId, int64_t d idStr, checkpointId, el, path); taosRemoveDir(path); } else { - stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", - idStr, checkpointId, el); + stDebug("s-task:%s failed to upload checkpointId:%" PRId64 " keep local checkpoint data, elapsed time:%.2fs", idStr, + checkpointId, el); } taosMemoryFree(path); @@ -639,9 +643,10 @@ int32_t streamTaskRemoteBackupCheckpoint(SStreamTask* pTask, int64_t checkpointI } int64_t dbRefId = taskGetDBRef(pTask->pBackend); - void* pBackend = taskAcquireDb(dbRefId); + void* pBackend = taskAcquireDb(dbRefId); if (pBackend == NULL) { - stError("s-task:%s failed to acquire db during update checkpoint data, failed to upload checkpointData", pTask->id.idStr); + stError("s-task:%s failed to acquire db during update checkpoint data, failed to upload checkpointData", + pTask->id.idStr); return -1; } @@ -773,11 +778,11 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { ASSERT(pTask->info.taskLevel > TASK_LEVEL__SOURCE); SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); - for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { SStreamUpstreamEpInfo* pInfo = taosArrayGetP(pList, i); bool recved = false; - for(int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) { + for (int32_t j = 0; j < taosArrayGetSize(pActiveInfo->pReadyMsgList); ++j) { STaskCheckpointReadyInfo* pReady = taosArrayGet(pActiveInfo->pReadyMsgList, j); if (pInfo->nodeId == pReady->upstreamNodeId) { recved = true; @@ -785,7 +790,7 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { } } - if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message + if (!recved) { // make sure the inputQ is opened for not recv upstream checkpoint-trigger message streamTaskOpenUpstreamInput(pTask, pInfo->taskId); taosArrayPush(pNotSendList, pInfo); } @@ -870,7 +875,7 @@ bool streamTaskAlreadySendTrigger(SStreamTask* pTask, int32_t downstreamNodeId) return false; } - for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* pSendInfo = taosArrayGet(pInfo->pDispatchTriggerList, i); if (pSendInfo->nodeId != downstreamNodeId) { continue; @@ -939,10 +944,10 @@ int32_t streamTaskGetNumOfConfirmed(SStreamTask* pTask) { int32_t num = 0; taosThreadMutexLock(&pInfo->lock); - for(int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); if (p->recved) { - num ++; + num++; } } taosThreadMutexUnlock(&pInfo->lock); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 0f5559df89..5595085197 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -65,7 +65,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas return TSDB_CODE_SUCCESS; } -void streamTaskSendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp){ +void streamTaskSendRetrieveRsp(SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp)); ((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId); @@ -119,7 +119,6 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r return code; } - buf = NULL; stDebug("s-task:%s (child %d) send retrieve req to task:0x%x (vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, pTask->info.selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req->reqId); } @@ -127,7 +126,7 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r return code; } -static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req){ +static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req) { SRetrieveTableRsp* pRetrieve = NULL; int32_t len = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; @@ -147,7 +146,7 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data+ PAYLOAD_PREFIX_LEN, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; @@ -165,8 +164,8 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock) { SStreamRetrieveReq req; - int32_t code = buildStreamRetrieveReq(pTask, pBlock, &req); - if(code != 0){ + int32_t code = buildStreamRetrieveReq(pTask, pBlock, &req); + if (code != 0) { return code; } @@ -310,7 +309,6 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD addDispatchEntry(&pTask->msgInfo, pTask->outputInfo.fixedDispatcher.nodeId, now, true); pTask->msgInfo.pData = pReqs; } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgroups = taosArrayGetSize(vgInfo); @@ -568,8 +566,8 @@ void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { } } -int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, - int64_t groupId, int64_t now) { +int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int64_t groupId, + int64_t now) { uint32_t hashValue = 0; SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; if (pTask->pNameMap == NULL) { @@ -656,9 +654,7 @@ static void clearDispatchInfo(SDispatchMsgInfo* pInfo) { pInfo->rspTs = -1; } -static void updateDispatchInfo(SDispatchMsgInfo* pInfo, int64_t recvTs) { - pInfo->rspTs = recvTs; -} +static void updateDispatchInfo(SDispatchMsgInfo* pInfo, int64_t recvTs) { pInfo->rspTs = recvTs; } int32_t streamDispatchStreamBlock(SStreamTask* pTask) { ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || @@ -719,8 +715,8 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { taosThreadMutexLock(&pTask->msgInfo.lock); if (pTask->msgInfo.inMonitor == 0) { int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS, ref, - tstrerror(code)); + stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS, + ref, tstrerror(code)); streamStartMonitorDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); pTask->msgInfo.inMonitor = 1; } else { @@ -829,13 +825,13 @@ static void checkpointReadyMsgSendMonitorFn(void* param, void* tmrId) { int32_t checkpointId = pActiveInfo->activeId; int32_t notRsp = taosArrayGetSize(pNotRspList); - if (notRsp > 0) { // send checkpoint-ready msg again + if (notRsp > 0) { // send checkpoint-ready msg again for (int32_t i = 0; i < taosArrayGetSize(pNotRspList); ++i) { int32_t taskId = *(int32_t*)taosArrayGet(pNotRspList, i); for (int32_t j = 0; j < num; ++j) { STaskCheckpointReadyInfo* pReadyInfo = taosArrayGet(pList, j); - if (taskId == pReadyInfo->upstreamTaskId) { // send msg again + if (taskId == pReadyInfo->upstreamTaskId) { // send msg again SRpcMsg msg = {0}; initCheckpointReadyMsg(pTask, pReadyInfo->upstreamNodeId, pReadyInfo->upstreamTaskId, pReadyInfo->childId, @@ -880,7 +876,8 @@ int32_t streamTaskSendCheckpointReadyMsg(SStreamTask* pTask) { STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, i); SRpcMsg msg = {0}; - initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, &msg); + initCheckpointReadyMsg(pTask, pInfo->upstreamNodeId, pInfo->upstreamTaskId, pInfo->childId, pInfo->checkpointId, + &msg); tmsgSendReq(&pInfo->upstreamNodeEpset, &msg); stDebug("s-task:%s level:%d checkpoint-ready msg sent to upstream:0x%x", id, pTask->info.taskLevel, @@ -918,9 +915,11 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { tmsgSendRsp(&pInfo->msg); taosArrayClear(pList); - stDebug("s-task:%s level:%d checkpoint-source rsp completed msg sent to mnode", pTask->id.idStr, pTask->info.taskLevel); + stDebug("s-task:%s level:%d checkpoint-source rsp completed msg sent to mnode", pTask->id.idStr, + pTask->info.taskLevel); } else { - stDebug("s-task:%s level:%d already send checkpoint-source rsp success to mnode", pTask->id.idStr, pTask->info.taskLevel); + stDebug("s-task:%s level:%d already send checkpoint-source rsp success to mnode", pTask->id.idStr, + pTask->info.taskLevel); } taosThreadMutexUnlock(&pTask->chkInfo.pActiveInfo->lock); @@ -1010,7 +1009,7 @@ FAIL: } int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, - int32_t setCode) { + int32_t setCode) { int32_t len = 0; int32_t code = 0; SEncoder encoder; @@ -1049,7 +1048,8 @@ int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRp return 0; } -int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask) { +int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, + SStreamTask* pTask) { STaskCheckpointReadyInfo info = { .recvTs = taosGetTimestampMs(), .transId = pReq->transId, .checkpointId = pReq->checkpointId}; @@ -1070,7 +1070,7 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa stError("s-task:%s checkpointId:%" PRId64 " transId:%d not completed, new transId:%d checkpointId:%" PRId64 " recv from mnode", pTask->id.idStr, pReady->checkpointId, pReady->transId, pReq->transId, pReq->checkpointId); - ASSERT(0); // failed to handle it + ASSERT(0); // failed to handle it } } else { taosArrayPush(pActiveInfo->pReadyMsgList, &info); @@ -1105,8 +1105,7 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId, STaskCheckpointReadyInfo info = {0}; initCheckpointReadyInfo(&info, pInfo->nodeId, pInfo->taskId, pInfo->childId, &pInfo->epSet, checkpointId); - stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 - "-0x%x (vgId:%d) idx:%d", + stDebug("s-task:%s (level:%d) prepare checkpoint-ready msg to upstream s-task:0x%" PRIx64 "-0x%x (vgId:%d) idx:%d", pTask->id.idStr, pTask->info.taskLevel, pTask->id.streamId, pInfo->taskId, pInfo->nodeId, index); SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; @@ -1178,7 +1177,7 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3 bool updated = false; taosThreadMutexLock(&pMsgInfo->lock); - for(int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { + for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); if (pEntry->nodeId == vgId) { ASSERT(!alreadySet); @@ -1186,7 +1185,7 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3 pEntry->status = code; alreadySet = true; updated = true; - stDebug("s-task:%s record the rsp recv, ts:%"PRId64" code:%d, idx:%d", id, now, code, j); + stDebug("s-task:%s record the rsp recv, ts:%" PRId64 " code:%d, idx:%d", id, now, code, j); } if (pEntry->rspTs != -1) { @@ -1301,12 +1300,14 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { if (notRsp > 0) { stDebug( - "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, waiting " + "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, " + "waiting " "for %d rsp", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), notRsp); } else { stDebug( - "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all rsp", + "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all " + "rsp", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code)); } } else { @@ -1407,7 +1408,7 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S if (pReq->stage > pInfo->stage) { // upstream task has restarted/leader-follower switch/transferred to other dnodes stError("s-task:%s upstream task:0x%x (vgId:%d) has restart/leader-switch/vnode-transfer, prev stage:%" PRId64 - ", current:%" PRId64 " dispatch msg rejected", + ", current:%" PRId64 " dispatch msg rejected", id, pReq->upstreamTaskId, pReq->upstreamNodeId, pInfo->stage, pReq->stage); status = TASK_INPUT_STATUS__REFUSED; } else { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 03c7b93f91..a6ecd7f648 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -151,7 +151,7 @@ int32_t streamMetaCheckBackendCompatible(SStreamMeta* pMeta) { int8_t ret = STREAM_STATA_COMPATIBLE; TBC* pCur = NULL; - if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream + if (tdbTbcOpen(pMeta->pTaskDb, &pCur, NULL) < 0) { // no task info, no stream return ret; } @@ -557,7 +557,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { return -1; } - if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->ver < SSTREAM_TASK_SUBTABLE_CHANGED_VER) { pTask->ver = SSTREAM_TASK_VER; } @@ -907,7 +907,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { if (p == NULL) { code = pMeta->buildTaskFn(pMeta->ahandle, pTask, pTask->chkInfo.checkpointVer + 1); if (code < 0) { - stError("failed to expand s-task:0x%"PRIx64", code:%s, continue", id.taskId, tstrerror(terrno)); + stError("failed to expand s-task:0x%" PRIx64 ", code:%s, continue", id.taskId, tstrerror(terrno)); tFreeStreamTask(pTask); continue; } @@ -1012,7 +1012,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { for (int32_t i = 0; i < numOfTasks; ++i) { SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); - STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (pTask == NULL) { continue; @@ -1052,12 +1052,14 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { } if ((*pTask)->chkInfo.pActiveInfo->activeId != 0) { - entry.checkpointInfo.failed = ((*pTask)->chkInfo.pActiveInfo->failedId >= (*pTask)->chkInfo.pActiveInfo->activeId) ? 1 : 0; + entry.checkpointInfo.failed = + ((*pTask)->chkInfo.pActiveInfo->failedId >= (*pTask)->chkInfo.pActiveInfo->activeId) ? 1 : 0; entry.checkpointInfo.activeId = (*pTask)->chkInfo.pActiveInfo->activeId; entry.checkpointInfo.activeTransId = (*pTask)->chkInfo.pActiveInfo->transId; if (entry.checkpointInfo.failed) { - stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr, (*pTask)->chkInfo.pActiveInfo->transId); + stInfo("s-task:%s set kill checkpoint trans in hb, transId:%d", (*pTask)->id.idStr, + (*pTask)->chkInfo.pActiveInfo->transId); } } @@ -1384,7 +1386,7 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta, __stream_task_expand_fn expa int64_t now = taosGetTimestampMs(); int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); - stInfo("vgId:%d start to check all %d stream task(s) downstream status, start ts:%"PRId64, vgId, numOfTasks, now); + stInfo("vgId:%d start to check all %d stream task(s) downstream status, start ts:%" PRId64, vgId, numOfTasks, now); if (numOfTasks == 0) { stInfo("vgId:%d no tasks to be started", pMeta->vgId); @@ -1513,8 +1515,8 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta) { int32_t num = taosArrayGetSize(pMeta->pTaskList); for (int32_t i = 0; i < num; ++i) { SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); - STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; - SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL) { continue; } @@ -1598,7 +1600,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3 if (pStartInfo->startAllTasks != 1) { int64_t el = endTs - startTs; stDebug("vgId:%d not start all task(s), not record status, s-task:0x%x launch succ:%d elapsed time:%" PRId64 "ms", - pMeta->vgId, taskId, ready, el); + pMeta->vgId, taskId, ready, el); streamMetaWUnLock(pMeta); return 0; } @@ -1725,7 +1727,8 @@ void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SSt taosHashPut(pMeta->updateInfo.pTasks, &hEntry, sizeof(hEntry), NULL, 0); stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask/hTask closed, elapsed:%" PRId64 - " ms", id, vgId, transId, el); + " ms", + id, vgId, transId, el); } else { stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", id, vgId, transId, el); diff --git a/source/libs/stream/src/streamSched.c b/source/libs/stream/src/streamSched.c index 9c817d565b..0a54dfa4c8 100644 --- a/source/libs/stream/src/streamSched.c +++ b/source/libs/stream/src/streamSched.c @@ -26,7 +26,8 @@ int32_t streamSetupScheduleTrigger(SStreamTask* pTask) { stDebug("s-task:%s setup scheduler trigger, delay:%" PRId64 " ms", pTask->id.idStr, pTask->info.delaySchedParam); - pTask->schedInfo.pDelayTimer = taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer); + pTask->schedInfo.pDelayTimer = + taosTmrStart(streamTaskSchedHelper, (int32_t)pTask->info.delaySchedParam, pTask, streamTimer); pTask->schedInfo.status = TASK_TRIGGER_STATUS__INACTIVE; } @@ -78,7 +79,7 @@ int32_t streamTaskResumeInFuture(SStreamTask* pTask) { pTask->status.schedIdleTime, ref); // add one ref count for task - /*SStreamTask* pAddRefTask = */streamMetaAcquireOneTask(pTask); + /*SStreamTask* pAddRefTask = */ streamMetaAcquireOneTask(pTask); if (pTask->schedInfo.pIdleTimer == NULL) { pTask->schedInfo.pIdleTimer = taosTmrStart(streamTaskResumeHelper, pTask->status.schedIdleTime, pTask, streamTimer); diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 7a723647ab..30bccd6479 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -105,9 +105,11 @@ int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration) stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks * 0.1, ref); if (pTask->schedHistoryInfo.pTimer == NULL) { - pTask->schedHistoryInfo.pTimer = taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); + pTask->schedHistoryInfo.pTimer = + taosTmrStart(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer); } else { - taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); + taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer); } return TSDB_CODE_SUCCESS; @@ -204,7 +206,8 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { // check stream task status in the first place. SStreamTaskState* pStatus = streamTaskGetStatus(pTask); - if (pStatus->state != TASK_STATUS__READY && pStatus->state != TASK_STATUS__HALT && pStatus->state != TASK_STATUS__PAUSE) { + if (pStatus->state != TASK_STATUS__READY && pStatus->state != TASK_STATUS__HALT && + pStatus->state != TASK_STATUS__PAUSE) { stDebug("s-task:%s not launch related fill-history task:0x%" PRIx64 "-0x%x, status:%s", idStr, hStreamId, hTaskId, pStatus->name); @@ -389,7 +392,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { // not in timer anymore int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:0x%x fill-history task launch completed, retry times:%d, ref:%d", (int32_t)pInfo->id.taskId, - pHTaskInfo->retryTimes, ref); + pHTaskInfo->retryTimes, ref); } streamMetaReleaseTask(pMeta, pHTask); } @@ -434,7 +437,7 @@ int32_t launchNotBuiltFillHistoryTask(SStreamTask* pTask) { stWarn("s-task:%s vgId:%d failed to launch history task:0x%x, since not built yet", idStr, pMeta->vgId, hTaskId); - STaskId id = streamTaskGetTaskId(pTask); + STaskId id = streamTaskGetTaskId(pTask); SLaunchHTaskInfo* pInfo = createHTaskLaunchInfo(pMeta, &id, hStreamId, hTaskId); if (pInfo == NULL) { stError("s-task:%s failed to launch related fill-history task, since Out Of Memory", idStr); @@ -494,8 +497,8 @@ bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t nextProcessVe // [pTask->dataRange.range.maxVer, ver1] pTask->step2Range.minVer = walScanStartVer; pTask->step2Range.maxVer = nextProcessVer - 1; - stDebug("s-task:%s set step2 verRange:%" PRId64 "-%" PRId64 ", step1 verRange:%" PRId64 "-%" PRId64, pTask->id.idStr, - pTask->step2Range.minVer, pTask->step2Range.maxVer, pRange->minVer, pRange->maxVer); + stDebug("s-task:%s set step2 verRange:%" PRId64 "-%" PRId64 ", step1 verRange:%" PRId64 "-%" PRId64, + pTask->id.idStr, pTask->step2Range.minVer, pTask->step2Range.maxVer, pRange->minVer, pRange->maxVer); return false; } } @@ -525,7 +528,7 @@ void streamTaskSetRangeStreamCalc(SStreamTask* pTask) { pRange->range.maxVer); SVersionRange verRange = pRange->range; - STimeWindow win = pRange->window; + STimeWindow win = pRange->window; streamSetParamForStreamScannerStep2(pTask, &verRange, &win); } } @@ -553,7 +556,8 @@ void doExecScanhistoryInFuture(void* param, void* tmrId) { // release the task. streamMetaReleaseTask(pTask->pMeta, pTask); } else { - taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, &pTask->schedHistoryInfo.pTimer); + taosTmrReset(doExecScanhistoryInFuture, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamTimer, + &pTask->schedHistoryInfo.pTimer); } } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 70e3790209..8abac4ce85 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -121,7 +121,7 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, SEpSet* pEpset, pTask->id.idStr = taosStrdup(buf); pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; - pTask->status.taskStatus = fillHistory? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY; + pTask->status.taskStatus = fillHistory ? TASK_STATUS__SCAN_HISTORY : TASK_STATUS__READY; pTask->inputq.status = TASK_INPUT_STATUS__NORMAL; pTask->outputq.status = TASK_OUTPUT_STATUS__NORMAL; @@ -208,9 +208,8 @@ void tFreeStreamTask(SStreamTask* pTask) { stDebug("s-task:0x%x task exec summary: create:%" PRId64 ", init:%" PRId64 ", start:%" PRId64 ", updateCount:%d latestUpdate:%" PRId64 ", latestCheckPoint:%" PRId64 ", ver:%" PRId64 " nextProcessVer:%" PRId64 ", checkpointCount:%d", - taskId, pStatis->created, pStatis->checkTs, pStatis->readyTs, pStatis->updateCount, - pStatis->latestUpdateTs, pCkInfo->checkpointId, pCkInfo->checkpointVer, pCkInfo->nextProcessVer, - pStatis->checkpoint); + taskId, pStatis->created, pStatis->checkTs, pStatis->readyTs, pStatis->updateCount, pStatis->latestUpdateTs, + pCkInfo->checkpointId, pCkInfo->checkpointVer, pCkInfo->nextProcessVer, pStatis->checkpoint); // remove the ref by timer while (pTask->status.timerActive > 0) { @@ -224,12 +223,12 @@ void tFreeStreamTask(SStreamTask* pTask) { } if (pTask->hTaskInfo.pTimer != NULL) { - /*bool ret = */taosTmrStop(pTask->hTaskInfo.pTimer); + /*bool ret = */ taosTmrStop(pTask->hTaskInfo.pTimer); pTask->hTaskInfo.pTimer = NULL; } if (pTask->msgInfo.pRetryTmr != NULL) { - /*bool ret = */taosTmrStop(pTask->msgInfo.pRetryTmr); + /*bool ret = */ taosTmrStop(pTask->msgInfo.pRetryTmr); pTask->msgInfo.pRetryTmr = NULL; } @@ -414,7 +413,8 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i streamTaskInitTokenBucket(pOutputInfo->pTokenBucket, 35, 35, tsSinkDataRate, pTask->id.idStr); pOutputInfo->pNodeEpsetUpdateList = taosArrayInit(4, sizeof(SDownstreamTaskEpset)); if (pOutputInfo->pNodeEpsetUpdateList == NULL) { - stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + stError("s-task:%s failed to prepare downstreamUpdateList, code:%s", pTask->id.idStr, + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); return TSDB_CODE_OUT_OF_MEMORY; } @@ -448,9 +448,7 @@ int32_t streamTaskGetNumOfDownstream(const SStreamTask* pTask) { } } -int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask) { - return taosArrayGetSize(pTask->upstreamInfo.pList); -} +int32_t streamTaskGetNumOfUpstream(const SStreamTask* pTask) { return taosArrayGetSize(pTask->upstreamInfo.pList); } int32_t streamTaskSetUpstreamInfo(SStreamTask* pTask, const SStreamTask* pUpstreamTask) { SStreamUpstreamEpInfo* pEpInfo = createStreamTaskEpInfo(pUpstreamTask); @@ -512,7 +510,7 @@ void streamTaskSetFixedDownstreamInfo(SStreamTask* pTask, const SStreamTask* pDo pTask->msgInfo.msgType = TDMT_STREAM_TASK_DISPATCH; } -void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool *pUpdated) { +void streamTaskUpdateDownstreamInfo(SStreamTask* pTask, int32_t nodeId, const SEpSet* pEpSet, bool* pUpdated) { char buf[512] = {0}; epsetToStr(pEpSet, buf, tListLen(buf)); @@ -740,10 +738,10 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI } int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) { - int32_t code; - int32_t tlen = 0; - int32_t vgId = pTask->pMeta->vgId; - const char* id = pTask->id.idStr; + int32_t code; + int32_t tlen = 0; + int32_t vgId = pTask->pMeta->vgId; + const char* id = pTask->id.idStr; SActiveCheckpointInfo* pActive = pCheckpointInfo->pActiveInfo; SCheckpointReport req = {.streamId = pTask->id.streamId, @@ -862,7 +860,8 @@ void streamTaskResume(SStreamTask* pTask) { int32_t num = atomic_sub_fetch_32(&pMeta->numOfPausedTasks, 1); stInfo("s-task:%s status:%s resume from %s, paused task(s):%d", pTask->id.idStr, pNew, prevState.name, num); } else { - stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name, pMeta->numOfPausedTasks); + stInfo("s-task:%s status:%s no need to resume, paused task(s):%d", pTask->id.idStr, prevState.name, + pMeta->numOfPausedTasks); } } @@ -925,7 +924,7 @@ SEpSet* streamTaskGetDownstreamEpInfo(SStreamTask* pTask, int32_t taskId) { } } else if (pTask->info.taskLevel == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* pList = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; - for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { SVgroupInfo* pVgInfo = taosArrayGet(pList, i); if (pVgInfo->taskId == taskId) { return &pVgInfo->epSet; @@ -979,9 +978,7 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) { return streamTrySchedExec(pTask); } -void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { - pTask->status.removeBackendFiles = true; -} +void streamTaskSetRemoveBackendFiles(SStreamTask* pTask) { pTask->status.removeBackendFiles = true; } int32_t streamTaskGetActiveCheckpointInfo(const SStreamTask* pTask, int32_t* pTransId, int64_t* pCheckpointId) { if (pTransId != NULL) { diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 75d62ff324..5e991593e6 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -200,7 +200,6 @@ static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent eve SStreamTaskSM* pSM = pTask->status.pSM; bool removed = false; - taosThreadMutexLock(&pTask->lock); int32_t num = taosArrayGetSize(pSM->pWaitingEventList); for (int32_t i = 0; i < num; ++i) { @@ -218,7 +217,6 @@ static int32_t removeEventInWaitingList(SStreamTask* pTask, EStreamTaskEvent eve stDebug("s-task:%s failed to remove event:%s in waiting list", pTask->id.idStr, StreamTaskEventList[event].name); } - taosThreadMutexUnlock(&pTask->lock); return TSDB_CODE_SUCCESS; } @@ -671,4 +669,4 @@ void doInitStateTransferTable(void) { trans = createStateTransform(TASK_STATUS__CK, TASK_STATUS__DROPPING, TASK_EVENT_DROPPING, NULL, NULL, NULL); taosArrayPush(streamTaskSMTrans, &trans); } -//clang-format on \ No newline at end of file +//clang-format on diff --git a/source/libs/stream/src/streammsg.c b/source/libs/stream/src/streammsg.c index a6ab6a60c2..ac67742ef5 100644 --- a/source/libs/stream/src/streammsg.c +++ b/source/libs/stream/src/streammsg.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "os.h" #include "streammsg.h" +#include "os.h" #include "tstream.h" int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamUpstreamEpInfo* pInfo) { @@ -77,7 +77,6 @@ int32_t tEncodeStreamCheckpointSourceRsp(SEncoder* pEncoder, const SStreamCheckp return pEncoder->pos; } - int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpdateMsg* pMsg) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pMsg->streamId) < 0) return -1; @@ -99,7 +98,6 @@ int32_t tEncodeStreamTaskUpdateMsg(SEncoder* pEncoder, const SStreamTaskNodeUpda return pEncoder->pos; } - int32_t tDecodeStreamTaskUpdateMsg(SDecoder* pDecoder, SStreamTaskNodeUpdateMsg* pMsg) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pMsg->streamId) < 0) return -1; @@ -180,7 +178,6 @@ int32_t tDecodeStreamTaskCheckRsp(SDecoder* pDecoder, SStreamTaskCheckRsp* pRsp) return 0; } - int32_t tEncodeStreamCheckpointReadyMsg(SEncoder* pEncoder, const SStreamCheckpointReadyMsg* pReq) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; @@ -589,7 +586,7 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tDecodeCStrTo(pDecoder, pTask->outputInfo.shuffleDispatcher.stbFullName) < 0) return -1; } if (tDecodeI64(pDecoder, &pTask->info.delaySchedParam) < 0) return -1; - if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER) { if (tDecodeI8(pDecoder, &pTask->subtableWithoutMd5) < 0) return -1; } if (tDecodeCStrTo(pDecoder, pTask->reserve) < 0) return -1; diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 1d5074fb03..4cca63a966 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -351,7 +351,12 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL } pCell = tdbPageGetCell(btc.pPage, btc.idx); - tdbBtreeDecodeCell(btc.pPage, pCell, &cd, btc.pTxn, pBt); + ret = tdbBtreeDecodeCell(btc.pPage, pCell, &cd, btc.pTxn, pBt); + if (ret < 0) { + tdbBtcClose(&btc); + tdbError("tdb/btree-pget: decode cell failed with ret: %d.", ret); + return -1; + } if (ppKey) { pTKey = tdbRealloc(*ppKey, cd.kLen); @@ -1072,6 +1077,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // pack partial val to local if any space left if (nLocal > nHeader + kLen + sizeof(SPgno)) { if (ASSERT(pVal != NULL && vLen != 0)) { + tdbFree(pBuf); return -1; } memcpy(pCell + nHeader + kLen, pVal, nLocal - nHeader - kLen - sizeof(SPgno)); @@ -1859,7 +1865,11 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); - tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt); + ret = tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt); + if (ret < 0) { + tdbError("tdb/btree-next: decode cell failed with ret: %d.", ret); + return -1; + } pKey = tdbRealloc(*ppKey, cd.kLen); if (pKey == NULL) { @@ -1918,7 +1928,11 @@ int tdbBtreePrev(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); - tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt); + ret = tdbBtreeDecodeCell(pBtc->pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt); + if (ret < 0) { + tdbError("tdb/btree-prev: decode cell failed with ret: %d.", ret); + return -1; + } pKey = tdbRealloc(*ppKey, cd.kLen); if (pKey == NULL) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7bd746837f..ffcb1fbdb5 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -672,6 +672,7 @@ static SCliConn* getConnFromPool(SCliThrd* pThrd, char* key, bool* exceed) { SCliConn* conn = QUEUE_DATA(h, SCliConn, q); conn->status = ConnNormal; QUEUE_INIT(&conn->q); + tDebug("conn %p get from pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr); if (conn->task != NULL) { transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); @@ -754,6 +755,7 @@ static SCliConn* getConnFromPool2(SCliThrd* pThrd, char* key, SCliMsg** pMsg) { SCliConn* conn = QUEUE_DATA(h, SCliConn, q); conn->status = ConnNormal; QUEUE_INIT(&conn->q); + tDebug("conn %p get from pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr); if (conn->task != NULL) { transDQCancel(((SCliThrd*)conn->hostThrd)->timeoutQueue, conn->task); @@ -806,6 +808,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { conn->status = ConnInPool; QUEUE_PUSH(&conn->list->conns, &conn->q); conn->list->size += 1; + tDebug("conn %p added to pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr); if (conn->list->size >= 10) { STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); @@ -825,6 +828,9 @@ static int32_t allocConnRef(SCliConn* conn, bool update) { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); exh->handle = conn; exh->pThrd = conn->hostThrd; + QUEUE_INIT(&exh->q); + taosInitRWLatch(&exh->latch); + exh->refId = transAddExHandle(transGetRefMgt(), exh); QUEUE_INIT(&exh->q); taosInitRWLatch(&exh->latch); @@ -849,6 +855,8 @@ static int32_t specifyConnRef(SCliConn* conn, bool update, int64_t handle) { taosWLockLatch(&exh->latch); exh->handle = conn; exh->pThrd = conn->hostThrd; + taosWUnLockLatch(&exh->latch); + conn->refId = exh->refId; taosWUnLockLatch(&exh->latch); @@ -911,13 +919,11 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { uv_timer_init(pThrd->loop, timer); } timer->data = conn; - conn->timer = timer; + conn->timer = timer; conn->connReq.data = conn; transReqQueueInit(&conn->wreqQueue); - transQueueInit(&conn->cliMsgs, NULL); - transInitBuffer(&conn->readBuf); QUEUE_INIT(&conn->q); conn->hostThrd = pThrd; @@ -1027,7 +1033,7 @@ static void cliSendCb(uv_write_t* req, int status) { SCliConn* pConn = transReqQueueRemove(req); if (pConn == NULL) return; - SCliMsg* pMsg = !transQueueEmpty(&pConn->cliMsgs) ? transQueueGet(&pConn->cliMsgs, 0) : NULL; + SCliMsg* pMsg = transQueueGet(&pConn->cliMsgs, 0); if (pMsg != NULL) { int64_t cost = taosGetTimestampUs() - pMsg->st; if (cost > 1000 * 50) { @@ -1133,6 +1139,7 @@ void cliSend(SCliConn* pConn) { STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); if (pMsg->pCont == 0) { pMsg->pCont = (void*)rpcMallocCont(0); + tDebug("malloc memory: %p", pMsg->pCont); pMsg->contLen = 0; } @@ -1579,7 +1586,7 @@ static FORCE_INLINE void cliUpdateFqdnCache(SHashObj* cache, char* fqdn) { tinet_ntoa(old, *v); tinet_ntoa(new, addr); tWarn("update ip of fqdn:%s, old: %s, new: %s", fqdn, old, new); - taosHashPut(cache, fqdn, strlen(fqdn) + 1, &addr, sizeof(addr)); + taosHashPut(cache, fqdn, strlen(fqdn), &addr, sizeof(addr)); } } return; @@ -1914,7 +1921,7 @@ void cliDestroyConnMsgs(SCliConn* conn, bool destroy) { } } -void cliIteraConnMsgs(SCliConn* conn) { +void cliConnFreeMsgs(SCliConn* conn) { SCliThrd* pThrd = conn->hostThrd; STrans* pTransInst = pThrd->pTransInst; @@ -1956,7 +1963,7 @@ bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) { } } - cliIteraConnMsgs(conn); + cliConnFreeMsgs(conn); tDebug("%s conn %p receive release request, refId:%" PRId64 "", CONN_GET_INST_LABEL(conn), conn, conn->refId); destroyCmsg(pMsg); @@ -2017,6 +2024,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) { if (pMsg == NULL) { return; } + tDebug("free memory:%p, free ctx: %p", pMsg, pMsg->ctx); transDestroyConnCtx(pMsg->ctx); transFreeMsg(pMsg->msg.pCont); @@ -2523,6 +2531,8 @@ static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle); if (exh == NULL) { return NULL; + } else { + tDebug("conn %p got", exh->handle); } taosWLockLatch(&exh->latch); if (exh->pThrd == NULL && trans != NULL) { @@ -2568,7 +2578,7 @@ int transReleaseCliHandle(void* handle) { cmsg->ctx = pCtx; STraceId* trace = &tmsg.info.traceId; - tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid); + tGDebug("send release request at thread:%08" PRId64 ", malloc memory:%p", pThrd->pid, cmsg); if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) { destroyCmsg(cmsg); @@ -2819,6 +2829,9 @@ int transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { int64_t transAllocHandle() { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + QUEUE_INIT(&exh->q); + taosInitRWLatch(&exh->latch); + exh->refId = transAddExHandle(transGetRefMgt(), exh); QUEUE_INIT(&exh->q); taosInitRWLatch(&exh->latch); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index e894d73c59..68ff5dc5e5 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -305,7 +305,7 @@ void transCtxCleanup(STransCtx* ctx) { ctx->freeFunc(iter->val); iter = taosHashIterate(ctx->args, iter); } - ctx->freeFunc(ctx->brokenVal.val); + if (ctx->freeFunc) ctx->freeFunc(ctx->brokenVal.val); taosHashCleanup(ctx->args); ctx->args = NULL; } @@ -676,6 +676,10 @@ void transDestroyExHandle(void* handle) { if (handle == NULL) { return; } + SExHandle* eh = handle; + if (!QUEUE_IS_EMPTY(&eh->q)) { + tDebug("handle %p mem leak", handle); + } taosMemoryFree(handle); } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index c205c1412c..94d2aa4b00 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1095,6 +1095,7 @@ static FORCE_INLINE SSvrConn* createConn(void* hThrd) { exh->handle = pConn; exh->pThrd = pThrd; exh->refId = transAddExHandle(transGetRefMgt(), exh); + QUEUE_INIT(&exh->q); transAcquireExHandle(transGetRefMgt(), exh->refId); STrans* pTransInst = pThrd->pTransInst; diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c index 470e43c957..aa1a8e3148 100644 --- a/source/util/src/tdecompress.c +++ b/source/util/src/tdecompress.c @@ -14,8 +14,8 @@ */ #include "os.h" -#include "ttypes.h" #include "tcompression.h" +#include "ttypes.h" int32_t getWordLength(char type) { int32_t wordLength = 0; @@ -54,7 +54,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, #if __AVX2__ || __AVX512F__ while (_pos < nelements) { - uint64_t w = *(uint64_t*) ip; + uint64_t w = *(uint64_t *)ip; char selector = (char)(w & INT64MASK(4)); // selector = 4 char bit = bit_per_integer[(int32_t)selector]; // bit = 3 @@ -67,22 +67,22 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, switch (type) { case TSDB_DATA_TYPE_BIGINT: { - int64_t* p = (int64_t*) output; + int64_t *p = (int64_t *)output; int32_t gRemainder = (nelements - _pos); - int32_t num = (gRemainder > elems)? elems:gRemainder; + int32_t num = (gRemainder > elems) ? elems : gRemainder; int32_t batch = 0; int32_t remain = 0; if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { #if __AVX512F__ - batch = num >> 3; - remain = num & 0x07; + batch = num >> 3; + remain = num & 0x07; #endif } else if (tsSIMDEnable && tsAVX2Supported) { #if __AVX2__ - batch = num >> 2; - remain = num & 0x03; + batch = num >> 2; + remain = num & 0x03; #endif } @@ -90,12 +90,12 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { #if __AVX512F__ for (int32_t i = 0; i < batch; ++i) { - __m512i prev = _mm512_set1_epi64(prevValue); - _mm512_storeu_si512((__m512i *)&p[_pos], prev); - _pos += 8; //handle 64bit x 8 = 512bit + __m512i prev = _mm512_set1_epi64(prevValue); + _mm512_storeu_si512((__m512i *)&p[_pos], prev); + _pos += 8; // handle 64bit x 8 = 512bit } for (int32_t i = 0; i < remain; ++i) { - p[_pos++] = prevValue; + p[_pos++] = prevValue; } #endif } else if (tsSIMDEnable && tsAVX2Supported) { @@ -109,7 +109,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, p[_pos++] = prevValue; } - } else { // alternative implementation without SIMD instructions. + } else { // alternative implementation without SIMD instructions. for (int32_t i = 0; i < elems && count < nelements; i++, count++) { p[_pos++] = prevValue; v += bit; @@ -117,69 +117,67 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, } } else { if (tsSIMDEnable && tsAVX512Supported && tsAVX512Enable) { - #if __AVX512F__ +#if __AVX512F__ __m512i sum_mask1 = _mm512_set_epi64(6, 6, 4, 4, 2, 2, 0, 0); __m512i sum_mask2 = _mm512_set_epi64(5, 5, 5, 5, 1, 1, 1, 1); __m512i sum_mask3 = _mm512_set_epi64(3, 3, 3, 3, 3, 3, 3, 3); __m512i base = _mm512_set1_epi64(w); __m512i maskVal = _mm512_set1_epi64(mask); - __m512i shiftBits = _mm512_set_epi64(bit * 7 + 4, bit * 6 + 4, bit * 5 + 4, bit * 4 + 4, bit * 3 + 4, bit * 2 + 4, bit + 4, 4); + __m512i shiftBits = _mm512_set_epi64(bit * 7 + 4, bit * 6 + 4, bit * 5 + 4, bit * 4 + 4, bit * 3 + 4, + bit * 2 + 4, bit + 4, 4); __m512i inc = _mm512_set1_epi64(bit << 3); for (int32_t i = 0; i < batch; ++i) { + __m512i after = _mm512_srlv_epi64(base, shiftBits); + __m512i zigzagVal = _mm512_and_si512(after, maskVal); - __m512i after = _mm512_srlv_epi64(base, shiftBits); - __m512i zigzagVal = _mm512_and_si512(after, maskVal); + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m512i signmask = _mm512_and_si512(_mm512_set1_epi64(1), zigzagVal); + signmask = _mm512_sub_epi64(_mm512_setzero_si512(), signmask); + __m512i delta = _mm512_xor_si512(_mm512_srli_epi64(zigzagVal, 1), signmask); - // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) - __m512i signmask = _mm512_and_si512(_mm512_set1_epi64(1), zigzagVal); - signmask = _mm512_sub_epi64(_mm512_setzero_si512(), signmask); - __m512i delta = _mm512_xor_si512(_mm512_srli_epi64(zigzagVal, 1), signmask); + // calculate the cumulative sum (prefix sum) for each number + // decode[0] = prevValue + final[0] + // decode[1] = decode[0] + final[1] -----> prevValue + final[0] + final[1] + // decode[2] = decode[1] + final[2] -----> prevValue + final[0] + final[1] + final[2] + // decode[3] = decode[2] + final[3] -----> prevValue + final[0] + final[1] + final[2] + final[3] - // calculate the cumulative sum (prefix sum) for each number - // decode[0] = prevValue + final[0] - // decode[1] = decode[0] + final[1] -----> prevValue + final[0] + final[1] - // decode[2] = decode[1] + final[2] -----> prevValue + final[0] + final[1] + final[2] - // decode[3] = decode[2] + final[3] -----> prevValue + final[0] + final[1] + final[2] + final[3] + // 7 6 5 4 3 2 1 + // 0 D7 D6 D5 D4 D3 D2 D1 + // D0 D6 0 D4 0 D2 0 D0 + // 0 D7+D6 D6 D5+D4 D4 D3+D2 D2 + // D1+D0 D0 13 6 9 4 5 2 + // 1 0 + __m512i prev = _mm512_set1_epi64(prevValue); + __m512i cum_sum = _mm512_add_epi64(delta, _mm512_maskz_permutexvar_epi64(0xaa, sum_mask1, delta)); + cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xcc, sum_mask2, cum_sum)); + cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xf0, sum_mask3, cum_sum)); + // 13 6 9 4 5 2 1 + // 0 D7,D6 D6 D5,D4 D4 D3,D2 D2 + // D1,D0 D0 +D5,D4 D5,D4, 0 0 D1,D0 D1,D0 + //0 0 D7~D4 D6~D4 D5~D4 D4 D3~D0 D2~D0 + // D1~D0 D0 22 15 9 4 6 3 + // 1 0 + // + // D3~D0 D3~D0 D3~D0 D3~D0 0 0 0 + // 0 28 21 15 10 6 3 1 + // 0 - //7 6 5 4 3 2 1 0 - //D7 D6 D5 D4 D3 D2 D1 D0 - //D6 0 D4 0 D2 0 D0 0 - //D7+D6 D6 D5+D4 D4 D3+D2 D2 D1+D0 D0 - //13 6 9 4 5 2 1 0 - __m512i prev = _mm512_set1_epi64(prevValue); - __m512i cum_sum = _mm512_add_epi64(delta, _mm512_maskz_permutexvar_epi64(0xaa, sum_mask1, delta)); - cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xcc, sum_mask2, cum_sum)); - cum_sum = _mm512_add_epi64(cum_sum, _mm512_maskz_permutexvar_epi64(0xf0, sum_mask3, cum_sum)); - - - - //13 6 9 4 5 2 1 0 - //D7,D6 D6 D5,D4 D4 D3,D2 D2 D1,D0 D0 - //+D5,D4 D5,D4, 0 0 D1,D0 D1,D0 0 0 - //D7~D4 D6~D4 D5~D4 D4 D3~D0 D2~D0 D1~D0 D0 - //22 15 9 4 6 3 1 0 - // - //D3~D0 D3~D0 D3~D0 D3~D0 0 0 0 0 - //28 21 15 10 6 3 1 0 - - - cum_sum = _mm512_add_epi64(cum_sum, prev); - _mm512_storeu_si512((__m512i *)&p[_pos], cum_sum); - - shiftBits = _mm512_add_epi64(shiftBits, inc); - prevValue = p[_pos + 7]; - _pos += 8; + cum_sum = _mm512_add_epi64(cum_sum, prev); + _mm512_storeu_si512((__m512i *)&p[_pos], cum_sum); + shiftBits = _mm512_add_epi64(shiftBits, inc); + prevValue = p[_pos + 7]; + _pos += 8; } // handle the remain value for (int32_t i = 0; i < remain; i++) { - zigzag_value = ((w >> (v + (batch * bit * 8))) & mask); - prevValue += ZIGZAG_DECODE(int64_t, zigzag_value); + zigzag_value = ((w >> (v + (batch * bit * 8))) & mask); + prevValue += ZIGZAG_DECODE(int64_t, zigzag_value); - p[_pos++] = prevValue; - v += bit; + p[_pos++] = prevValue; + v += bit; } #endif } else if (tsSIMDEnable && tsAVX2Supported) { @@ -255,7 +253,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, } } break; case TSDB_DATA_TYPE_INT: { - int32_t* p = (int32_t*) output; + int32_t *p = (int32_t *)output; if (selector == 0 || selector == 1) { for (int32_t i = 0; i < elems && count < nelements; i++, count++) { @@ -272,7 +270,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, } } break; case TSDB_DATA_TYPE_SMALLINT: { - int16_t* p = (int16_t*) output; + int16_t *p = (int16_t *)output; if (selector == 0 || selector == 1) { for (int32_t i = 0; i < elems && count < nelements; i++, count++) { @@ -317,7 +315,7 @@ int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { #if __AVX512F__ - // todo add it + // todo add it #endif return 0; } @@ -336,12 +334,12 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen int32_t ipos = 1, opos = 0; #if __AVX2__ - __m128i prevVal = _mm_setzero_si128(); - __m128i prevDelta = _mm_setzero_si128(); + __m128i prevVal = _mm_setzero_si128(); + __m128i prevDelta = _mm_setzero_si128(); int32_t batch = nelements >> 1; int32_t remainder = nelements & 0x01; -// __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; + // __mmask16 mask2[16] = {0, 0x0001, 0x0003, 0x0007, 0x000f, 0x001f, 0x003f, 0x007f, 0x00ff}; int32_t i = 0; if (batch > 1) { @@ -354,14 +352,14 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen __m128i data1 = _mm_setzero_si128(); if (nbytes1 > 0) { int64_t tmp = 0; - memcpy(&tmp, (const void*) (input + ipos), nbytes1); + memcpy(&tmp, (const void *)(input + ipos), nbytes1); data1 = _mm_set1_epi64x(tmp); } __m128i data2 = _mm_setzero_si128(); if (nbytes2 > 0) { int64_t tmp = 0; - memcpy(&tmp, (const void*) (input + ipos + nbytes1), nbytes2); + memcpy(&tmp, (const void *)(input + ipos + nbytes1), nbytes2); data2 = _mm_set1_epi64x(tmp); } @@ -382,7 +380,7 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen _mm_storeu_si128((__m128i *)&ostream[opos], finalVal); // keep the previous value - prevVal = _mm_shuffle_epi32 (finalVal, 0xEE); + prevVal = _mm_shuffle_epi32(finalVal, 0xEE); // keep the previous delta of delta, for the first item prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); @@ -393,7 +391,7 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen } // the remain - for(; i < batch; ++i) { + for (; i < batch; ++i) { uint8_t flags = input[ipos++]; int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 @@ -402,14 +400,14 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen __m128i data1 = _mm_setzero_si128(); if (nbytes1 > 0) { int64_t dd = 0; - memcpy(&dd, (const void*) (input + ipos), nbytes1); + memcpy(&dd, (const void *)(input + ipos), nbytes1); data1 = _mm_loadu_si64(&dd); } __m128i data2 = _mm_setzero_si128(); if (nbytes2 > 0) { int64_t dd = 0; - memcpy(&dd, (const void*) (input + ipos + nbytes1), nbytes2); + memcpy(&dd, (const void *)(input + ipos + nbytes1), nbytes2); data2 = _mm_loadu_si64(&dd); } @@ -433,10 +431,10 @@ int32_t tsDecompressTimestampAvx2(const char *const input, const int32_t nelemen _mm_storeu_si128((__m128i *)&ostream[opos], finalVal); // keep the previous value - prevVal = _mm_shuffle_epi32 (finalVal, 0xEE); + prevVal = _mm_shuffle_epi32(finalVal, 0xEE); // keep the previous delta of delta - prevDelta = _mm_shuffle_epi32 (deltaCurrent, 0xEE); + prevDelta = _mm_shuffle_epi32(deltaCurrent, 0xEE); opos += 2; ipos += nbytes1 + nbytes2; @@ -478,8 +476,8 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem #if __AVX512VL__ - __m128i prevVal = _mm_setzero_si128(); - __m128i prevDelta = _mm_setzero_si128(); + __m128i prevVal = _mm_setzero_si128(); + __m128i prevDelta = _mm_setzero_si128(); int32_t numOfBatch = nelements >> 1; int32_t remainder = nelements & 0x01; @@ -493,8 +491,8 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes2 = (flags >> 4) & INT8MASK(4); - __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); - __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void *)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void *)(input + ipos + nbytes1)); data2 = _mm_broadcastq_epi64(data2); __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); @@ -513,7 +511,7 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem _mm_storeu_si128((__m128i *)&ostream[opos], val); // keep the previous value - prevVal = _mm_shuffle_epi32 (val, 0xEE); + prevVal = _mm_shuffle_epi32(val, 0xEE); // keep the previous delta of delta, for the first item prevDelta = _mm_shuffle_epi32(deltaOfDelta, 0xEE); @@ -524,14 +522,14 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem } // the remain - for(; i < numOfBatch; ++i) { + for (; i < numOfBatch; ++i) { uint8_t flags = input[ipos++]; int8_t nbytes1 = flags & INT8MASK(4); // range of nbytes starts from 0 to 7 int8_t nbytes2 = (flags >> 4) & INT8MASK(4); - __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void*)(input + ipos)); - __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void*)(input + ipos + nbytes1)); + __m128i data1 = _mm_maskz_loadu_epi8(mask2[nbytes1], (const void *)(input + ipos)); + __m128i data2 = _mm_maskz_loadu_epi8(mask2[nbytes2], (const void *)(input + ipos + nbytes1)); data2 = _mm_broadcastq_epi64(data2); __m128i zzVal = _mm_blend_epi32(data2, data1, 0x03); @@ -550,7 +548,7 @@ int32_t tsDecompressTimestampAvx512(const char *const input, const int32_t nelem _mm_storeu_si128((__m128i *)&ostream[opos], val); // keep the previous value - prevVal = _mm_shuffle_epi32 (val, 0xEE); + prevVal = _mm_shuffle_epi32(val, 0xEE); // keep the previous delta of delta __m128i delta = _mm_add_epi64(_mm_slli_si128(deltaOfDelta, 8), deltaOfDelta);