From aa8d25f6e548eb2e67d424c82aac38a9ef6b8d2a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 19 Apr 2023 10:03:06 +0800 Subject: [PATCH 001/145] feat: support fetching table tag value --- include/libs/catalog/catalog.h | 1 + source/libs/catalog/inc/catalogInt.h | 8 ++ source/libs/catalog/src/ctgAsync.c | 127 ++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 1 deletion(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2c684f8f76..2f8e7846f3 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -82,6 +82,7 @@ typedef struct SCatalogReq { SArray* pUser; // element is SUserAuthInfo SArray* pTableIndex; // element is SNAME SArray* pTableCfg; // element is SNAME + SArray* pTableTag; // element is SNAME bool qNodeRequired; // valid qnode bool dNodeRequired; // valid dnode bool svrVerRequired; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 85a130d293..f0e5024c59 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -58,6 +58,7 @@ typedef enum { CTG_CI_OTHERTABLE_META, CTG_CI_TBL_SMA, CTG_CI_TBL_CFG, + CTG_CI_TBL_TAG, CTG_CI_INDEX_INFO, CTG_CI_USER, CTG_CI_UDF, @@ -110,6 +111,7 @@ typedef enum { CTG_TASK_GET_SVR_VER, CTG_TASK_GET_TB_META_BATCH, CTG_TASK_GET_TB_HASH_BATCH, + CTG_TASK_GET_TB_TAG, } CTG_TASK_TYPE; typedef enum { @@ -186,6 +188,11 @@ typedef struct SCtgTbCfgCtx { SVgroupInfo* pVgInfo; } SCtgTbCfgCtx; +typedef struct SCtgTbTagCtx { + SName* pName; + SVgroupInfo* pVgInfo; +} SCtgTbTagCtx; + typedef struct SCtgDbVgCtx { char dbFName[TSDB_DB_FNAME_LEN]; } SCtgDbVgCtx; @@ -304,6 +311,7 @@ typedef struct SCtgJob { catalogCallback userFp; int32_t tbMetaNum; int32_t tbHashNum; + int32_t tbTagNum; int32_t dbVgNum; int32_t udfNum; int32_t qnodeNum; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index f2a354997d..b10da5cc73 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -386,6 +386,37 @@ int32_t ctgInitGetTbCfgTask(SCtgJob* pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbTagTask(SCtgJob* pJob, int32_t taskIdx, void* param) { + SName* name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_TAG; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTagCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbTagCtx* ctx = task.taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(task.taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, + ctgTaskTypeStr(task.type), name->tname); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -437,6 +468,15 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(name, dbFName); taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } + + for (int32_t i = 0; i < pJob->tbTagNum; ++i) { + SName* name = taosArrayGet(pReq->pTableTag, i); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(name, dbFName); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); } char* dbFName = taosHashIterate(pDb, NULL); @@ -505,9 +545,10 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo); int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); + int32_t tbTagNum = (int32_t)ctgGetTablesReqNum(pReq->pTableTag); int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + - userNum + dbInfoNum + tbIndexNum + tbCfgNum; + userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum; *job = taosMemoryCalloc(1, sizeof(SCtgJob)); if (NULL == *job) { @@ -537,6 +578,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const pJob->tbIndexNum = tbIndexNum; pJob->tbCfgNum = tbCfgNum; pJob->svrVerNum = svrVerNum; + pJob->tbTagNum = tbTagNum; #if CTG_BATCH_FETCH pJob->pBatchs = @@ -604,6 +646,12 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } + for (int32_t i = 0; i < tbCfgNum; ++i) { + SName* name = taosArrayGet(pReq->pTableTag, i); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); + } + + for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX_INFO, indexName, NULL)); @@ -1473,6 +1521,24 @@ _return: CTG_RET(code); } + +int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTask* pTask = tReq->pTask; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out; + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + + int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; @@ -1935,6 +2001,45 @@ _return: CTG_RET(code); } + +int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; + SArray* pRes = NULL; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pCtx->pName, dbFName); + SCtgJob* pJob = pTask->pJob; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); + if (NULL == pMsgCtx->pBatchs) { + pMsgCtx->pBatchs = pJob->pBatchs; + } + + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); + return TSDB_CODE_SUCCESS; + } + } + + CTG_CACHE_NHIT_INC(CTG_CI_TBL_TAG, 1); + + CTG_ERR_JRET(ctgGetTableCfgFromVnode(pCtg, pConn, pCtx->pName, pCtx->pVgInfo, NULL, pTask)); + + return TSDB_CODE_SUCCESS; + +_return: + + if (CTG_TASK_LAUNCHED == pTask->status) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + int32_t ctgLaunchGetQnodeTask(SCtgTask* pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; @@ -2138,6 +2243,25 @@ _return: CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); } +int32_t ctgGetTbTagCb(SCtgTask* pTask) { + int32_t code = 0; + + CTG_ERR_JRET(pTask->subRes.code); + + SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; + SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; + + pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); + + CTG_RET(ctgLaunchGetTbTagTask(pTask)); + +_return: + + CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); +} + + int32_t ctgGetUserCb(SCtgTask* pTask) { int32_t code = 0; @@ -2197,6 +2321,7 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL}, {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, + {ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob* pJob) { From 25542c26c7998641bf62039a35a8edbac318944f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 10:43:13 +0800 Subject: [PATCH 002/145] fix:remove tmPushMsg in consumer --- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqUtil.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8f26d5868c..aca2729731 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -535,7 +535,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg atomic_store_32(&pHandle->epoch, -1); // remove if it has been register in the push manager, and return one empty block to consumer - tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); + //tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 5ac747947f..81dd8abc3e 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -265,7 +265,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, // till now, all data has been transferred to consumer, new data needs to push client once arrived. if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + //code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); taosWUnLockLatch(&pTq->lock); return code; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b62bf27def..3c8687fa4d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -447,11 +447,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp walApplyVer(pVnode->pWal, version); - if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { + //if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ - vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); - return -1; - } + //vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); + //return -1; + //} // commit if need if (needCommit) { From 1061eef1441c56058b7fc4d31df51c7d9e0d157a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 10:49:18 +0800 Subject: [PATCH 003/145] fix:move consumer msg from fetch thread to query thread --- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d61eb3ec03..fc724f2b45 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -519,7 +519,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToQueryQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 3c8687fa4d..579ef8a952 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -487,11 +487,16 @@ int32_t vnodePreprocessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { vTrace("message in vnode query queue is processing"); - if ((pMsg->msgType == TDMT_SCH_QUERY) && !syncIsReadyForRead(pVnode->sync)) { + if ((pMsg->msgType == TDMT_SCH_QUERY || pMsg->msgType == TDMT_VND_TMQ_CONSUME) && !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } + if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) { + vnodeRedirectRpcMsg(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); + return 0; + } + SReadHandle handle = {.meta = pVnode->pMeta, .config = &pVnode->config, .vnode = pVnode, .pMsgCb = &pVnode->msgCb}; switch (pMsg->msgType) { case TDMT_SCH_QUERY: @@ -499,6 +504,8 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { return qWorkerProcessQueryMsg(&handle, pVnode->pQuery, pMsg, 0); case TDMT_SCH_QUERY_CONTINUE: return qWorkerProcessCQueryMsg(&handle, pVnode->pQuery, pMsg, 0); + case TDMT_VND_TMQ_CONSUME: + return tqProcessPollReq(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in query queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; @@ -508,17 +515,12 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { vTrace("vgId:%d, msg:%p in fetch queue is processing", pVnode->config.vgId, pMsg); if ((pMsg->msgType == TDMT_SCH_FETCH || pMsg->msgType == TDMT_VND_TABLE_META || pMsg->msgType == TDMT_VND_TABLE_CFG || - pMsg->msgType == TDMT_VND_BATCH_META || pMsg->msgType == TDMT_VND_TMQ_CONSUME) && + pMsg->msgType == TDMT_VND_BATCH_META) && !syncIsReadyForRead(pVnode->sync)) { vnodeRedirectRpcMsg(pVnode, pMsg, terrno); return 0; } - if (pMsg->msgType == TDMT_VND_TMQ_CONSUME && !pVnode->restored) { - vnodeRedirectRpcMsg(pVnode, pMsg, TSDB_CODE_SYN_RESTORING); - return 0; - } - switch (pMsg->msgType) { case TDMT_SCH_FETCH: case TDMT_SCH_MERGE_FETCH: @@ -537,8 +539,6 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetTableCfg(pVnode, pMsg, true); case TDMT_VND_BATCH_META: return vnodeGetBatchMeta(pVnode, pMsg); - case TDMT_VND_TMQ_CONSUME: - return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH: From 14fd2e790402c11f0c68d118f685ab6ee87fd626 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 11:44:52 +0800 Subject: [PATCH 004/145] fix:remove lock for consume handler --- source/dnode/vnode/src/tq/tq.c | 22 +++++++++++----------- source/dnode/vnode/src/tq/tqUtil.c | 16 ++++++++-------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aca2729731..bccfe2b9e1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -325,25 +325,25 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } // 2. check re-balance status - taosRLockLatch(&pTq->lock); +// taosRLockLatch(&pTq->lock); if (pHandle->consumerId != consumerId) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; - taosRUnLockLatch(&pTq->lock); +// taosRUnLockLatch(&pTq->lock); return -1; } - taosRUnLockLatch(&pTq->lock); +// taosRUnLockLatch(&pTq->lock); // 3. update the epoch value - taosWLockLatch(&pTq->lock); +// taosWLockLatch(&pTq->lock); int32_t savedEpoch = pHandle->epoch; if (savedEpoch < reqEpoch) { tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, reqEpoch); pHandle->epoch = reqEpoch; } - taosWUnLockLatch(&pTq->lock); +// taosWUnLockLatch(&pTq->lock); char buf[80]; tFormatOffset(buf, 80, &reqOffset); @@ -358,12 +358,12 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey); - taosWLockLatch(&pTq->lock); - int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); - if (code != 0) { - tqDebug("vgId:%d, tq remove push handle %s", pTq->pVnode->config.vgId, pReq->subKey); - } - taosWUnLockLatch(&pTq->lock); +// taosWLockLatch(&pTq->lock); +// int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); +// if (code != 0) { +// tqDebug("vgId:%d, tq remove push handle %s", pTq->pVnode->config.vgId, pReq->subKey); +// } +// taosWUnLockLatch(&pTq->lock); STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); if (pHandle) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 81dd8abc3e..ab1be15271 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -254,7 +254,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); // lock - taosWLockLatch(&pTq->lock); +// taosWLockLatch(&pTq->lock); qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); @@ -263,12 +263,12 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, } // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - //code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - taosWUnLockLatch(&pTq->lock); - return code; - } +// if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && +// dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { +// //code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); +// taosWUnLockLatch(&pTq->lock); +// return code; +// } code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); @@ -281,7 +281,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, tFormatOffset(buf, 80, &dataRsp.rspOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 " code:%d", consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); - taosWUnLockLatch(&pTq->lock); +// taosWUnLockLatch(&pTq->lock); tDeleteSMqDataRsp(&dataRsp); } return code; From 594d68b8a4b86f116f489e927607e25687c356a1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 11:46:59 +0800 Subject: [PATCH 005/145] fix:remove lock for consume handler --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index bccfe2b9e1..36fcb35791 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -357,7 +357,7 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; tqDebug("vgId:%d, tq process delete sub req %s", pTq->pVnode->config.vgId, pReq->subKey); - + int32_t code = 0; // taosWLockLatch(&pTq->lock); // int32_t code = taosHashRemove(pTq->pPushMgr, pReq->subKey, strlen(pReq->subKey)); // if (code != 0) { From 6f94281ab7ebb76b3ccd1b26be87f7e12912bacf Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 17:30:58 +0800 Subject: [PATCH 006/145] fix:add log for wal --- include/libs/wal/wal.h | 2 ++ source/libs/wal/src/walRead.c | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index b51289de5e..835f786d97 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -149,6 +149,8 @@ typedef struct SWalReader { int64_t capacity; // int8_t curInvalid; // int8_t curStopped; + int64_t bodyCnt; + int64_t bodyTotalSize; TdThreadMutex mutex; SWalFilterCond cond; // TODO remove it diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index dc3ff3e6de..9a46b4af2a 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -262,8 +262,8 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d", pReader->pWal->cfg.vgId, ver, - pReadHead->bodyLen); + wInfo("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d, total cnt:%"PRId64 ", total size:%"PRId64, pReader->pWal->cfg.vgId, ver, + pReadHead->bodyLen, pReader->bodyCnt, pReader->bodyTotalSize); if (pReader->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); @@ -300,6 +300,8 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); pReader->curVersion = ver + 1; + pReader->bodyCnt++; + pReader->bodyTotalSize += pReadHead->bodyLen; return 0; } From 4d9d1b520d95b55af544638e69a51affea055ed1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 19 Apr 2023 18:16:57 +0800 Subject: [PATCH 007/145] fix:add log for wal --- source/libs/wal/src/walRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9a46b4af2a..09b6db6afe 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -262,7 +262,7 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wInfo("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d, total cnt:%"PRId64 ", total size:%"PRId64, pReader->pWal->cfg.vgId, ver, + wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d, total cnt:%"PRId64 ", total size:%"PRId64, pReader->pWal->cfg.vgId, ver, pReadHead->bodyLen, pReader->bodyCnt, pReader->bodyTotalSize); if (pReader->capacity < pReadHead->bodyLen) { From 562e76524832fe1bce455d405f8b616b315f1570 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Thu, 20 Apr 2023 17:10:21 +0800 Subject: [PATCH 008/145] =?UTF-8?q?test=EF=BC=9Aadd=20test=20case=20for=20?= =?UTF-8?q?multistage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/parallel_test/cases.task | 1 + tests/system-test/0-others/multilevel.py | 223 +++++++++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 tests/system-test/0-others/multilevel.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index dda4ec3e84..b12d1b2da7 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -127,6 +127,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_control.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/user_manage.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/fsync.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/multilevel.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py new file mode 100644 index 0000000000..6780ea9fef --- /dev/null +++ b/tests/system-test/0-others/multilevel.py @@ -0,0 +1,223 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + def basic(self): + tdLog.info("============== basic test ===============") + cfg={ + '/mnt/data1' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.start(1) + + tdLog.info("================= step2") + tdSql.haveFile('/mnt/data1/',1) + tdSql.haveFile('/mnt/data2/',0) + tdDnodes.stop(1) + def dir_not_exist(self): + tdLog.info("============== dir_not_exist test ===============") + cfg={ + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + os.system('rm -rf /mnt/data2') + + + tdLog.info("================= step1") + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + def dir_permission_denied(self): + tdDnodes.stop(1) + tdLog.info("============== dir_permission_denied test ===============") + cfg={ + '/mnt/data1 0 0' : 'dataDir', + '/mnt/data2 0 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + tdSql.createDir('/mnt/data2') + os.system('chmod 111 /mnt/data2') + + tdLog.info("================= step1") + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + def file_distribution_same_level(self): + tdLog.info("============== file_distribution_same_level test ===============") + dataDir = ['data00','data01','data02','data03','data04'] + dataDict = {'data00':0,'data01':0,'data02':0,'data03':0,'data04':0} + tdDnodes.stop(1) + self.ntables = 1000 + self.ts = 1520000010000 + tdLog.info("================= step1") + cfg={ + '/mnt/data00 0 1' : 'dataDir', + '/mnt/data01 0 0' : 'dataDir', + '/mnt/data02 0 0' : 'dataDir', + '/mnt/data03 0 0' : 'dataDir', + '/mnt/data04 0 0' : 'dataDir' + } + dir_list = ['/mnt/data00','/mnt/data01','/mnt/data02','/mnt/data03','/mnt/data04'] + for i in dir_list: + tdSql.createDir(i) + tdDnodes.deploy(1,cfg) + tdDnodes.start(1) + + tdSql.execute("create database test duration 1") + tdSql.execute("use test") + + tdSql.execute("create table stb(ts timestamp, c int) tags(t int)") + + for i in range(self.ntables): + tdSql.execute("create table tb%d using stb tags(%d)" %(i, i)) + tdSql.execute("insert into tb%d values(%d, 1)" % (i,self.ts + int (i / 100) * 86400000)) + + tdLog.info("================= step2") + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from test.stb") + tdSql.checkRows(1000) + tdLog.info("================= step3") + tdSql.execute('drop database test') + for i in range(50): + tdSql.execute("create database test%d duration 1" %(i)) + tdSql.execute("use test%d" %(i)) + tdSql.execute("create table tb (ts timestamp,i int)") + for j in range(10): + tdSql.execute("insert into tb values(%d, 1)" % (self.ts + int (i / 100) * 86400000)) + tdDnodes.stop(1) + tdDnodes.start(1) + flag = True + for i in range(4): + if dataDict[dataDir[i]] == dataDict[dataDir[i+1]]: + flag = flag & True + else: + flag = flag & False + break + if not flag : tdLog.exit("%s failed, expect not occured" % (sys.argv[0])) + + def three_level_basic(self): + tdLog.info("============== three_level_basic test ===============") + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data000 0 1' : 'dataDir', + '/mnt/data001 0 0' : 'dataDir', + '/mnt/data002 0 0' : 'dataDir', + '/mnt/data010 1 0' : 'dataDir', + '/mnt/data011 1 0' : 'dataDir', + '/mnt/data012 1 0' : 'dataDir', + '/mnt/data020 2 0' : 'dataDir', + '/mnt/data021 2 0' : 'dataDir', + '/mnt/data022 2 0' : 'dataDir' + } + dir_list = ['/mnt/data000','/mnt/data001','/mnt/data002','/mnt/data010','/mnt/data011','/mnt/data012','/mnt/data020','/mnt/data021''/mnt/data022'] + for i in dir_list: + tdSql.createDir(i) + + tdDnodes.deploy(1,cfg) + tdDnodes.start(1) + for i in dir_list: + if i == '/mnt/data000': + tdSql.haveFile(i,1) + else: + tdSql.haveFile(i,0) + + def more_than_16_disks(self): + tdLog.info("============== more_than_16_disks test ===============") + cfg={} + for i in range(17): + if i == 0 : + datadir = '/mnt/data%d 0 1' % (i+1) + else: + datadir = '/mnt/data%d 0 0' % (i+1) + cfg.update({ datadir : 'dataDir' }) + tdSql.createDir('/mnt/data%d' % (i+1)) + + tdLog.info("================= step1") + tdDnodes.stop(1) + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdLog.info("================= step2") + tdSql.taosdStatus(0) + + def missing_middle_level(self): + tdLog.info("============== missing_middle_level test ===============") + tdDnodes.stop(1) + # Test1 1 dataDir + cfg={ + '/mnt/data1 1 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + tdDnodes.stop(1) + # Test2 2 dataDir + cfg={ + '/mnt/data1 0 1' : 'dataDir', + '/mnt/data1 2 0' : 'dataDir' + } + tdSql.createDir('/mnt/data1') + + tdDnodes.deploy(1,cfg) + tdDnodes.startWithoutSleep(1) + + tdSql.taosdStatus(0) + def run(self): + self.basic() + self.dir_not_exist() + self.dir_permission_denied() + self.file_distribution_same_level() + self.three_level_basic() + self.more_than_16_disks() + self.missing_middle_level() + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From e36bf05f98b7b973f742df9ab7238a4cad518f3f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 20 Apr 2023 17:30:09 +0800 Subject: [PATCH 009/145] enh: support get subtable tag and auth --- include/libs/catalog/catalog.h | 1 + source/libs/catalog/inc/catalogInt.h | 10 ++ source/libs/catalog/src/catalog.c | 2 +- source/libs/catalog/src/ctgAsync.c | 96 +++++++++++++++++--- source/libs/catalog/src/ctgCache.c | 23 +++++ source/libs/catalog/src/ctgUtil.c | 131 ++++++++++++++++++--------- 6 files changed, 202 insertions(+), 61 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 2f8e7846f3..6f2fb4eb6b 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -106,6 +106,7 @@ typedef struct SMetaData { SArray* pUser; // pRes = SUserAuthRes* SArray* pQnodeList; // pRes = SArray* SArray* pTableCfg; // pRes = STableCfg* + SArray* pTableTag; // pRes = SArray* SArray* pDnodeList; // pRes = SArray* SMetaRes* pSvrVer; // pRes = char* } SMetaData; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f0e5024c59..1eaf45dafe 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -154,6 +154,11 @@ typedef struct SCtgTbCacheInfo { int32_t tbType; } SCtgTbCacheInfo; +typedef struct SCtgTbMetaParam { + SName* pName; + int32_t flag; +} SCtgTbMetaParam; + typedef struct SCtgTbMetaCtx { SCtgTbCacheInfo tbInfo; int32_t vgId; @@ -631,6 +636,7 @@ typedef struct SCtgCacheItemInfo { #define CTG_FLAG_SYS_DB 0x8 #define CTG_FLAG_FORCE_UPDATE 0x10 #define CTG_FLAG_ONLY_CACHE 0x20 +#define CTG_FLAG_SYNC_OP 0x40 #define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v)) @@ -933,6 +939,10 @@ void ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCach void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); void ctgGetGlobalCacheStat(SCtgCacheStat* pStat); int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); +int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName); +int32_t ctgGetTbTagCb(SCtgTask* pTask); +int32_t ctgGetUserCb(SCtgTask* pTask); extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bddc6c01a7..b263654e70 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -208,7 +208,7 @@ int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgTbMetaCtx* ctx } while (true) { - CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, false)); + CTG_ERR_JRET(ctgRefreshTbMeta(pCtg, pConn, ctx, &output, ctx->flag & CTG_FLAG_SYNC_OP)); if (CTG_IS_META_TABLE(output->metaType)) { *pTableMeta = output->tbMeta; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index b10da5cc73..affcfeb8ac 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -21,7 +21,8 @@ #include "trpc.h" int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { - SName* name = (SName*)param; + SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param; + SName* name = pParam->pName; SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_META; @@ -41,7 +42,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob* pJob, int32_t taskIdx, void* param) { } memcpy(ctx->pName, name, sizeof(*name)); - ctx->flag = CTG_FLAG_UNKNOWN_STB; + ctx->flag = pParam->flag | CTG_FLAG_UNKNOWN_STB; taosArrayPush(pJob->pTasks, &task); @@ -545,7 +546,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo); int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); - int32_t tbTagNum = (int32_t)ctgGetTablesReqNum(pReq->pTableTag); + int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag); int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum; @@ -646,7 +647,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } - for (int32_t i = 0; i < tbCfgNum; ++i) { + for (int32_t i = 0; i < tbTagNum; ++i) { SName* name = taosArrayGet(pReq->pTableTag, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TAG, name, NULL)); } @@ -761,7 +762,11 @@ int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { - pJob->jobRes.pTableIndex = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); + SArray* pRes = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableIndex, NULL, pRes)) { + taosArrayDestroy(pRes); + } + if (NULL == pJob->jobRes.pTableIndex) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -776,7 +781,11 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableCfg) { - pJob->jobRes.pTableCfg = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); + SArray* pRes = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableCfg, NULL, pRes)) { + taosArrayDestroy(pRes); + } + if (NULL == pJob->jobRes.pTableCfg) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -788,6 +797,26 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbTagRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableTag) { + SArray* pRes = taosArrayInit(pJob->tbTagNum, sizeof(SMetaRes)); + if (atomic_val_compare_exchange_ptr(&pJob->jobRes.pTableTag, NULL, pRes)) { + taosArrayDestroy(pRes); + } + + if (NULL == pJob->jobRes.pTableTag) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; + taosArrayPush(pJob->jobRes.pTableTag, &res); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgDumpIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pIndex) { @@ -1123,7 +1152,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; - ctgUpdateTbMetaToCache(pCtg, pOut, false); + ctgUpdateTbMetaToCache(pCtg, pOut, flag & CTG_FLAG_SYNC_OP); if (CTG_IS_META_BOTH(pOut->metaType)) { memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); @@ -1525,11 +1554,36 @@ _return: int32_t ctgHandleGetTbTagRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; + SCatalog* pCtg = pTask->pJob->pCtg; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableCfgRsp* pRsp = (STableCfgRsp*)pTask->msgCtx.out; + if (NULL == pRsp->pTags || pRsp->tagsLen <= 0) { + ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pRsp->pTags, pRsp->tagsLen); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + } + + SArray* pTagVals = NULL; + STag* pTag = (STag*)pRsp->pTags; + + if (tTagIsJson(pTag)) { + pTagVals = taosArrayInit(1, sizeof(STagVal)); + if (NULL == pTagVals) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + char* pJson = parseTagDatatoJson(pTag); + STagVal tagVal; + tagVal.cid = 0; + tagVal.type = TSDB_DATA_TYPE_JSON; + tagVal.pData = pJson; + tagVal.nData = strlen(pJson); + taosArrayPush(pTagVals, &tagVal); + } else { + CTG_ERR_JRET(tTagToValArray((const STag*)pRsp->pTags, &pTagVals)); + } - TSWAP(pTask->res, pTask->msgCtx.out); + pTask->res = pTagVals; _return: @@ -1971,7 +2025,10 @@ int32_t ctgLaunchGetTbCfgTask(SCtgTask* pTask) { if (pCtx->tbType <= 0) { CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType)); if (pCtx->tbType <= 0) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, pCtx->pName)); + SCtgTbMetaParam param; + param.pName = pCtx->pName; + param.flag = 0; + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, ¶m)); return TSDB_CODE_SUCCESS; } } @@ -2019,7 +2076,7 @@ int32_t ctgLaunchGetTbTagTask(SCtgTask* pTask) { if (NULL == pCtx->pVgInfo) { CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); if (NULL == pCtx->pVgInfo) { - CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbTagCb, dbFName)); return TSDB_CODE_SUCCESS; } } @@ -2189,7 +2246,10 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { taosMemoryFreeClear(rsp.pRawRes); if (rsp.metaNotExists) { - CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &pCtx->user.tbName)); + SCtgTbMetaParam param; + param.pName = &pCtx->user.tbName; + param.flag = CTG_FLAG_SYNC_OP; + CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetUserCb, ¶m)); } else { CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); } @@ -2251,9 +2311,11 @@ int32_t ctgGetTbTagCb(SCtgTask* pTask) { SCtgTbTagCtx* pCtx = (SCtgTbTagCtx*)pTask->taskCtx; SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; - pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); - + if (NULL == pCtx->pVgInfo) { + pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); + } + CTG_RET(ctgLaunchGetTbTagTask(pTask)); _return: @@ -2286,8 +2348,12 @@ int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) { int32_t ctgCompTbMetaTasks(SCtgTask* pTask, void* param, bool* equal) { SCtgTbMetaCtx* ctx = pTask->taskCtx; + SCtgTbMetaParam* pParam = (SCtgTbMetaParam*)param; - *equal = tNameTbNameEqual(ctx->pName, (SName*)param); + *equal = tNameTbNameEqual(ctx->pName, (SName*)pParam->pName); + if (*equal) { + ctx->flag |= pParam->flag; + } return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 592b6e9c72..ee864d985e 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -703,6 +703,29 @@ _return: CTG_RET(code); } +int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid, char **stbName) { + *stbName = NULL; + + SCtgDBCache *dbCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + return TSDB_CODE_SUCCESS; + } + + char *stb = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + if (NULL == stb) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); + return TSDB_CODE_SUCCESS; + } + + *stbName = taosStrdup(stb); + + taosHashRelease(dbCache->stbCache, stb); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { if (IS_SYS_DBNAME(pReq->tbName.dbname)) { *inCache = true; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index b2b2b5a87e..7ef2e34d1e 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -486,6 +486,18 @@ void ctgFreeBatchHash(void* hash) { taosMemoryFreeClear(pRes->pRes); } +void ctgFreeJsonTagVal(void *val) { + if (NULL == val) { + return; + } + + STagVal *pVal = (STagVal *)val; + + if (TSDB_DATA_TYPE_JSON == pVal->type) { + taosMemoryFree(pVal->pData); + } +} + void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { switch (type) { case CTG_TASK_GET_QNODE: @@ -526,6 +538,14 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_TAG: { + if (1 == taosArrayGetSize(*pRes)) { + taosArrayDestroyEx(*pRes, ctgFreeJsonTagVal); + } else { + taosArrayDestroy(*pRes); + } + *pRes = NULL; + } case CTG_TASK_GET_TB_META_BATCH: { SArray* pArray = (SArray*)*pRes; int32_t num = taosArrayGetSize(pArray); @@ -679,6 +699,12 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_TAG: { + SCtgTbTagCtx* taskCtx = (SCtgTbTagCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + taosMemoryFreeClear(taskCtx->pVgInfo); + break; + } case CTG_TASK_GET_DB_VGROUP: case CTG_TASK_GET_DB_CFG: case CTG_TASK_GET_DB_INFO: @@ -1336,54 +1362,69 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { STableMeta* pMeta = NULL; SGetUserAuthRsp* pInfo = &req->authInfo; SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs; + char* stbName = NULL; + + char tbFName[TSDB_TABLE_FNAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameExtractFullName(&req->pRawReq->tbName, tbFName); + tNameGetFullDbName(&req->pRawReq->tbName, dbFName); - char tbFullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(&req->pRawReq->tbName, tbFullName); - char* pCond = taosHashGet(pTbs, tbFullName, strlen(tbFullName)); - if (pCond) { - if (strlen(pCond) > 1) { - CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + while (true) { + char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName)); + if (pCond) { + if (strlen(pCond) > 1) { + CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + } + + res->pRawRes->pass = true; + return TSDB_CODE_SUCCESS; } - res->pRawRes->pass = true; - return TSDB_CODE_SUCCESS; + if (stbName) { + res->pRawRes->pass = false; + goto _return; + } + + CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); + if (NULL == pMeta) { + if (req->onlyCache) { + res->metaNotExists = true; + ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + return TSDB_CODE_SUCCESS; + } + + SCtgTbMetaCtx ctx = {0}; + ctx.pName = (SName*)&req->pRawReq->tbName; + ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP; + + CTG_ERR_RET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); + } + + if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { + res->pRawRes->pass = false; + goto _return; + } + + if (TSDB_CHILD_TABLE == pMeta->tableType) { + CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, dbFName, pMeta->suid, &stbName)); + if (NULL == stbName) { + if (req->onlyCache) { + res->metaNotExists = true; + ctgDebug("suid %" PRIu64 " name not in cache for auth", pMeta->suid); + return TSDB_CODE_SUCCESS; + } + + continue; + } + + sprintf(tbFName, "%s.%s", dbFName, stbName); + continue; + } + + ctgError("Invalid table type %d for %s", pMeta->tableType, tbFName); + CTG_ERR_JRET(TSDB_CODE_INVALID_PARA); } - res->pRawRes->pass = false; - - // CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); - // if (NULL == pMeta) { - // if (req->onlyCache) { - // res->metaNotExists = true; - // ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); - // return TSDB_CODE_SUCCESS; - // } - - // CTG_ERR_RET(catalogGetTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, &pMeta)); - // } - - // if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { - // res->pRawRes->pass = false; - // goto _return; - // } - - // if (TSDB_CHILD_TABLE == pMeta->tableType) { - // res->pRawRes->pass = true; - - // /* - // char stbName[TSDB_TABLE_NAME_LEN] = {0}; - // CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, pMeta->suid, stbName)); - // if (0 == stbName[0]) { - // if (req->onlyCache) { - // res->notExists = true; - // return TSDB_CODE_SUCCESS; - // } - - // CTG_ERR_RET(catalogRefreshTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, 0)); - // } - // */ - // } - _return: taosMemoryFree(pMeta); @@ -1423,7 +1464,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { req->singleType = AUTH_TYPE_READ; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); - if (pRes->pass) { + if (pRes->pass || res->metaNotExists) { return TSDB_CODE_SUCCESS; } } @@ -1439,7 +1480,7 @@ int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { req->singleType = AUTH_TYPE_WRITE; CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); - if (pRes->pass) { + if (pRes->pass || res->metaNotExists) { return TSDB_CODE_SUCCESS; } } From 96abcfe8768fa3be6d99cb0b7da1ebf8e851d15d Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 20 Apr 2023 17:42:34 +0800 Subject: [PATCH 010/145] test: modify tmq case --- tests/system-test/7-tmq/subscribeDb3.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index bddb196f4a..1de9b62bcd 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -82,7 +82,7 @@ class TDTestCase: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) if tdSql.getRows() == 2 : - print(tdSql.getData(0, 1), tdSql.getData(1, 1)) + tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1))) if tdSql.getData(1, 1) == 1: break time.sleep(0.1) @@ -122,6 +122,7 @@ class TDTestCase: os.system(shellCmd) def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl): + tdLog.info("start create tables......") tsql.execute("create database if not exists %s vgroups %d wal_retention_period 3600"%(dbName, vgroups)) tsql.execute("use %s" %dbName) tsql.execute("create table if not exists %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName) @@ -137,11 +138,11 @@ class TDTestCase: tsql.execute(sql) event.set() - tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) + tdLog.info("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum)) return def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs): - tdLog.debug("start to insert data ............") + tdLog.info("start to insert data ............") tsql.execute("use %s" %dbName) pre_insert = "insert into " sql = pre_insert @@ -163,7 +164,7 @@ class TDTestCase: if sql != pre_insert: #print("insert sql:%s"%sql) tsql.execute(sql) - tdLog.debug("insert data ............ [OK]") + tdLog.info("insert data ............ [OK]") return def prepareEnv(self, **parameterDict): @@ -286,7 +287,7 @@ class TDTestCase: prepareEnvThread.start() tdLog.info("create topics from db") - topicName1 = 'topic_db1' + topicName1 = 'topic_db11' tdSql.execute("create topic %s as database %s" %(topicName1, parameterDict['dbName'])) consumerId = 0 From a952c008af937cd3f5a43614887e3a98835ac1ab Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Fri, 21 Apr 2023 15:45:25 +0800 Subject: [PATCH 011/145] update test case for bug TS-3216 --- tests/system-test/0-others/multilevel.py | 43 ++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index 6780ea9fef..350337942b 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -194,7 +194,7 @@ class TDTestCase: tdSql.taosdStatus(0) tdDnodes.stop(1) # Test2 2 dataDir - cfg={ + cfg = { '/mnt/data1 0 1' : 'dataDir', '/mnt/data1 2 0' : 'dataDir' } @@ -204,6 +204,45 @@ class TDTestCase: tdDnodes.startWithoutSleep(1) tdSql.taosdStatus(0) + + def trim_database(self): + tdLog.info("============== trim_database test ===============") + tdDnodes.stop(1) + cfg = { + '/mnt/data1 0 1' : 'dataDir' + + } + tdSql.createDir('/mnt/data1') + tdDnodes.deploy(1,cfg) + tdDnodes.start(1) + + tdSql.execute('create database dbtest') + tdSql.execute('use dbtest') + tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)') + tdSql.execute('create table tb1 using stb tags(1)') + for i in range(10,30): + tdSql.execute(f'insert into tb1 values(now-{i}d,10)') + tdSql.execute('flush database dbtest') + tdSql.haveFile('/mnt/data1/',1) + tdDnodes.stop(1) + cfg={ + '/mnt/data1 0 1' : 'dataDir', + '/mnt/data2 1 0' : 'dataDir', + '/mnt/data3 2 0' : 'dataDir', + } + tdSql.createDir('/mnt/data2') + tdSql.createDir('/mnt/data3') + tdDnodes.deploy(1,cfg) + tdDnodes.start(1) + tdSql.haveFile('/mnt/data1/',1) + tdSql.haveFile('/mnt/data2/',0) + tdSql.haveFile('/mnt/data3/',0) + tdSql.execute('alter database dbtest keep 10d,365d,3650d') + tdSql.execute('trim database dbtest') + time.sleep(3) + tdSql.haveFile('/mnt/data1/',1) + tdSql.haveFile('/mnt/data2/',1) + def run(self): self.basic() self.dir_not_exist() @@ -212,7 +251,7 @@ class TDTestCase: self.three_level_basic() self.more_than_16_disks() self.missing_middle_level() - + self.trim_database() def stop(self): From c5b37377026167e2abac828617d4182296fc715f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 23 Apr 2023 11:21:01 +0800 Subject: [PATCH 012/145] fix: forbid delete from system table --- include/common/systable.h | 1 + source/common/src/systable.c | 4 ++++ source/libs/planner/src/planLogicCreater.c | 6 +++++- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/include/common/systable.h b/include/common/systable.h index 558a1ca297..6e0c67bb3f 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -77,6 +77,7 @@ void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size); void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size); void getVisibleInfosTablesNum(bool sysInfo, size_t* size); bool invisibleColumn(bool sysInfo, int8_t tableType, int8_t flags); +bool isSystemDb(const char *dbName); #ifdef __cplusplus } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index cd3dd63ef0..de95a4a5be 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -448,3 +448,7 @@ bool invisibleColumn(bool sysInfo, int8_t tableType, int8_t flags) { } return 0 != (flags & COL_IS_SYSINFO); } + +bool isSystemDb(const char *dbName) { + return ((strcasecmp(dbName, TSDB_INFORMATION_SCHEMA_DB) == 0) || (strcasecmp(dbName, TSDB_PERFORMANCE_SCHEMA_DB) == 0)); +} diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c9ee83a647..9e03095d2e 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1442,8 +1442,12 @@ static int32_t createDeleteRootLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p } static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { + if (isSystemDb(((SRealTableNode*)pDelete->pFromTable)->table.dbName)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + SScanLogicNode* pScan = NULL; - int32_t code = makeScanLogicNode(pCxt, (SRealTableNode*)pDelete->pFromTable, false, (SLogicNode**)&pScan); + int32_t code = makeScanLogicNode(pCxt, (SRealTableNode*)pDelete->pFromTable, false, (SLogicNode**)&pScan); // set columns to scan if (TSDB_CODE_SUCCESS == code) { From cd85485c07e0ce9b37911da9e1b05df2f52ba5e8 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 23 Apr 2023 11:21:35 +0800 Subject: [PATCH 013/145] add test cases --- tests/system-test/1-insert/delete_systable.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/system-test/1-insert/delete_systable.py diff --git a/tests/system-test/1-insert/delete_systable.py b/tests/system-test/1-insert/delete_systable.py new file mode 100644 index 0000000000..40422a7515 --- /dev/null +++ b/tests/system-test/1-insert/delete_systable.py @@ -0,0 +1,111 @@ + +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import random +import string + +from numpy import logspace +from util import constant +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import TDSetSql + +info_schema_db = "information_schema" +perf_schema_db = "performance_schema" + +info_schema_tables = [ + "ins_dnodes", + "ins_mnodes", + "ins_modules", + "ins_qnodes", + "ins_snodes", + "ins_cluster", + "ins_databases", + "ins_functions", + "ins_indexes", + "ins_stables", + "ins_tables", + "ins_tags", + "ins_columns", + "ins_users", + "ins_grants", + "ins_vgroups", + "ins_configs", + "ins_dnode_variables", + "ins_topics", + "ins_subscriptions", + "ins_streams", + "ins_streams_tasks", + "ins_vnodes", + "ins_user_privileges" +] + +perf_schema_tables = [ + "perf_connections", + "perf_queries", + "perf_consumers", + "perf_trans", + "perf_apps" +] + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def delete_systb(self): + tdSql.execute(f'use {info_schema_db}') + for i in info_schema_tables: + tdSql.error(f'delete from {i}') + tdSql.error(f'delete from {info_schema_db}.{i}') + tdSql.error(f'delete from {i} where ts >= 0') + tdSql.error(f'delete from {info_schema_db}.{i} where ts >= 0') + + tdSql.execute(f'use {perf_schema_db}') + for i in perf_schema_tables: + tdSql.error(f'delete from {i}') + tdSql.error(f'delete from {perf_schema_db}.{i}') + tdSql.error(f'delete from {i} where ts >= 0') + tdSql.error(f'delete from {perf_schema_db}.{i} where ts >= 0') + + def drop_systb(self): + tdSql.execute(f'use {info_schema_db}') + for i in info_schema_tables: + tdSql.error(f'drop table {i}') + tdSql.error(f'drop {info_schema_db}.{i}') + tdSql.error(f'drop database {info_schema_db}') + + tdSql.execute(f'use {perf_schema_db}') + for i in perf_schema_tables: + tdSql.error(f'drop table {i}') + tdSql.error(f'drop table {perf_schema_db}.{i}') + tdSql.error(f'drop database {perf_schema_db}') + + def delete_from_systb(self): + self.delete_systb() + self.drop_systb() + def run(self): + self.delete_from_systb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_from_systb() + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From bdd2c8b81f6f2b754c15b519c65204d727538bdb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sun, 23 Apr 2023 11:22:42 +0800 Subject: [PATCH 014/145] add cases --- tests/parallel_test/cases.task | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index dda4ec3e84..f1a1a11185 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -337,6 +337,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/tb_100w_data_order.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_childtable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_normaltable.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/delete_systable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/drop.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/drop.py -N 3 -M 3 -i False -n 3 From da42cf47992de24ae822dedcb641b69908aec32d Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sun, 23 Apr 2023 11:36:32 +0800 Subject: [PATCH 015/145] update --- tests/system-test/0-others/multilevel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index 350337942b..7ad4eba645 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -250,8 +250,9 @@ class TDTestCase: self.file_distribution_same_level() self.three_level_basic() self.more_than_16_disks() - self.missing_middle_level() self.trim_database() + self.missing_middle_level() + def stop(self): From 4c0cfc18e013cf109f9bfe3caf06a8f34549a48b Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sun, 23 Apr 2023 11:36:59 +0800 Subject: [PATCH 016/145] update --- tests/system-test/0-others/multilevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index 7ad4eba645..8460249d04 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -251,7 +251,7 @@ class TDTestCase: self.three_level_basic() self.more_than_16_disks() self.trim_database() - self.missing_middle_level() + # self.missing_middle_level() From 4bb7a25fc7491f5f1722b8994c646f94fa3c6ec0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 23 Apr 2023 16:30:28 +0800 Subject: [PATCH 017/145] feat: subtable level privilege --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 9 +- source/dnode/mnode/impl/src/mndUser.c | 2 + source/libs/catalog/src/ctgUtil.c | 9 +- source/libs/nodes/src/nodesUtilFuncs.c | 8 +- source/libs/parser/src/parAuthenticator.c | 15 +- source/libs/parser/src/parTranslater.c | 4 +- tests/script/tsim/user/privilege_table.sim | 280 +++++++++++++++++++++ 8 files changed, 307 insertions(+), 21 deletions(-) create mode 100644 tests/script/tsim/user/privilege_table.sim diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 02c097b8d0..df76edffc9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -689,6 +689,7 @@ typedef struct { int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); int32_t tDeserializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); +void tFreeSAlterUserReq(SAlterUserReq* pReq); typedef struct { char user[TSDB_USER_LEN]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9802244b7..189fa1326f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1409,6 +1409,8 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq return 0; } +void tFreeSAlterUserReq(SAlterUserReq *pReq) { taosMemoryFreeClear(pReq->tagCond); } + int32_t tSerializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1635,6 +1637,7 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs int32_t ref = 0; if (tDecodeI32(pDecoder, &ref) < 0) return -1; taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); + taosMemoryFree(key); } } @@ -1831,7 +1834,6 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; } - if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1; tEndEncode(&encoder); @@ -1876,7 +1878,6 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; } - if (!tDecodeIsEnd(&decoder)) { if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1; } else { @@ -2053,12 +2054,12 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp if (pRsp->pFuncExtraInfos == NULL) return -1; if (tDecodeIsEnd(&decoder)) { for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { - SFuncExtraInfo extraInfo = { 0 }; + SFuncExtraInfo extraInfo = {0}; taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); } } else { for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { - SFuncExtraInfo extraInfo = { 0 }; + SFuncExtraInfo extraInfo = {0}; if (tDecodeI32(&decoder, &extraInfo.funcVersion) < 0) return -1; if (tDecodeI64(&decoder, &extraInfo.funcCreatedTime) < 0) return -1; taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index d08227927a..523753d7c6 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -390,6 +390,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &ref, _OVER); taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)); + taosMemoryFree(key); } } @@ -956,6 +957,7 @@ _OVER: mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); } + tFreeSAlterUserReq(&alterReq); mndReleaseUser(pMnode, pOperUser); mndReleaseUser(pMnode, pUser); mndUserFreeObj(&newUser); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 7ef2e34d1e..62896e4307 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -486,12 +486,12 @@ void ctgFreeBatchHash(void* hash) { taosMemoryFreeClear(pRes->pRes); } -void ctgFreeJsonTagVal(void *val) { +void ctgFreeJsonTagVal(void* val) { if (NULL == val) { return; } - STagVal *pVal = (STagVal *)val; + STagVal* pVal = (STagVal*)val; if (TSDB_DATA_TYPE_JSON == pVal->type) { taosMemoryFree(pVal->pData); @@ -545,6 +545,7 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { taosArrayDestroy(*pRes); } *pRes = NULL; + break; } case CTG_TASK_GET_TB_META_BATCH: { SArray* pArray = (SArray*)*pRes; @@ -1363,7 +1364,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { SGetUserAuthRsp* pInfo = &req->authInfo; SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs; char* stbName = NULL; - + char tbFName[TSDB_TABLE_FNAME_LEN]; char dbFName[TSDB_DB_FNAME_LEN]; tNameExtractFullName(&req->pRawReq->tbName, tbFName); @@ -1396,7 +1397,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)&req->pRawReq->tbName; ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP; - + CTG_ERR_RET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 3f571e22ae..422a196c50 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -953,8 +953,12 @@ void nodesDestroyNode(SNode* pNode) { break; case QUERY_NODE_SPLIT_VGROUP_STMT: // no pointer field case QUERY_NODE_SYNCDB_STMT: // no pointer field - case QUERY_NODE_GRANT_STMT: // no pointer field - case QUERY_NODE_REVOKE_STMT: // no pointer field + break; + case QUERY_NODE_GRANT_STMT: + nodesDestroyNode(((SGrantStmt*)pNode)->pTagCond); + break; + case QUERY_NODE_REVOKE_STMT: + nodesDestroyNode(((SRevokeStmt*)pNode)->pTagCond); break; case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index b06d48a690..d182f5bd73 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -70,7 +70,7 @@ static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) { return TSDB_CODE_SUCCESS == authQuery(pCxt, pStmt) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } -static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { +static int32_t mergeStableTagCond(SNode** pWhere, SNode* pTagCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); if (NULL == pLogicCond) { return TSDB_CODE_OUT_OF_MEMORY; @@ -78,7 +78,7 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; pLogicCond->condType = LOGIC_COND_TYPE_AND; - int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, *pTagCond); + int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, pTagCond); if (TSDB_CODE_SUCCESS == code) { code = nodesListMakeAppend(&pLogicCond->pParameterList, *pWhere); } @@ -91,22 +91,17 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { } static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) { - SNode* pTagCondCopy = nodesCloneNode(pTagCond); - if (NULL == pTagCondCopy) { - return TSDB_CODE_OUT_OF_MEMORY; - } - if (NULL == *pWhere) { - *pWhere = pTagCondCopy; + *pWhere = pTagCond; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pWhere) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pWhere)->condType) { - return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy); + return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCond); } - return mergeStableTagCond(pWhere, &pTagCondCopy); + return mergeStableTagCond(pWhere, pTagCond); } static EDealRes authSelectImpl(SNode* pNode, void* pContext) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b44c36dde1..200206755b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1310,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveVectorFunction(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || isInterpPseudoColumnFunc(pNode)) { + if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || + isInterpPseudoColumnFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -6617,6 +6618,7 @@ static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { if (TSDB_CODE_SUCCESS == code) { code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); } + tFreeSAlterUserReq(&req); return code; } diff --git a/tests/script/tsim/user/privilege_table.sim b/tests/script/tsim/user/privilege_table.sim new file mode 100644 index 0000000000..5256cdd21d --- /dev/null +++ b/tests/script/tsim/user/privilege_table.sim @@ -0,0 +1,280 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== init env +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create stable st1(ts timestamp, i int) tags(id int, loc varchar(20)); +sql create table st1s1 using st1 tags(1, 'beijing'); +sql create table st1s2 using st1 tags(2, 'shanghai'); +sql insert into st1s1 values(now, 1) st1s2 values(now, 2); +sql create stable st2(ts timestamp, i int) tags(id int, loc varchar(20)); +sql create table st2s1 using st2 tags(1, 'beijing'); +sql create table st2s2 using st2 tags(2, 'shanghai'); +sql insert into st2s1 values(now, 1) st2s2 values(now, 2); +sql create user wxy pass 'taosdata'; + +print =============== case 1: database unauthorized and table unauthorized +sql close +sql connect wxy + +sql reset query cache; +sql_error select * from test.st1; +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + + +print =============== case 2: database unauthorized and table read privilege +sql close +sql connect + +sql grant read on test.st1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 3: database unauthorized and table read privilege with condition +sql close +sql connect + +sql revoke read on test.st1 from wxy; +sql grant read on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 1 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 4: database unauthorized and table write privilege +sql close +sql connect + +sql revoke read on test.st1 with id = 1 from wxy; +sql grant write on test.st1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql_error select tbname, * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 5: database unauthorized and table write privilege with condition +sql close +sql connect + +sql revoke write on test.st1 from wxy; +sql grant write on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql_error select tbname, * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 6: database read privilege and table unauthorized +sql close +sql connect + +sql revoke write on test.st1 with id = 1 from wxy; +sql grant read on test.* to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 5 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 7: database read privilege and table read privilege +sql close +sql connect + +sql grant read on test.st1 to wxy; + +sql close +sql connect wxy + +sql reset query cache; +sql select * from test.st1; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 8: database read privilege and table read privilege with condition +sql close +sql connect + +sql revoke read on test.st1 from wxy; +sql grant read on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +if $rows != 1 then + return -1 +endi +sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 9: database read privilege and table write privilege +sql close +sql connect + +sql revoke read on test.st1 with id = 1 from wxy; +sql grant write on test.st1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +if $rows != 2 then + return -1 +endi +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql select * from test.st2; +if $rows != 2 then + return -1 +endi +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 10: database read privilege and table write privilege with condition +sql close +sql connect + +sql revoke write on test.st1 from wxy; +sql grant write on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql select * from test.st2; +sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 11: database write privilege and table unauthorized +sql close +sql connect + +sql revoke read on test.* from wxy; +sql revoke write on test.st1 with id = 1 from wxy; +sql grant write on test.* to wxy; + +sql close +sql connect wxy + +sql_error select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 12: database write privilege and table read privilege +sql close +sql connect + +sql grant read on test.st1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 13: database write privilege and table read privilege with condition +sql close +sql connect + +sql revoke read on test.st1 from wxy; +sql grant read on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 14: database write privilege and table write privilege +sql close +sql connect + +sql revoke read on test.st1 with id = 1 from wxy; +sql grant write on test.st1 to wxy; + +sql close +sql connect wxy + +sql_error select * from test.st1; +sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +print =============== case 15: database write privilege and table write privilege with condition +sql close +sql connect + +sql revoke write on test.st1 from wxy; +sql grant write on test.st1 with id = 1 to wxy; + +sql close +sql connect wxy + +sql_error select * from test.st1; +sql insert into test.st1s1 values(now, 10); +sql_error insert into test.st1s2 values(now, 20); +sql_error select * from test.st2; +sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 3eca2ff8189b280819ba1c26dd1be3efc31bbb37 Mon Sep 17 00:00:00 2001 From: jiacy-jcy <714897623@qq.com> Date: Sun, 23 Apr 2023 16:54:50 +0800 Subject: [PATCH 018/145] update --- tests/system-test/0-others/multilevel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/multilevel.py b/tests/system-test/0-others/multilevel.py index 8460249d04..7ad4eba645 100644 --- a/tests/system-test/0-others/multilevel.py +++ b/tests/system-test/0-others/multilevel.py @@ -251,7 +251,7 @@ class TDTestCase: self.three_level_basic() self.more_than_16_disks() self.trim_database() - # self.missing_middle_level() + self.missing_middle_level() From 77e03bfd782eb5609e596ea69f90b44967cdc351 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sun, 23 Apr 2023 20:14:49 +0800 Subject: [PATCH 019/145] opti:change push mgr to consume msg for subscribe --- source/dnode/mnode/impl/src/mndConsumer.c | 2 - source/dnode/vnode/src/inc/tq.h | 2 + source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 62 ++++++++++---- source/dnode/vnode/src/tq/tqPush.c | 100 +++++++++++----------- source/dnode/vnode/src/tq/tqUtil.c | 27 +++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 10 +-- 7 files changed, 123 insertions(+), 81 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 65a2fa72a2..ca71e17d7e 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -449,7 +449,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { // 1. check consumer status int32_t status = atomic_load_32(&pConsumer->status); -#if 1 if (status == MQ_CONSUMER_STATUS__LOST_REBD) { mInfo("try to recover consumer:0x%" PRIx64, consumerId); SMqConsumerRecoverMsg *pRecoverMsg = rpcMallocCont(sizeof(SMqConsumerRecoverMsg)); @@ -463,7 +462,6 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &pRpcMsg); } -#endif if (status != MQ_CONSUMER_STATUS__READY) { mInfo("consumer:0x%" PRIx64 " not ready, status: %s", consumerId, mndConsumerStatusName(status)); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index acc0d29382..e1b1092c28 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -100,6 +100,7 @@ typedef struct { SWalRef* pRef; STqPushHandle pushHandle; // push STqExecHandle execHandle; // exec + SRpcMsg* msg; } STqHandle; typedef struct { @@ -114,6 +115,7 @@ struct STQ { int64_t walLogLastVer; SRWLatch lock; SHashObj* pPushMgr; // consumerId -> STqPushEntry + SArray * pPushArray; SHashObj* pHandle; // subKey -> STqHandle SHashObj* pCheckInfo; // topic -> SAlterCheckInfo STqOffsetStore* pOffsetStore; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 416bc6cdc7..b24cb7e136 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -214,6 +214,7 @@ int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit); +int32_t tqProcessSubmitReqForSubscribe(STQ* pTq); int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a78239a4b5..ae4a7e1d61 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -71,6 +71,11 @@ static void destroyTqHandle(void* data) { walCloseReader(pData->pWalReader); tqCloseReader(pData->execHandle.pTqReader); } + if(pData->msg != NULL) { + rpcFreeCont(pData->msg->pCont); + taosMemoryFree(pData->msg); + pData->msg = NULL; + } } static void tqPushEntryFree(void* data) { @@ -104,6 +109,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pHandle, destroyTqHandle); + pTq->pPushArray = taosArrayInit(8, POINTER_BYTES); + taosInitRWLatch(&pTq->lock); pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); taosHashSetFreeFp(pTq->pPushMgr, tqPushEntryFree); @@ -152,6 +159,7 @@ void tqClose(STQ* pTq) { tqMetaClose(pTq); streamMetaClose(pTq->pStreamMeta); taosMemoryFree(pTq); + taosArrayDestroy(pTq->pPushArray); } void tqNotifyClose(STQ* pTq) { @@ -350,25 +358,15 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } // 2. check re-balance status -// taosRLockLatch(&pTq->lock); + taosRLockLatch(&pTq->lock); if (pHandle->consumerId != consumerId) { tqDebug("ERROR tmq poll: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; -// taosRUnLockLatch(&pTq->lock); + taosRUnLockLatch(&pTq->lock); return -1; } -// taosRUnLockLatch(&pTq->lock); - - // 3. update the epoch value -// taosWLockLatch(&pTq->lock); - int32_t savedEpoch = pHandle->epoch; - if (savedEpoch < reqEpoch) { - tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, - reqEpoch); - pHandle->epoch = reqEpoch; - } -// taosWUnLockLatch(&pTq->lock); + taosRUnLockLatch(&pTq->lock); char buf[80]; tFormatOffset(buf, 80, &reqOffset); @@ -560,8 +558,20 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg atomic_store_32(&pHandle->epoch, -1); // remove if it has been register in the push manager, and return one empty block to consumer - //tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); - +// tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); + for(size_t i = 0; i < taosArrayGetSize(pTq->pPushArray); i++) { + void* handle = taosArrayGetP(pTq->pPushArray, i); + if(handle == pHandle) { + tqInfo("vgId:%d remove handle when switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); + taosArrayRemove(pTq->pPushArray, i); + break; + } + } + if(pHandle->msg != NULL) { + rpcFreeCont(pHandle->msg->pCont); + taosMemoryFree(pHandle->msg); + pHandle->msg = NULL; + } atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); @@ -1067,6 +1077,28 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } +int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + tqDebug("vgId:%d start set submit for subscribe", vgId); + + taosWLockLatch(&pTq->lock); + for(size_t i = 0; i < taosArrayGetSize(pTq->pPushArray); i++){ + STqHandle* pHandle = (STqHandle*)taosArrayGetP(pTq->pPushArray, i); + if(pHandle->msg == NULL){ + tqError("pHandle->msg should not be null"); + } + SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen}; + tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + taosMemoryFree(pHandle->msg); + pHandle->msg = NULL; + } + taosArrayClear(pTq->pPushArray); + // unlock + taosWUnLockLatch(&pTq->lock); + + return 0; +} + int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { #if 0 void* pIter = NULL; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 7a1a6b7454..d2d17792d3 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -268,59 +268,61 @@ static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int6 } } + int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); - int32_t len = msgLen - sizeof(SSubmitReq2Msg); - int32_t vgId = TD_VID(pTq->pVnode); +// void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); +// int32_t len = msgLen - sizeof(SSubmitReq2Msg); +// int32_t vgId = TD_VID(pTq->pVnode); if (msgType == TDMT_VND_SUBMIT) { + tqProcessSubmitReqForSubscribe(pTq); // lock push mgr to avoid potential msg lost - taosWLockLatch(&pTq->lock); - - int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr); - if (numOfRegisteredPush > 0) { - tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d", - vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush); - - void* data = taosMemoryMalloc(len); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory, vgId:%d", vgId); - taosWUnLockLatch(&pTq->lock); - return -1; - } - - memcpy(data, pReq, len); - - SArray* cachedKey = taosArrayInit(0, sizeof(SItem)); - void* pIter = NULL; - - while (1) { - pIter = taosHashIterate(pTq->pPushMgr, pIter); - if (pIter == NULL) { - break; - } - - STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; - - STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey)); - if (pHandle == NULL) { - tqDebug("vgId:%d, failed to find handle %s in pushing data to consumer, ignore", pTq->pVnode->config.vgId, - pPushEntry->subKey); - continue; - } - - STqExecHandle* pExec = &pHandle->execHandle; - doPushDataForEntry(pIter, pExec, pTq, ver, vgId, data, len, cachedKey); - } - - doRemovePushedEntry(cachedKey, pTq); - taosArrayDestroyEx(cachedKey, freeItem); - taosMemoryFree(data); - } - - // unlock - taosWUnLockLatch(&pTq->lock); +// taosWLockLatch(&pTq->lock); +// +// int32_t numOfRegisteredPush = taosHashGetSize(pTq->pPushMgr); +// if (numOfRegisteredPush > 0) { +// tqDebug("vgId:%d tq push msg version:%" PRId64 " type:%s, head:%p, body:%p len:%d, numOfPushed consumers:%d", +// vgId, ver, TMSG_INFO(msgType), msg, pReq, len, numOfRegisteredPush); +// +// void* data = taosMemoryMalloc(len); +// if (data == NULL) { +// terrno = TSDB_CODE_OUT_OF_MEMORY; +// tqError("failed to copy data for stream since out of memory, vgId:%d", vgId); +// taosWUnLockLatch(&pTq->lock); +// return -1; +// } +// +// memcpy(data, pReq, len); +// +// SArray* cachedKey = taosArrayInit(0, sizeof(SItem)); +// void* pIter = NULL; +// +// while (1) { +// pIter = taosHashIterate(pTq->pPushMgr, pIter); +// if (pIter == NULL) { +// break; +// } +// +// STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; +// +// STqHandle* pHandle = taosHashGet(pTq->pHandle, pPushEntry->subKey, strlen(pPushEntry->subKey)); +// if (pHandle == NULL) { +// tqDebug("vgId:%d, failed to find handle %s in pushing data to consumer, ignore", pTq->pVnode->config.vgId, +// pPushEntry->subKey); +// continue; +// } +// +// STqExecHandle* pExec = &pHandle->execHandle; +// doPushDataForEntry(pIter, pExec, pTq, ver, vgId, data, len, cachedKey); +// } +// +// doRemovePushedEntry(cachedKey, pTq); +// taosArrayDestroyEx(cachedKey, freeItem); +// taosMemoryFree(data); +// } +// +// // unlock +// taosWUnLockLatch(&pTq->lock); } tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, (int)taosHashGetSize(pTq->pStreamMeta->pTasks)); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 128ddedf6d..f76e641f2b 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -169,22 +169,29 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - // lock -// taosWLockLatch(&pTq->lock); - qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); if(code != 0) { goto end; } - // till now, all data has been transferred to consumer, new data needs to push client once arrived. -// if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && -// dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { -// //code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); -// taosWUnLockLatch(&pTq->lock); -// return code; -// } +// till now, all data has been transferred to consumer, new data needs to push client once arrived. + if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { +// code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + // lock + taosWLockLatch(&pTq->lock); + if(pHandle->msg != NULL){ + tqError("pHandle->msg should be null"); + } + pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); + pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); + memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); + taosArrayPush(pTq->pPushArray, &pHandle); + taosWUnLockLatch(&pTq->lock); + return code; + } code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 579ef8a952..b29081170d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -447,11 +447,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp walApplyVer(pVnode->pWal, version); - //if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { - /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ - //vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); - //return -1; - //} + if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { +// /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ + vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); + return -1; + } // commit if need if (needCommit) { From d23ae400b849f931ccae3db2ff3cd1bdaca00200 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 09:22:59 +0800 Subject: [PATCH 020/145] fix: table meta double free issue --- source/libs/catalog/inc/catalogInt.h | 1 + source/libs/catalog/src/ctgAsync.c | 69 ++++++++++++++++++++++++++++ source/libs/catalog/src/ctgCache.c | 3 +- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 1eaf45dafe..c548a6c696 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -359,6 +359,7 @@ typedef struct SCtgSubRes { struct SCtgTask { CTG_TASK_TYPE type; + bool subTask; int32_t taskId; SCtgJob* pJob; void* taskCtx; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index affcfeb8ac..56c79eac1f 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -699,6 +699,10 @@ _return: } int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableMeta) { pJob->jobRes.pTableMeta = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); @@ -714,6 +718,10 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { } int32_t ctgDumpTbMetasRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableMeta = pTask->res; @@ -722,6 +730,10 @@ int32_t ctgDumpTbMetasRes(SCtgTask* pTask) { } int32_t ctgDumpDbVgRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbVgroup) { pJob->jobRes.pDbVgroup = taosArrayInit(pJob->dbVgNum, sizeof(SMetaRes)); @@ -737,6 +749,10 @@ int32_t ctgDumpDbVgRes(SCtgTask* pTask) { } int32_t ctgDumpTbHashRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableHash) { pJob->jobRes.pTableHash = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); @@ -752,6 +768,10 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { } int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableHash = pTask->res; @@ -760,6 +780,10 @@ int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { } int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { SArray* pRes = taosArrayInit(pJob->tbIndexNum, sizeof(SMetaRes)); @@ -779,6 +803,10 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { } int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableCfg) { SArray* pRes = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); @@ -798,6 +826,10 @@ int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { } int32_t ctgDumpTbTagRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableTag) { SArray* pRes = taosArrayInit(pJob->tbTagNum, sizeof(SMetaRes)); @@ -818,6 +850,10 @@ int32_t ctgDumpTbTagRes(SCtgTask* pTask) { int32_t ctgDumpIndexRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pIndex) { pJob->jobRes.pIndex = taosArrayInit(pJob->indexNum, sizeof(SMetaRes)); @@ -833,6 +869,10 @@ int32_t ctgDumpIndexRes(SCtgTask* pTask) { } int32_t ctgDumpQnodeRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pQnodeList) { pJob->jobRes.pQnodeList = taosArrayInit(1, sizeof(SMetaRes)); @@ -848,6 +888,10 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) { } int32_t ctgDumpDnodeRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDnodeList) { pJob->jobRes.pDnodeList = taosArrayInit(1, sizeof(SMetaRes)); @@ -863,6 +907,10 @@ int32_t ctgDumpDnodeRes(SCtgTask* pTask) { } int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbCfg) { pJob->jobRes.pDbCfg = taosArrayInit(pJob->dbCfgNum, sizeof(SMetaRes)); @@ -878,6 +926,10 @@ int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { } int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbInfo) { pJob->jobRes.pDbInfo = taosArrayInit(pJob->dbInfoNum, sizeof(SMetaRes)); @@ -893,6 +945,10 @@ int32_t ctgDumpDbInfoRes(SCtgTask* pTask) { } int32_t ctgDumpUdfRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pUdfList) { pJob->jobRes.pUdfList = taosArrayInit(pJob->udfNum, sizeof(SMetaRes)); @@ -908,6 +964,10 @@ int32_t ctgDumpUdfRes(SCtgTask* pTask) { } int32_t ctgDumpUserRes(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pUser) { pJob->jobRes.pUser = taosArrayInit(pJob->userNum, sizeof(SMetaRes)); @@ -923,6 +983,10 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) { } int32_t ctgDumpSvrVer(SCtgTask* pTask) { + if (pTask->subTask) { + return TSDB_CODE_SUCCESS; + } + SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pSvrVer) { pJob->jobRes.pSvrVer = taosMemoryCalloc(1, sizeof(SMetaRes)); @@ -2239,6 +2303,8 @@ int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { if (inCache) { pTask->res = rsp.pRawRes; + ctgTaskDebug("Final res got, pass:%d, pCond:%p", rsp.pRawRes->pass, rsp.pRawRes->pCond); + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } @@ -2475,6 +2541,9 @@ int32_t ctgLaunchSubTask(SCtgTask* pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, } SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); + if (newTask) { + pSub->subTask = true; + } CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask)); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index ee864d985e..0be2a78d48 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -765,8 +765,9 @@ _return: *inCache = false; CTG_CACHE_NHIT_INC(CTG_CI_USER, 1); + ctgDebug("Get user from cache failed, user:%s, metaNotExists:%d, code:%d", pReq->user, pRes->metaNotExists, code); - return TSDB_CODE_SUCCESS; + return code; } void ctgDequeue(SCtgCacheOperation **op) { From 1251245c2a2c42388dddfd152bb3cec910cc5245 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 24 Apr 2023 09:49:03 +0800 Subject: [PATCH 021/145] move code to parser --- source/libs/parser/src/parTranslater.c | 5 +++++ source/libs/planner/src/planLogicCreater.c | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b44c36dde1..52451d35a8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3792,6 +3792,11 @@ static int32_t translateDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet } static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { + // check delete from system tables + if (isSystemDb(((SRealTableNode*)pDelete->pFromTable)->table.dbName)) { + return TSDB_CODE_TSC_INVALID_OPERATION; + } + pCxt->pCurrStmt = (SNode*)pDelete; int32_t code = translateFrom(pCxt, pDelete->pFromTable); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 9e03095d2e..39783868b3 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1442,12 +1442,8 @@ static int32_t createDeleteRootLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p } static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pDelete, SLogicNode** pLogicNode) { - if (isSystemDb(((SRealTableNode*)pDelete->pFromTable)->table.dbName)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - SScanLogicNode* pScan = NULL; - int32_t code = makeScanLogicNode(pCxt, (SRealTableNode*)pDelete->pFromTable, false, (SLogicNode**)&pScan); + int32_t code = makeScanLogicNode(pCxt, (SRealTableNode*)pDelete->pFromTable, false, (SLogicNode**)&pScan); // set columns to scan if (TSDB_CODE_SUCCESS == code) { From 4b166e567e1ee6eb77c1bdeffa494b59e32db563 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:14:02 +0800 Subject: [PATCH 022/145] fix: fix client retry issue --- source/client/src/clientMain.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 60c7b44b3d..54ae1ab4b3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -975,8 +975,10 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (TSDB_CODE_SUCCESS == code) { pRequest->stmtType = pRequest->pQuery->pRoot->type; - phaseAsyncQuery(pWrapper); - } else { + code = phaseAsyncQuery(pWrapper); + } + + if (TSDB_CODE_SUCCESS != code) { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), pRequest->requestId); destorySqlCallbackWrapper(pWrapper); From af5bcb80bffddce7687a2b41efc220851ff9cb1c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Apr 2023 10:26:40 +0800 Subject: [PATCH 023/145] opti:test cases for tmq --- .../6-cluster/clusterCommonCreate.py | 40 ++++++++--------- tests/system-test/7-tmq/subscribeDb3.py | 45 ++++++++++--------- tests/system-test/7-tmq/tmqCommon.py | 22 +++++---- tests/system-test/7-tmq/tmqConsumerGroup.py | 2 +- tests/system-test/7-tmq/tmqDnodeRestart1.py | 4 +- 5 files changed, 56 insertions(+), 57 deletions(-) diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py index 6e699e2396..a06c1233d8 100644 --- a/tests/system-test/6-cluster/clusterCommonCreate.py +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -94,26 +94,26 @@ class ClusterComCreate: tdLog.info(shellCmd) os.system(shellCmd) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): - break - else: - time.sleep(0.1) - return - - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2 : - print(tdSql.getData(0, 1), tdSql.getData(1, 1)) - if tdSql.getData(1, 1) == 1: - break - time.sleep(0.1) - return + # def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): + # break + # else: + # time.sleep(0.1) + # return + # + # def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if tdSql.getRows() == 2 : + # print(tdSql.getData(0, 1), tdSql.getData(1, 1)) + # if tdSql.getData(1, 1) == 1: + # break + # time.sleep(0.1) + # return def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): if dropFlag == 1: diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 1de9b62bcd..22004a95f1 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -10,6 +10,7 @@ from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +from tmqCommon import * class TDTestCase: hostname = socket.gethostname() @@ -67,26 +68,26 @@ class TDTestCase: tdLog.info("consume info sql: %s"%sql) tdSql.query(sql) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): - break - else: - time.sleep(0.1) - return - - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): - while 1: - tdSql.query("select * from %s.notifyinfo"%cdbName) - #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) - if tdSql.getRows() == 2 : - tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1))) - if tdSql.getData(1, 1) == 1: - break - time.sleep(0.1) - return + # def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): + # break + # else: + # time.sleep(0.1) + # return + # + # def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): + # while 1: + # tdSql.query("select * from %s.notifyinfo"%cdbName) + # #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + # if tdSql.getRows() == 2 : + # tdLog.info("row[0][1]: %d, row[1][1]: %d"%(tdSql.getData(0, 1), tdSql.getData(1, 1))) + # if tdSql.getData(1, 1) == 1: + # break + # time.sleep(0.1) + # return def selectConsumeResult(self,expectRows,cdbName='cdb'): resultList=[] @@ -233,7 +234,7 @@ class TDTestCase: self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) tdLog.info("wait the notify info of start consume") - self.getStartConsumeNotifyFromTmqsim() + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("pkill consume processor") if (platform.system().lower() == 'windows'): @@ -311,7 +312,7 @@ class TDTestCase: # time.sleep(6) tdLog.info("start to wait commit notify") - self.getStartCommitNotifyFromTmqsim() + tmqCom.getStartCommitNotifyFromTmqsim() tdLog.info("pkill consume processor") if (platform.system().lower() == 'windows'): diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index f63c70a4c6..44eef8bf24 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -145,31 +145,29 @@ class TMQCom: processID = subprocess.check_output(psCmd, shell=True).decode("utf-8") tdLog.debug("%s is stopped by kill -INT" % (processorName)) - def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb',rows=1): + def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): loopFlag = 1 while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) actRows = tdSql.getRows() - if (actRows >= rows): - for i in range(actRows): - if tdSql.getData(i, 1) == 0: - loopFlag = 0 - break + for i in range(actRows): + if tdSql.getData(i, 1) == 0: + loopFlag = 0 + break time.sleep(0.02) return - def getStartCommitNotifyFromTmqsim(self,cdbName='cdb',rows=2): + def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): loopFlag = 1 while loopFlag: tdSql.query("select * from %s.notifyinfo"%cdbName) #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) actRows = tdSql.getRows() - if (actRows >= rows): - for i in range(actRows): - if tdSql.getData(i, 1) == 1: - loopFlag = 0 - break + for i in range(actRows): + if tdSql.getData(i, 1) == 1: + loopFlag = 0 + break time.sleep(0.02) return diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index f05f600f27..ae9671bcf4 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -100,7 +100,7 @@ class TDTestCase: tdLog.info("wait consumer commit notify") # tmqCom.getStartCommitNotifyFromTmqsim(rows=4) - tmqCom.getStartConsumeNotifyFromTmqsim(rows=2) + tmqCom.getStartConsumeNotifyFromTmqsim() tdLog.info("pkill one consume processor") tmqCom.stopTmqSimProcess('tmq_sim_new') diff --git a/tests/system-test/7-tmq/tmqDnodeRestart1.py b/tests/system-test/7-tmq/tmqDnodeRestart1.py index cff55a1239..2bde32800b 100644 --- a/tests/system-test/7-tmq/tmqDnodeRestart1.py +++ b/tests/system-test/7-tmq/tmqDnodeRestart1.py @@ -121,7 +121,7 @@ class TDTestCase: tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) # time.sleep(3) - tmqCom.getStartCommitNotifyFromTmqsim('cdb',1) + tmqCom.getStartCommitNotifyFromTmqsim() tdLog.info("create some new child table and insert data for latest mode") paraDict["batchNum"] = 100 @@ -205,7 +205,7 @@ class TDTestCase: tdLog.info("start consume processor") tmqCom.startTmqSimProcess(pollDelay=paraDict['pollDelay'],dbName=paraDict["dbName"],showMsg=paraDict['showMsg'], showRow=paraDict['showRow'],snapshot=paraDict['snapshot']) - tmqCom.getStartCommitNotifyFromTmqsim('cdb',1) + tmqCom.getStartCommitNotifyFromTmqsim() tdLog.info("create some new child table and insert data for latest mode") paraDict["batchNum"] = 10 From 3b8e8018b2991107b542c7c5830cbaae078d696b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:34:07 +0800 Subject: [PATCH 024/145] fix: get user errcode issue --- source/libs/catalog/src/ctgCache.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 0be2a78d48..ec087c1168 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -727,6 +727,7 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid } int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { + int32_t code = 0; if (IS_SYS_DBNAME(pReq->tbName.dbname)) { *inCache = true; pRes->pRawRes->pass = true; @@ -751,7 +752,7 @@ int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, CTG_LOCK(CTG_READ, &pUser->lock); memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); - int32_t code = ctgChkSetAuthRes(pCtg, &req, pRes); + code = ctgChkSetAuthRes(pCtg, &req, pRes); CTG_UNLOCK(CTG_READ, &pUser->lock); CTG_ERR_JRET(code); From 905df6f7ac9b233b973d343b3e7990d79eef9b2a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:37:58 +0800 Subject: [PATCH 025/145] fix: memory leak issue --- source/libs/catalog/src/ctgUtil.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 62896e4307..4a5ddbac34 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1371,6 +1371,8 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { tNameGetFullDbName(&req->pRawReq->tbName, dbFName); while (true) { + taosMemoryFreeClear(pMeta); + char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName)); if (pCond) { if (strlen(pCond) > 1) { @@ -1412,7 +1414,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { if (req->onlyCache) { res->metaNotExists = true; ctgDebug("suid %" PRIu64 " name not in cache for auth", pMeta->suid); - return TSDB_CODE_SUCCESS; + goto _return; } continue; From 8480c42234a2a948f296b5d318dd2125eeaafa5c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:41:28 +0800 Subject: [PATCH 026/145] fix: memory leak issue --- source/libs/catalog/src/ctgUtil.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 4a5ddbac34..0ec7a92b84 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1376,11 +1376,11 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { char* pCond = taosHashGet(pTbs, tbFName, strlen(tbFName)); if (pCond) { if (strlen(pCond) > 1) { - CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + CTG_ERR_JRET(nodesStringToNode(pCond, &res->pRawRes->pCond)); } res->pRawRes->pass = true; - return TSDB_CODE_SUCCESS; + goto _return; } if (stbName) { @@ -1388,19 +1388,19 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { goto _return; } - CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); + CTG_ERR_JRET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); if (NULL == pMeta) { if (req->onlyCache) { res->metaNotExists = true; ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); - return TSDB_CODE_SUCCESS; + goto _return; } SCtgTbMetaCtx ctx = {0}; ctx.pName = (SName*)&req->pRawReq->tbName; ctx.flag = CTG_FLAG_UNKNOWN_STB | CTG_FLAG_SYNC_OP; - CTG_ERR_RET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); + CTG_ERR_JRET(ctgGetTbMeta(pCtg, req->pConn, &ctx, &pMeta)); } if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { @@ -1431,6 +1431,7 @@ int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { _return: taosMemoryFree(pMeta); + taosMemoryFree(stbName); CTG_RET(code); } From 24aff3d680d3cb2abd9d1fae2145bfcd2a935b7c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 10:46:32 +0800 Subject: [PATCH 027/145] fix: memory leak issue --- source/libs/catalog/src/ctgUtil.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 0ec7a92b84..a42a189e75 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -528,11 +528,18 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { } break; } + case CTG_TASK_GET_USER: { + if (*pRes) { + SUserAuthRes* pAuth = (SUserAuthRes *)*pRes; + nodesDestroyNode(pAuth->pCond); + taosMemoryFreeClear(*pRes); + } + break; + } case CTG_TASK_GET_TB_HASH: case CTG_TASK_GET_DB_INFO: case CTG_TASK_GET_INDEX_INFO: case CTG_TASK_GET_UDF: - case CTG_TASK_GET_USER: case CTG_TASK_GET_SVR_VER: case CTG_TASK_GET_TB_META: { taosMemoryFreeClear(*pRes); From 5663550de22a2c88cb704d410b39c6aa1a0a0788 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 15:47:02 +0800 Subject: [PATCH 028/145] enh(query): opt query perf by allocate the SLDataIter when opening reader. --- source/dnode/vnode/inc/vnode.h | 1 - source/dnode/vnode/src/inc/tsdb.h | 20 +++- source/dnode/vnode/src/tsdb/tsdbCache.c | 10 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 11 ++- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 101 ++++++-------------- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +- 6 files changed, 68 insertions(+), 80 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index fb2c2f4be3..aecfb9c3e5 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -162,7 +162,6 @@ int32_t metaTbCursorPrev(SMTbCursor *pTbCur, ETableType jumpTableType); #endif // tsdb -// typedef struct STsdb STsdb; typedef struct STsdbReader STsdbReader; #define TSDB_DEFAULT_STT_FILE 8 diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2a85b191a4..8f6f9daf84 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -705,7 +705,6 @@ typedef struct SSttBlockLoadInfo { typedef struct SMergeTree { int8_t backward; SRBTree rbt; - SArray *pIterList; SLDataIter *pIter; bool destroyLoadInfo; SSttBlockLoadInfo *pLoadInfo; @@ -751,9 +750,25 @@ struct SDiskDataBuilder { SBlkInfo bi; }; +typedef struct SLDataIter { + SRBTreeNode node; + SSttBlk *pSttBlk; + SDataFReader *pReader; + int32_t iStt; + int8_t backward; + int32_t iSttBlk; + int32_t iRow; + SRowInfo rInfo; + uint64_t uid; + STimeWindow timeWindow; + SVersionRange verRange; + SSttBlockLoadInfo *pBlockLoadInfo; + bool ignoreEarlierTs; +} SLDataIter; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr, bool strictTimeRange); + bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); @@ -782,6 +797,7 @@ typedef struct SCacheRowsReader { STableKeyInfo *pTableList; // table id list int32_t numOfTables; SSttBlockLoadInfo *pLoadInfo; + SLDataIter *pDataIter; STsdbReadSnap *pReadSnap; SDataFReader *pDataFReader; SDataFReader *pDataFReaderLast; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3c7edd931b..42f332c54a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -598,6 +598,7 @@ typedef struct { SMergeTree mergeTree; SMergeTree *pMergeTree; SSttBlockLoadInfo *pLoadInfo; + SLDataIter* pDataIter; int64_t lastTs; } SFSLastNextRowIter; @@ -645,7 +646,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa } tMergeTreeOpen(&state->mergeTree, 1, *state->pDataFReader, state->suid, state->uid, &(STimeWindow){.skey = state->lastTs, .ekey = TSKEY_MAX}, - &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true); + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, state->pLoadInfo, false, NULL, true, state->pDataIter); state->pMergeTree = &state->mergeTree; state->state = SFSLASTNEXTROW_BLOCKROW; } @@ -1211,7 +1212,7 @@ typedef struct { } CacheNextRowIter; static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTsdb, STSchema *pTSchema, tb_uid_t suid, - SSttBlockLoadInfo *pLoadInfo, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader, + SSttBlockLoadInfo *pLoadInfo, SLDataIter* pLDataIter, STsdbReadSnap *pReadSnap, SDataFReader **pDataFReader, SDataFReader **pDataFReaderLast, int64_t lastTs) { int code = 0; @@ -1274,6 +1275,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsLastState.pLoadInfo = pLoadInfo; pIter->fsLastState.pDataFReader = pDataFReaderLast; pIter->fsLastState.lastTs = lastTs; + pIter->fsLastState.pDataIter = pLDataIter; pIter->fsState.state = SFSNEXTROW_FS; pIter->fsState.pTsdb = pTsdb; @@ -1465,7 +1467,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader, + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, &pr->pDataFReaderLast, pr->lastTs); do { @@ -1622,7 +1624,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach TSKEY lastRowTs = TSKEY_MAX; CacheNextRowIter iter = {0}; - nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pReadSnap, &pr->pDataFReader, + nextRowIterOpen(&iter, uid, pTsdb, pTSchema, pr->suid, pr->pLoadInfo, pr->pDataIter, pr->pReadSnap, &pr->pDataFReader, &pr->pDataFReaderLast, pr->lastTs); do { diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 95981c2f08..64d30c77a3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -187,13 +187,21 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, } } - int32_t numOfStt = ((SVnode*)pVnode)->config.sttTrigger; + SVnodeCfg* pCfg = &((SVnode*)pVnode)->config; + + int32_t numOfStt = pCfg->sttTrigger; p->pLoadInfo = tCreateLastBlockLoadInfo(p->pSchema, NULL, 0, numOfStt); if (p->pLoadInfo == NULL) { tsdbCacherowsReaderClose(p); return TSDB_CODE_OUT_OF_MEMORY; } + p->pDataIter = taosMemoryCalloc(pCfg->sttTrigger, sizeof(SLDataIter)); + if (p->pDataIter == NULL) { + tsdbCacherowsReaderClose(p); + return TSDB_CODE_OUT_OF_MEMORY; + } + p->idstr = taosStrdup(idstr); taosThreadMutexInit(&p->readerMutex, NULL); @@ -215,6 +223,7 @@ void* tsdbCacherowsReaderClose(void* pReader) { taosMemoryFree(p->pSchema); } + taosMemoryFreeClear(p->pDataIter); taosMemoryFree(p->pCurrSchema); destroyLastBlockLoadInfo(p->pLoadInfo); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index eb383df48d..e354a50253 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -16,22 +16,6 @@ #include "tsdb.h" // SLDataIter ================================================= -struct SLDataIter { - SRBTreeNode node; - SSttBlk *pSttBlk; - SDataFReader *pReader; - int32_t iStt; - int8_t backward; - int32_t iSttBlk; - int32_t iRow; - SRowInfo rInfo; - uint64_t uid; - STimeWindow timeWindow; - SVersionRange verRange; - SSttBlockLoadInfo *pBlockLoadInfo; - bool ignoreEarlierTs; -}; - SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfSttTrigger) { SSttBlockLoadInfo *pLoadInfo = taosMemoryCalloc(numOfSttTrigger, sizeof(SSttBlockLoadInfo)); @@ -268,25 +252,19 @@ static int32_t binarySearchForStartRowIndex(uint64_t *uidList, int32_t num, uint } } -int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, +int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo *pBlockLoadInfo, const char *idStr, bool strictTimeRange) { int32_t code = TSDB_CODE_SUCCESS; - *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); - if (*pIter == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } + pIter->uid = uid; + pIter->pReader = pReader; + pIter->iStt = iStt; + pIter->backward = backward; + pIter->verRange = *pRange; + pIter->timeWindow = *pTimeWindow; - (*pIter)->uid = uid; - (*pIter)->pReader = pReader; - (*pIter)->iStt = iStt; - (*pIter)->backward = backward; - (*pIter)->verRange = *pRange; - (*pIter)->timeWindow = *pTimeWindow; - - (*pIter)->pBlockLoadInfo = pBlockLoadInfo; + pIter->pBlockLoadInfo = pBlockLoadInfo; if (!pBlockLoadInfo->sttBlockLoaded) { int64_t st = taosGetTimestampUs(); @@ -294,7 +272,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); if (code) { - goto _exit; + return code; } // only apply to the child tables, ordinary tables will not incur this filter procedure. @@ -310,7 +288,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t // no qualified stt block existed taosArrayClear(pBlockLoadInfo->aSttBlk); - (*pIter)->iSttBlk = -1; + pIter->iSttBlk = -1; double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug("load the last file info completed, elapsed time:%.2fms, %s", el, idStr); return code; @@ -343,31 +321,27 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); // find the start block - (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); - if ((*pIter)->iSttBlk != -1) { - (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk); - (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; + pIter->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); + if (pIter->iSttBlk != -1) { + pIter->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, pIter->iSttBlk); + pIter->iRow = (pIter->backward) ? pIter->pSttBlk->nRow : -1; - if ((!backward) && ((strictTimeRange && (*pIter)->pSttBlk->minKey >= (*pIter)->timeWindow.ekey) || - (!strictTimeRange && (*pIter)->pSttBlk->minKey > (*pIter)->timeWindow.ekey))) { - (*pIter)->pSttBlk = NULL; + if ((!backward) && ((strictTimeRange && pIter->pSttBlk->minKey >= pIter->timeWindow.ekey) || + (!strictTimeRange && pIter->pSttBlk->minKey > pIter->timeWindow.ekey))) { + pIter->pSttBlk = NULL; } - if (backward && ((strictTimeRange && (*pIter)->pSttBlk->maxKey <= (*pIter)->timeWindow.skey) || - (!strictTimeRange && (*pIter)->pSttBlk->maxKey < (*pIter)->timeWindow.skey))) { - (*pIter)->pSttBlk = NULL; - (*pIter)->ignoreEarlierTs = true; + if (backward && ((strictTimeRange && pIter->pSttBlk->maxKey <= pIter->timeWindow.skey) || + (!strictTimeRange && pIter->pSttBlk->maxKey < pIter->timeWindow.skey))) { + pIter->pSttBlk = NULL; + pIter->ignoreEarlierTs = true; } } return code; - -_exit: - taosMemoryFree(*pIter); - return code; } -void tLDataIterClose(SLDataIter *pIter) { taosMemoryFree(pIter); } +void tLDataIterClose(SLDataIter *pIter) { /*taosMemoryFree(pIter); */} void tLDataIterNextBlock(SLDataIter *pIter, const char *idStr) { int32_t step = pIter->backward ? -1 : 1; @@ -594,43 +568,38 @@ static FORCE_INLINE int32_t tLDataIterDescCmprFn(const SRBTreeNode *p1, const SR int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, - bool destroyLoadInfo, const char *idStr, bool strictTimeRange) { + bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter) { + int32_t code = TSDB_CODE_SUCCESS; + pMTree->backward = backward; pMTree->pIter = NULL; - pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); - if (pMTree->pIterList == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pMTree->idStr = idStr; + if (!pMTree->backward) { // asc tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); } else { // desc tRBTreeCreate(&pMTree->rbt, tLDataIterDescCmprFn); } - int32_t code = TSDB_CODE_SUCCESS; pMTree->pLoadInfo = pBlockLoadInfo; pMTree->destroyLoadInfo = destroyLoadInfo; pMTree->ignoreEarlierTs = false; for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file - struct SLDataIter *pIter = NULL; - code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, + memset(&pLDataIter[i], 0, sizeof(SLDataIter)); + code = tLDataIterOpen(&pLDataIter[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pMTree->pLoadInfo[i], pMTree->idStr, strictTimeRange); if (code != TSDB_CODE_SUCCESS) { goto _end; } - bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); + bool hasVal = tLDataIterNextRow(&pLDataIter[i], pMTree->idStr); if (hasVal) { - taosArrayPush(pMTree->pIterList, &pIter); - tMergeTreeAddIter(pMTree, pIter); + tMergeTreeAddIter(pMTree, &pLDataIter[i]); } else { if (!pMTree->ignoreEarlierTs) { - pMTree->ignoreEarlierTs = pIter->ignoreEarlierTs; + pMTree->ignoreEarlierTs = pLDataIter[i].ignoreEarlierTs; } - tLDataIterClose(pIter); } } @@ -681,15 +650,7 @@ bool tMergeTreeNext(SMergeTree *pMTree) { TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; } void tMergeTreeClose(SMergeTree *pMTree) { - size_t size = taosArrayGetSize(pMTree->pIterList); - for (int32_t i = 0; i < size; ++i) { - SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i); - tLDataIterClose(pIter); - } - - pMTree->pIterList = taosArrayDestroy(pMTree->pIterList); pMTree->pIter = NULL; - if (pMTree->destroyLoadInfo) { pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo); pMTree->destroyLoadInfo = false; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5bd41dd86f..3aa3bdbd94 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -156,6 +156,7 @@ typedef struct SReaderStatus { SBlockData fileBlockData; SFilesetIter fileIter; SDataBlockIter blockIter; + SLDataIter* pLDataIter; } SReaderStatus; typedef struct SBlockInfoBuf { @@ -185,7 +186,6 @@ struct STsdbReader { STsdbReadSnap* pReadSnap; SIOCostSummary cost; STSchema* pSchema; // the newest version schema - // STSchema* pMemSchema; // the previous schema for in-memory data, to avoid load schema too many times SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema SDataFReader* pFileReader; // the file reader SDelFReader* pDelFReader; // the del file reader @@ -741,6 +741,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket + pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); if (pReader->pResBlock == NULL) { pReader->freeBlock = true; @@ -2547,7 +2548,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan pScanInfo->uid, pReader->idStr); int32_t code = tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, - pLBlockReader->pInfo, false, pReader->idStr, false); + pLBlockReader->pInfo, false, pReader->idStr, false, pReader->status.pLDataIter); if (code != TSDB_CODE_SUCCESS) { return false; } From d73306f6165a1cfec3091625096aa89c9763193e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 16:08:49 +0800 Subject: [PATCH 029/145] enh(query): opt last row read performance. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 52 ++++++++++++++------------ 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 3aa3bdbd94..b40e5abf8e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -741,7 +741,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket - pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); if (pReader->pResBlock == NULL) { pReader->freeBlock = true; @@ -774,6 +773,12 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } + pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); + if (pReader->status.pLDataIter == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); tsdbInitReaderLock(pReader); @@ -4368,29 +4373,38 @@ _err: return code; } +static void clearSharedPtr(STsdbReader* p) { + p->status.pLDataIter = NULL; + p->status.pTableMap = NULL; + p->status.uidList.tableUidList = NULL; + p->pReadSnap = NULL; + p->pSchema = NULL; + p->pSchemaMap = NULL; +} + +static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { + pDst->status.pTableMap = pSrc->status.pTableMap; + pDst->status.pLDataIter = pSrc->status.pLDataIter; + pDst->status.uidList = pSrc->status.uidList; + pDst->pSchema = pSrc->pSchema; + pDst->pSchemaMap = pSrc->pSchemaMap; + pDst->pReadSnap = pSrc->pReadSnap; +} + void tsdbReaderClose(STsdbReader* pReader) { if (pReader == NULL) { return; } tsdbAcquireReader(pReader); + { if (pReader->innerReader[0] != NULL || pReader->innerReader[1] != NULL) { STsdbReader* p = pReader->innerReader[0]; - - p->status.pTableMap = NULL; - p->status.uidList.tableUidList = NULL; - p->pReadSnap = NULL; - p->pSchema = NULL; - p->pSchemaMap = NULL; + clearSharedPtr(p); p = pReader->innerReader[1]; - - p->status.pTableMap = NULL; - p->status.uidList.tableUidList = NULL; - p->pReadSnap = NULL; - p->pSchema = NULL; - p->pSchemaMap = NULL; + clearSharedPtr(p); tsdbReaderClose(pReader->innerReader[0]); tsdbReaderClose(pReader->innerReader[1]); @@ -4627,18 +4641,10 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { // we need only one row pPrevReader->capacity = 1; - pPrevReader->status.pTableMap = pReader->status.pTableMap; - pPrevReader->status.uidList = pReader->status.uidList; - pPrevReader->pSchema = pReader->pSchema; - pPrevReader->pSchemaMap = pReader->pSchemaMap; - pPrevReader->pReadSnap = pReader->pReadSnap; + setSharedPtr(pPrevReader, pReader); pNextReader->capacity = 1; - pNextReader->status.pTableMap = pReader->status.pTableMap; - pNextReader->status.uidList = pReader->status.uidList; - pNextReader->pSchema = pReader->pSchema; - pNextReader->pSchemaMap = pReader->pSchemaMap; - pNextReader->pReadSnap = pReader->pReadSnap; + setSharedPtr(pNextReader, pReader); code = doOpenReaderImpl(pPrevReader); if (code != TSDB_CODE_SUCCESS) { From 7360a6b65ca75f8ebc0e48f3d641ef6341503899 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 24 Apr 2023 16:39:55 +0800 Subject: [PATCH 030/145] enh(query): opt hash. --- source/dnode/vnode/src/inc/tsdb.h | 3 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 5 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 124 +++++++++++---------- source/util/src/tsimplehash.c | 4 - 4 files changed, 69 insertions(+), 67 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8f6f9daf84..c3a0db466b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -16,6 +16,7 @@ #ifndef _TD_VNODE_TSDB_H_ #define _TD_VNODE_TSDB_H_ +#include "tsimplehash.h" #include "vnodeInt.h" #ifdef __cplusplus @@ -224,7 +225,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); -void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum); +void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj *pTableMap, int64_t *rowsNum); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d0ff403bf7..f27a28acb3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include #include "tsdb.h" #define MEM_MIN_HASH 1024 @@ -298,12 +299,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) { return rowsNum; } -void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) { +void tsdbMemTableCountRows(SMemTable *pMemTable, SSHashObj* pTableMap, int64_t *rowsNum) { taosRLockLatch(&pMemTable->latch); for (int32_t i = 0; i < pMemTable->nBucket; ++i) { STbData *pTbData = pMemTable->aBucket[i]; while (pTbData) { - void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + void* p = tSimpleHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); if (p == NULL) { pTbData = pTbData->next; continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b40e5abf8e..042b31ee09 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -125,12 +125,12 @@ typedef struct SFileDataBlockInfo { } SFileDataBlockInfo; typedef struct SDataBlockIter { - int32_t numOfBlocks; - int32_t index; - SArray* blockList; // SArray - int32_t order; - SDataBlk block; // current SDataBlk data - SHashObj* pTableMap; + int32_t numOfBlocks; + int32_t index; + SArray* blockList; // SArray + int32_t order; + SDataBlk block; // current SDataBlk data + SSHashObj* pTableMap; } SDataBlockIter; typedef struct SFileBlockDumpInfo { @@ -148,7 +148,7 @@ typedef struct STableUidList { typedef struct SReaderStatus { bool loadFromFile; // check file stage bool composedDataBlock; // the returned data block is a composed block or not - SHashObj* pTableMap; // SHash + SSHashObj* pTableMap; // SHash STableBlockScanInfo** pTableIter; // table iterator used in building in-memory buffer data blocks. STableUidList uidList; // check tables in uid order, to avoid the repeatly load of blocks in STT. SFileBlockDumpInfo fBlockDumpInfo; @@ -233,7 +233,7 @@ static bool hasDataInFileBlock(const SBlockData* pBlockData, const SFil static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter); static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order); -static STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, const char* id); +static STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); @@ -384,12 +384,11 @@ static int32_t uidComparFunc(const void* p1, const void* p2) { } // NOTE: speedup the whole processing by preparing the buffer for STableBlockScanInfo in batch model -static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, +static SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf, const STableKeyInfo* idList, STableUidList* pUidList, int32_t numOfTables) { // allocate buffer in order to load data blocks from file // todo use simple hash instead, optimize the memory consumption - SHashObj* pTableMap = - taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + SSHashObj* pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT)); if (pTableMap == NULL) { return NULL; } @@ -399,7 +398,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf pUidList->tableUidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); if (pUidList->tableUidList == NULL) { - taosHashCleanup(pTableMap); + tSimpleHashCleanup(pTableMap); return NULL; } @@ -421,7 +420,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf pScanInfo->lastKeyInStt = ekey; } - taosHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); + tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, pScanInfo->lastKey, pTsdbReader->idStr); } @@ -436,9 +435,11 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf return pTableMap; } -static void resetAllDataBlockScanInfo(SHashObj* pTableMap, int64_t ts, int32_t step) { - STableBlockScanInfo** p = NULL; - while ((p = taosHashIterate(pTableMap, p)) != NULL) { +static void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { + void *p = NULL; + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; pInfo->iterInit = false; @@ -478,13 +479,15 @@ static void clearBlockScanInfo(STableBlockScanInfo* p) { tMapDataClear(&p->mapData); } -static void destroyAllBlockScanInfo(SHashObj* pTableMap) { +static void destroyAllBlockScanInfo(SSHashObj* pTableMap) { void* p = NULL; - while ((p = taosHashIterate(pTableMap, p)) != NULL) { + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*(STableBlockScanInfo**)p); } - taosHashCleanup(pTableMap); + tSimpleHashCleanup(pTableMap); } static bool isEmptyQueryTimeWindow(STimeWindow* pWindow) { return pWindow->skey > pWindow->ekey; } @@ -800,7 +803,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, goto _end; } - int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); size_t num = taosArrayGetSize(aBlockIdx); @@ -864,10 +867,12 @@ _end: return code; } -static void cleanupTableScanInfo(SHashObj* pTableMap) { +static void cleanupTableScanInfo(SSHashObj* pTableMap) { STableBlockScanInfo** px = NULL; + int32_t iter = 0; + while (1) { - px = taosHashIterate(pTableMap, px); + px = tSimpleHashIterate(pTableMap, px, &iter); if (px == NULL) { break; } @@ -1439,7 +1444,7 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte pBlockIter->pTableMap = pReader->status.pTableMap; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = (int32_t)tSimpleHashGetSize(pReader->status.pTableMap); int64_t st = taosGetTimestampUs(); int32_t code = initBlockOrderSupporter(&sup, numOfTables); @@ -1449,8 +1454,10 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte int32_t cnt = 0; void* ptr = NULL; + int32_t iter = 0; + while (1) { - ptr = taosHashIterate(pReader->status.pTableMap, ptr); + ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); if (ptr == NULL) { break; } @@ -2916,7 +2923,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { pBlockNum->numOfBlocks = 0; pBlockNum->numOfLastFiles = 0; - size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); while (1) { @@ -2985,18 +2992,18 @@ static void resetTableListIndex(SReaderStatus* pStatus) { pList->currentIndex = 0; uint64_t uid = pList->tableUidList[0]; - pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); } static bool moveToNextTable(STableUidList* pOrderedCheckInfo, SReaderStatus* pStatus) { pOrderedCheckInfo->currentIndex += 1; - if (pOrderedCheckInfo->currentIndex >= taosHashGetSize(pStatus->pTableMap)) { + if (pOrderedCheckInfo->currentIndex >= tSimpleHashGetSize(pStatus->pTableMap)) { pStatus->pTableIter = NULL; return false; } uint64_t uid = pOrderedCheckInfo->tableUidList[pOrderedCheckInfo->currentIndex]; - pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); + pStatus->pTableIter = tSimpleHashGet(pStatus->pTableMap, &uid, sizeof(uid)); return (pStatus->pTableIter != NULL); } @@ -3006,7 +3013,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; int32_t code = TSDB_CODE_SUCCESS; - if (taosHashGetSize(pStatus->pTableMap) == 0) { + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { return TSDB_CODE_SUCCESS; } @@ -3168,7 +3175,7 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade goto _end; } - int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); size_t num = taosArrayGetSize(aBlockIdx); @@ -3178,14 +3185,13 @@ static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReade } SBlockIdx* pBlockIdx = NULL; - int32_t i = 0; for (int32_t i = 0; i < num; ++i) { pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); if (pBlockIdx->suid != pReader->suid) { continue; } - STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); + STableBlockScanInfo** p = tSimpleHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); if (p == NULL) { continue; } @@ -3231,13 +3237,13 @@ static int32_t doSumSttBlockRows(STsdbReader* pReader) { taosArrayClear(pBlockLoadInfo->aSttBlk); continue; } - for (int32_t i = 0; i < size; ++i) { - SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); pReader->rowsNum += p->nRow; } } else { - for (int32_t i = 0; i < size; ++i) { - SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + for (int32_t j = 0; j < size; ++j) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, j); uint64_t s = p->suid; if (s < pReader->suid) { continue; @@ -3307,13 +3313,6 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; while (1) { - // if (pStatus->pTableIter == NULL) { - // pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); - // if (pStatus->pTableIter == NULL) { - // return TSDB_CODE_SUCCESS; - // } - // } - STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; initMemDataIterator(*pBlockScanInfo, pReader); @@ -3341,7 +3340,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) SDataBlk* pBlock = getCurrentBlock(pBlockIter); SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); if (pBlockInfo) { - STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); + STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); if (pScanInfo) { lastKey = pScanInfo->lastKey; } @@ -4169,10 +4168,12 @@ int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t e // TODO refactor: with createDataBlockScanInfo int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t num) { - int32_t size = taosHashGetSize(pReader->status.pTableMap); + int32_t size = tSimpleHashGetSize(pReader->status.pTableMap); STableBlockScanInfo** p = NULL; - while ((p = taosHashIterate(pReader->status.pTableMap, p)) != NULL) { + int32_t iter = 0; + + while ((p = tSimpleHashIterate(pReader->status.pTableMap, p, &iter)) != NULL) { clearBlockScanInfo(*p); } @@ -4190,7 +4191,7 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n pReader->status.uidList.tableUidList = (uint64_t*)p1; } - taosHashClear(pReader->status.pTableMap); + tSimpleHashClear(pReader->status.pTableMap); STableUidList* pUidList = &pReader->status.uidList; pUidList->currentIndex = 0; @@ -4211,7 +4212,7 @@ int32_t tsdbSetTableList(STsdbReader* pReader, const void* pTableList, int32_t n pInfo->lastKeyInStt = ekey; } - taosHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); + tSimpleHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); } return TDB_CODE_SUCCESS; @@ -4428,7 +4429,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tBlockDataDestroy(&pReader->status.fileBlockData); cleanupDataBlockIterator(&pReader->status.blockIter); - size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (pReader->status.pTableMap != NULL) { destroyAllBlockScanInfo(pReader->status.pTableMap); clearBlockScanInfoBuf(&pReader->blockInfoBuf); @@ -4510,8 +4511,9 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; + int32_t iter = 0; - while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; pInfo->iterInit = false; @@ -4532,8 +4534,9 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { } else { // resetDataBlockScanInfo excluding lastKey STableBlockScanInfo** p = NULL; + int32_t iter = 0; - while ((p = taosHashIterate(pStatus->pTableMap, p)) != NULL) { + while ((p = tSimpleHashIterate(pStatus->pTableMap, p, &iter)) != NULL) { STableBlockScanInfo* pInfo = *(STableBlockScanInfo**)p; pInfo->iterInit = false; @@ -4622,7 +4625,7 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { // restore reader's state // task snapshot - int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); if (numOfTables > 0) { qTrace("tsdb/reader: %p, take snapshot", pReader); code = tsdbTakeReadSnap(pReader, tsdbSetQueryReseek, &pReader->pReadSnap); @@ -4701,7 +4704,7 @@ static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { *hasNext = false; SReaderStatus* pStatus = &pReader->status; - if (taosHashGetSize(pStatus->pTableMap) == 0) { + if (tSimpleHashGetSize(pStatus->pTableMap) == 0) { return code; } @@ -4954,11 +4957,11 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, return code; } -STableBlockScanInfo* getTableBlockScanInfo(SHashObj* pTableMap, uint64_t uid, const char* id) { - STableBlockScanInfo** p = taosHashGet(pTableMap, &uid, sizeof(uid)); +STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id) { + STableBlockScanInfo** p = tSimpleHashGet(pTableMap, &uid, sizeof(uid)); if (p == NULL || *p == NULL) { terrno = TSDB_CODE_INVALID_PARA; - int32_t size = taosHashGetSize(pTableMap); + int32_t size = tSimpleHashGetSize(pTableMap); tsdbError("failed to locate the uid:%" PRIu64 " in query table uid list, total tables:%d, %s", uid, size, id); return NULL; } @@ -5044,7 +5047,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { pReader->suppInfo.tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; tsdbDataFReaderClose(&pReader->pFileReader); - int32_t numOfTables = taosHashGetSize(pStatus->pTableMap); + int32_t numOfTables = tSimpleHashGetSize(pStatus->pTableMap); initFilesetIterator(&pStatus->fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(pBlockIter, pReader->order); @@ -5115,7 +5118,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa pTableBlockInfo->numOfFiles += 1; - int32_t numOfTables = (int32_t)taosHashGetSize(pStatus->pTableMap); + int32_t numOfTables = (int32_t)tSimpleHashGetSize(pStatus->pTableMap); int defaultRows = 4096; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -5179,7 +5182,8 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { tsdbReaderResume(pReader); } - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, NULL); + int32_t iter = 0; + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, NULL, &iter); while (pStatus->pTableIter != NULL) { STableBlockScanInfo* pBlockScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; @@ -5201,7 +5205,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { } // current table is exhausted, let's try the next table - pStatus->pTableIter = taosHashIterate(pStatus->pTableMap, pStatus->pTableIter); + pStatus->pTableIter = tSimpleHashIterate(pStatus->pTableMap, pStatus->pTableIter, &iter); } tsdbReleaseReader(pReader); diff --git a/source/util/src/tsimplehash.c b/source/util/src/tsimplehash.c index ec1991923f..4c7983a983 100644 --- a/source/util/src/tsimplehash.c +++ b/source/util/src/tsimplehash.c @@ -361,10 +361,6 @@ int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t ke return TSDB_CODE_SUCCESS; } -static void destroyItems(void* pItem) { - taosMemoryFree(*(void**)pItem); -} - void tSimpleHashClear(SSHashObj *pHashObj) { if (!pHashObj || taosHashTableEmpty(pHashObj)) { return; From 782566ad081d7ecb35b2dd3720cc0214cea5b0eb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 24 Apr 2023 17:15:46 +0800 Subject: [PATCH 031/145] refactor --- include/common/systable.h | 3 +-- source/common/src/systable.c | 4 ---- source/libs/parser/src/parTranslater.c | 9 ++++----- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/include/common/systable.h b/include/common/systable.h index 6e0c67bb3f..ea18338e9e 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -12,7 +12,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - + #ifndef TDENGINE_SYSTABLE_H #define TDENGINE_SYSTABLE_H @@ -77,7 +77,6 @@ void getInfosDbMeta(const SSysTableMeta** pInfosTableMeta, size_t* size); void getPerfDbMeta(const SSysTableMeta** pPerfsTableMeta, size_t* size); void getVisibleInfosTablesNum(bool sysInfo, size_t* size); bool invisibleColumn(bool sysInfo, int8_t tableType, int8_t flags); -bool isSystemDb(const char *dbName); #ifdef __cplusplus } diff --git a/source/common/src/systable.c b/source/common/src/systable.c index de95a4a5be..cd3dd63ef0 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -448,7 +448,3 @@ bool invisibleColumn(bool sysInfo, int8_t tableType, int8_t flags) { } return 0 != (flags & COL_IS_SYSINFO); } - -bool isSystemDb(const char *dbName) { - return ((strcasecmp(dbName, TSDB_INFORMATION_SCHEMA_DB) == 0) || (strcasecmp(dbName, TSDB_PERFORMANCE_SCHEMA_DB) == 0)); -} diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 52451d35a8..97fee72bf8 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2580,6 +2580,10 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isSelectStmt(pCxt->pCurrStmt)) { ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; } + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isDeleteStmt(pCxt->pCurrStmt)) { + code = TSDB_CODE_TSC_INVALID_OPERATION; + break; + } code = addNamespace(pCxt, pRealTable); } break; @@ -3792,11 +3796,6 @@ static int32_t translateDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet } static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { - // check delete from system tables - if (isSystemDb(((SRealTableNode*)pDelete->pFromTable)->table.dbName)) { - return TSDB_CODE_TSC_INVALID_OPERATION; - } - pCxt->pCurrStmt = (SNode*)pDelete; int32_t code = translateFrom(pCxt, pDelete->pFromTable); if (TSDB_CODE_SUCCESS == code) { From 80621e14eba7e8d6e70793f730c95494524a644d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 24 Apr 2023 17:15:46 +0800 Subject: [PATCH 032/145] refactor --- source/libs/parser/src/parTranslater.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 97fee72bf8..c7552445e3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2577,12 +2577,13 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { if (TSDB_SUPER_TABLE == pRealTable->pMeta->tableType) { pCxt->stableQuery = true; } - if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isSelectStmt(pCxt->pCurrStmt)) { - ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; - } - if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType && isDeleteStmt(pCxt->pCurrStmt)) { - code = TSDB_CODE_TSC_INVALID_OPERATION; - break; + if (TSDB_SYSTEM_TABLE == pRealTable->pMeta->tableType) { + if (isSelectStmt(pCxt->pCurrStmt)) { + ((SSelectStmt*)pCxt->pCurrStmt)->isTimeLineResult = false; + } else if (isDeleteStmt(pCxt->pCurrStmt)) { + code = TSDB_CODE_TSC_INVALID_OPERATION; + break; + } } code = addNamespace(pCxt, pRealTable); } From 8c4b73a351b3f5e8c05b9babe9fd42298abc865a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 24 Apr 2023 19:19:12 +0800 Subject: [PATCH 033/145] feat: add get table tag sync API --- include/libs/catalog/catalog.h | 2 ++ source/libs/catalog/src/catalog.c | 57 +++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 6f2fb4eb6b..429e7ffa73 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -314,6 +314,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); +int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); + int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg); int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp); diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b263654e70..976a38c03d 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -429,6 +429,48 @@ int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, CTG_RET(TSDB_CODE_SUCCESS); } +int32_t ctgGetTbTag(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pRes) { + SVgroupInfo vgroupInfo = {0}; + STableCfg* pCfg = NULL; + int32_t code = 0; + + CTG_ERR_RET(ctgGetTbHashVgroup(pCtg, pConn, pTableName, &vgroupInfo, NULL)); + CTG_ERR_RET(ctgGetTableCfgFromVnode(pCtg, pConn, pTableName, &vgroupInfo, &pCfg, NULL)); + + if (NULL == pCfg->pTags || pCfg->tagsLen <= 0) { + ctgError("invalid tag in tbCfg rsp, pTags:%p, len:%d", pCfg->pTags, pCfg->tagsLen); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + } + + SArray* pTagVals = NULL; + STag* pTag = (STag*)pCfg->pTags; + + if (tTagIsJson(pTag)) { + pTagVals = taosArrayInit(1, sizeof(STagVal)); + if (NULL == pTagVals) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + char* pJson = parseTagDatatoJson(pTag); + STagVal tagVal; + tagVal.cid = 0; + tagVal.type = TSDB_DATA_TYPE_JSON; + tagVal.pData = pJson; + tagVal.nData = strlen(pJson); + taosArrayPush(pTagVals, &tagVal); + } else { + CTG_ERR_JRET(tTagToValArray((const STag*)pCfg->pTags, &pTagVals)); + } + + *pRes = pTagVals; + +_return: + + tFreeSTableCfgRsp((STableCfgRsp*)pCfg); + + CTG_RET(code); +} + int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** pVgList) { STableMeta* tbMeta = NULL; int32_t code = 0; @@ -1414,6 +1456,21 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogGetTableTag(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pRes) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgGetTbTag(pCtg, pConn, (SName*)pTableName, pRes)); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableCfg** pCfg) { CTG_API_ENTER(); From 8572f4a32fa6db44b563d01be8c0ce3aeb271af3 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Apr 2023 19:41:30 +0800 Subject: [PATCH 034/145] opti:change push mgr to consume msg for subscribe --- source/client/src/clientTmq.c | 4 ++-- source/common/src/tglobal.c | 8 ++++---- source/common/src/tmsg.c | 6 +++--- source/dnode/vnode/src/tq/tq.c | 12 +++++++----- source/dnode/vnode/src/tq/tqUtil.c | 7 ++++++- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 16a4f55840..f05a314e44 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1702,7 +1702,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms - tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, + tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); continue; } @@ -1710,7 +1710,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); - tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, + tscDebug("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); continue; #if 0 diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index aa35b298e6..8cd3d7f5ab 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1274,10 +1274,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false); if (taosMulModeMkDir(tsLogDir, 0777) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - printf("failed to create dir:%s since %s", tsLogDir, terrstr()); - cfgCleanup(pCfg); - return -1; +// terrno = TAOS_SYSTEM_ERROR(errno); +// printf("failed to create dir:%s since %s", tsLogDir, terrstr()); +// cfgCleanup(pCfg); +// return -1; } if (taosInitLog(logname, logFileNum) != 0) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9802244b7..cd980d028c 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -5328,9 +5328,9 @@ int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { int32_t headLen = sizeof(SMsgHead); - SMsgHead *pHead = buf; - pHead->vgId = pReq->head.vgId; - pHead->contLen = pReq->head.contLen; +// SMsgHead *pHead = buf; +// pHead->vgId = pReq->head.vgId; +// pHead->contLen = pReq->head.contLen; SDecoder decoder = {0}; tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ae4a7e1d61..73c7075d51 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1084,13 +1084,15 @@ int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { taosWLockLatch(&pTq->lock); for(size_t i = 0; i < taosArrayGetSize(pTq->pPushArray); i++){ STqHandle* pHandle = (STqHandle*)taosArrayGetP(pTq->pPushArray, i); - if(pHandle->msg == NULL){ + if(ASSERT(pHandle->msg != NULL)){ tqError("pHandle->msg should not be null"); + break; + }else{ + SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; + tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + taosMemoryFree(pHandle->msg); + pHandle->msg = NULL; } - SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen}; - tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); - taosMemoryFree(pHandle->msg); - pHandle->msg = NULL; } taosArrayClear(pTq->pPushArray); // unlock diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index f76e641f2b..3f92414c34 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -181,15 +181,20 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, // code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); // lock taosWLockLatch(&pTq->lock); - if(pHandle->msg != NULL){ + if(ASSERT(pHandle->msg == NULL)){ tqError("pHandle->msg should be null"); + taosWUnLockLatch(&pTq->lock); + goto end; } pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg)); memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); + pHandle->msg->contLen = pMsg->contLen; + tqError("data is over, register to handle:%p, pCont:%p, len:%d", pHandle, pHandle->msg->pCont, pHandle->msg->contLen); taosArrayPush(pTq->pPushArray, &pHandle); taosWUnLockLatch(&pTq->lock); + tDeleteSMqDataRsp(&dataRsp); return code; } From 168e6f8936dca0586e2a7a4e57d50310f4bccc9c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Apr 2023 19:44:10 +0800 Subject: [PATCH 035/145] opti:change push mgr to consume msg for subscribe --- source/common/src/tglobal.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8cd3d7f5ab..aa35b298e6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1274,10 +1274,10 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false); if (taosMulModeMkDir(tsLogDir, 0777) != 0) { -// terrno = TAOS_SYSTEM_ERROR(errno); -// printf("failed to create dir:%s since %s", tsLogDir, terrstr()); -// cfgCleanup(pCfg); -// return -1; + terrno = TAOS_SYSTEM_ERROR(errno); + printf("failed to create dir:%s since %s", tsLogDir, terrstr()); + cfgCleanup(pCfg); + return -1; } if (taosInitLog(logname, logFileNum) != 0) { From 59ae3ecc2c17130049beace271c5950875bb9800 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Apr 2023 20:10:39 +0800 Subject: [PATCH 036/145] opti:test cases for tmq --- tests/system-test/7-tmq/subscribeDb3.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 22004a95f1..5b5326cfba 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -10,6 +10,7 @@ from util.log import * from util.sql import * from util.cases import * from util.dnodes import * +sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: From 1c63408b3e85510e1a372286d7be8c92fc30fb89 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 24 Apr 2023 20:18:20 +0800 Subject: [PATCH 037/145] opti:change push mgr to consume msg for subscribe --- include/libs/wal/wal.h | 2 -- source/client/src/clientTmq.c | 7 ++++--- source/libs/wal/src/walRead.c | 6 ++---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 835f786d97..b51289de5e 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -149,8 +149,6 @@ typedef struct SWalReader { int64_t capacity; // int8_t curInvalid; // int8_t curStopped; - int64_t bodyCnt; - int64_t bodyTotalSize; TdThreadMutex mutex; SWalFilterCond cond; // TODO remove it diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index f05a314e44..9e60f8b04d 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1702,7 +1702,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { for (int j = 0; j < numOfVg; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); if (taosGetTimestampMs() - pVg->emptyBlockReceiveTs < EMPTY_BLOCK_POLL_IDLE_DURATION) { // less than 100ms - tscDebug("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, + tscTrace("consumer:0x%" PRIx64 " epoch %d, vgId:%d idle for 10ms before start next poll", tmq->consumerId, tmq->epoch, pVg->vgId); continue; } @@ -1710,7 +1710,7 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int32_t vgStatus = atomic_val_compare_exchange_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE, TMQ_VG_STATUS__WAIT); if (vgStatus == TMQ_VG_STATUS__WAIT) { int32_t vgSkipCnt = atomic_add_fetch_32(&pVg->vgSkipCnt, 1); - tscDebug("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, + tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); continue; #if 0 @@ -1805,12 +1805,13 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { " total:%" PRId64 " reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pVg->numOfRows, tmq->totalRows, pollRspWrapper->reqId); pRspWrapper = tmqFreeRspWrapper(pRspWrapper); + pVg->emptyBlockReceiveTs = taosGetTimestampMs(); taosFreeQitem(pollRspWrapper); } else { // build rsp int64_t numOfRows = 0; SMqRspObj* pRsp = tmqBuildRspFromWrapper(pollRspWrapper, pVg, &numOfRows); tmq->totalRows += numOfRows; - + pVg->emptyBlockReceiveTs = 0; tscDebug("consumer:0x%" PRIx64 " process poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 " vg total:%" PRId64 " total:%" PRId64 ", reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pDataRsp->blockNum, numOfRows, pVg->numOfRows, tmq->totalRows, diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 09b6db6afe..6154e30938 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -262,8 +262,8 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { SWalCont *pReadHead = &pReader->pHead->head; int64_t ver = pReadHead->version; - wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d, total cnt:%"PRId64 ", total size:%"PRId64, pReader->pWal->cfg.vgId, ver, - pReadHead->bodyLen, pReader->bodyCnt, pReader->bodyTotalSize); + wDebug("vgId:%d, wal starts to fetch body, ver:%" PRId64 " ,len:%d, total", pReader->pWal->cfg.vgId, ver, + pReadHead->bodyLen); if (pReader->capacity < pReadHead->bodyLen) { SWalCkHead *ptr = (SWalCkHead *)taosMemoryRealloc(pReader->pHead, sizeof(SWalCkHead) + pReadHead->bodyLen); @@ -300,8 +300,6 @@ static int32_t walFetchBodyNew(SWalReader *pReader) { wDebug("vgId:%d, index:%" PRId64 " is fetched, cursor advance", pReader->pWal->cfg.vgId, ver); pReader->curVersion = ver + 1; - pReader->bodyCnt++; - pReader->bodyTotalSize += pReadHead->bodyLen; return 0; } From c6daa68e5420f893218503c0000cb971ea1d9438 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 00:17:37 +0800 Subject: [PATCH 038/145] enh(query): opt merge life cycle. --- source/dnode/vnode/src/inc/tsdb.h | 8 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 189 +++++++++++++------------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 78 ++++------ 3 files changed, 134 insertions(+), 141 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c3a0db466b..b2b04abff1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -126,11 +126,13 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); - -// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); void tsdbRowMergerClear(SRowMerger *pMerger); -// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); + +int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema); +void tsdbRowMergerClear_rv(SRowMerger* pMerger); +void tsdbRowMergerCleanup_rv(SRowMerger* pMerger); + // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); // TSDBKEY diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 042b31ee09..dd6913039c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -157,6 +157,7 @@ typedef struct SReaderStatus { SFilesetIter fileIter; SDataBlockIter blockIter; SLDataIter* pLDataIter; + SRowMerger merger; } SReaderStatus; typedef struct SBlockInfoBuf { @@ -166,6 +167,15 @@ typedef struct SBlockInfoBuf { int32_t numOfTables; } SBlockInfoBuf; +typedef struct STsdbReaderAttr { + STSchema* pSchema; + EReadMode readMode; + uint64_t rowsNum; + STimeWindow window; + bool freeBlock; + SVersionRange verRange; +} STsdbReaderAttr; + struct STsdbReader { STsdb* pTsdb; SVersionRange verRange; @@ -185,28 +195,26 @@ struct STsdbReader { SBlockLoadSuppInfo suppInfo; STsdbReadSnap* pReadSnap; SIOCostSummary cost; - STSchema* pSchema; // the newest version schema - SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema - SDataFReader* pFileReader; // the file reader - SDelFReader* pDelFReader; // the del file reader - SArray* pDelIdx; // del file block index; - SBlockInfoBuf blockInfoBuf; - int32_t step; - STsdbReader* innerReader[2]; + STSchema* pSchema; // the newest version schema + SSHashObj* pSchemaMap; // keep the retrieved schema info, to avoid the overhead by repeatly load schema + SDataFReader* pFileReader; // the file reader + SDelFReader* pDelFReader; // the del file reader + SArray* pDelIdx; // del file block index; + SBlockInfoBuf blockInfoBuf; + int32_t step; + STsdbReader* innerReader[2]; }; static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter); static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader); static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); -static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, - SRowMerger* pMerger); +static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader); static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger, SVersionRange* pVerRange, const char* id); -static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, - STsdbReader* pReader); +static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pTSRow, - STableBlockScanInfo* pInfo); + STableBlockScanInfo* pScanInfo); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); @@ -214,7 +222,7 @@ static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, - TSDBROW* pTSRow, STsdbReader* pReader, bool* freeTSRow); + TSDBROW* pResRow, STsdbReader* pReader, bool* freeTSRow); static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow); static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, @@ -235,7 +243,7 @@ static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order static STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); -static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); +static STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } @@ -1889,7 +1897,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas return code; } -static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) { +STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) { if (pReader->pSchema != NULL) { return pReader->pSchema; } @@ -1912,6 +1920,12 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* terrno = code; return NULL; } + + code = tsdbRowMergerInit_rv(&pReader->status.merger, pReader->pSchema); + if (code != 0) { + terrno = code; + return NULL; + } } if (pReader->pSchema && sversion == pReader->pSchema->version) { @@ -1989,11 +2003,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; // todo check if pReader->pSchema is null or not - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } if (minKey == tsLast) { @@ -2002,7 +2016,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2019,12 +2033,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + int32_t code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2037,12 +2051,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -2054,7 +2068,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2067,12 +2081,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } } @@ -2112,7 +2126,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2136,7 +2150,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2145,7 +2159,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } code = tsdbRowMergerGetRow(&merge, &pTSRow); @@ -2185,22 +2199,22 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { - SRow* pTSRow = NULL; - SRowMerger merge = {0}; + SRow* pTSRow = NULL; + SRowMerger* pMerger = &pReader->status.merger; - int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, pMerger, &pReader->verRange, pReader->idStr); - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2208,7 +2222,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); return code; } else { return TSDB_CODE_SUCCESS; @@ -2296,12 +2310,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } if (minKey == tsLast) { @@ -2310,7 +2324,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2324,14 +2338,13 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2346,13 +2359,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2360,13 +2372,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { if (minKey == k.ts) { init = true; - code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2377,14 +2388,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2396,7 +2405,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2407,7 +2416,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2417,7 +2426,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } tsdbRowMergerAdd(&merge, &fRow, NULL); } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } } @@ -2598,15 +2607,13 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SRow* pTSRow = NULL; - SRowMerger merge = {0}; - - code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - code = tsdbRowMergerGetRow(&merge, &pTSRow); + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); + code = tsdbRowMergerGetRow(&pReader->status.merger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2614,7 +2621,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(&pReader->status.merger); return code; } } @@ -3685,8 +3692,9 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } } -int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, - STsdbReader* pReader) { +int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, STsdbReader* pReader) { + SRowMerger* pMerger = &pReader->status.merger; + while (1) { pIter->hasVal = tsdbTbDataIterNext(pIter->iter); if (!pIter->hasVal) { @@ -3765,10 +3773,10 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn return code; } -int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, - SRowMerger* pMerger) { +int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SRowMerger* pMerger = &pReader->status.merger; bool asc = ASCENDING_TRAVERSE(pReader->order); int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; int32_t step = asc ? 1 : -1; @@ -3847,7 +3855,6 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } } - SRowMerger merge = {0}; terrno = 0; int32_t code = 0; @@ -3859,8 +3866,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - STSchema* ps = (pReader->pSchema != NULL)? pReader->pSchema:pTSchema; - code = tsdbRowMergerInit(&merge, ps, ¤t, pTSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3870,28 +3876,28 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); + tsdbRowMergerAdd(&pReader->status.merger,pNextRow, pTSchema1); } else { // let's merge rows in file block - code = tsdbRowMergerInit(&merge, NULL, ¤t, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&merge, pNextRow, NULL); + tsdbRowMergerAdd(&pReader->status.merger,pNextRow, NULL); } - code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); + code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - code = tsdbRowMergerGetRow(&merge, &pResRow->pTSRow); + code = tsdbRowMergerGetRow(&pReader->status.merger, &pResRow->pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } pResRow->type = TSDBROW_ROW_FMT; - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(&pReader->status.merger); *freeTSRow = true; return TSDB_CODE_SUCCESS; @@ -3899,7 +3905,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, SRow** pTSRow) { - SRowMerger merge = {0}; + SRowMerger* pMerger = &pReader->status.merger; TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); @@ -3914,46 +3920,43 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&merge, pRow, pSchema); + tsdbRowMergerAdd(&pReader->status.merger,pRow, pSchema); code = - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } } else { - int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); - if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { return code; } - code = - doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); + code = doMergeRowsInBuf(&pBlockScanInfo->iter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMergerAdd(&merge, piRow, piSchema); - code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, - pReader); + tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, pReader); if (code != TSDB_CODE_SUCCESS) { return code; } } - int32_t code = tsdbRowMergerGetRow(&merge, pTSRow); - tsdbRowMergerClear(&merge); + int32_t code = tsdbRowMergerGetRow(pMerger, pTSRow); + tsdbRowMergerClear_rv(pMerger); return code; } @@ -4334,6 +4337,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } } + if (pReader->pSchema != NULL) { + tsdbRowMergerInit_rv(&pReader->status.merger, pReader->pSchema); + } + pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); if (pReader->pSchemaMap == NULL) { tsdbError("failed init schema hash for reader %s", pReader->idStr); @@ -4483,6 +4490,8 @@ void tsdbReaderClose(STsdbReader* pReader) { pCost->initDelSkylineIterTime, pReader->idStr); taosMemoryFree(pReader->idStr); + + tsdbRowMergerCleanup_rv(&pReader->status.merger); taosMemoryFree(pReader->pSchema); tSimpleHashCleanup(pReader->pSchemaMap); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8e778da877..909d354c38 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -778,58 +778,40 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pMerger->version = key.version; return code; } -/* -int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { - int32_t code = 0; - TSDBKEY key = TSDBROW_KEY(pRow); - SColVal *pColVal = &(SColVal){0}; - STColumn *pTColumn; - pMerger->pTSchema = pTSchema; - pMerger->version = key.version; - - pMerger->pArray = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal)); +int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) { + pMerger->pTSchema = pSchema; + pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); if (pMerger->pArray == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + return TSDB_CODE_OUT_OF_MEMORY; + } else { + return TSDB_CODE_SUCCESS; } - - // ts - pTColumn = &pTSchema->columns[0]; - - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); - if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // other - for (int16_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { - tsdbRowGetColVal(pRow, pTSchema, iCol, pColVal); - if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - uint8_t *pVal = pColVal->value.pData; - - pColVal->value.pData = NULL; - code = tRealloc(&pColVal->value.pData, pColVal->value.nData); - if (code) goto _exit; - - if (pColVal->value.nData) { - memcpy(pColVal->value.pData, pVal, pColVal->value.nData); - } - } - - if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - -_exit: - return code; } -*/ + +void tsdbRowMergerClear_rv(SRowMerger* pMerger) { + for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (IS_VAR_DATA_TYPE(pTColVal->type)) { + tFree(pTColVal->value.pData); + } + } + + taosArrayClear(pMerger->pArray); +} + +void tsdbRowMergerCleanup_rv(SRowMerger* pMerger) { + int32_t numOfCols = taosArrayGetSize(pMerger->pArray); + for (int32_t iCol = 1; iCol < numOfCols; iCol++) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (IS_VAR_DATA_TYPE(pTColVal->type)) { + tFree(pTColVal->value.pData); + } + } + + taosArrayDestroy(pMerger->pArray); +} + void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); From 9a14e0db47909ce9d141f3a4f5d6cf08c3efb832 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 00:30:43 +0800 Subject: [PATCH 039/145] fix(query): fix error in add row into merger. --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 142 +++++++++++++++++-------- 1 file changed, 99 insertions(+), 43 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 909d354c38..7d1cca37b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -712,71 +712,127 @@ _exit: int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); - SColVal *pColVal = &(SColVal){0}; + SColVal * pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 1; if (NULL == pTSchema) { pTSchema = pMerger->pTSchema; } - ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); - for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { - pTColumn = &pMerger->pTSchema->columns[iCol]; - if (pTSchema->columns[jCol].colId < pTColumn->colId) { - ++jCol; - --iCol; - continue; - } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { - continue; + if (taosArrayGetSize(pMerger->pArray) == 0) { + // ts + pTColumn = &pTSchema->columns[jCol++]; + + ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); + + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + // goto _exit; } - tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + // other + for (iCol = 1; jCol < pTSchema->numOfCols && iCol < pMerger->pTSchema->numOfCols; ++iCol) { + pTColumn = &pMerger->pTSchema->columns[iCol]; + if (pTSchema->columns[jCol].colId < pTColumn->colId) { + ++jCol; + --iCol; + continue; + } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { + taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); + continue; + } - if (key.version > pMerger->version) { - if (!COL_VAL_IS_NONE(pColVal)) { - if (IS_VAR_DATA_TYPE(pColVal->type)) { - SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); - if (!COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); - if (code) return code; + tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + uint8_t *pVal = pColVal->value.pData; - pTColVal->value.nData = pColVal->value.nData; - if (pTColVal->value.nData) { - memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); + pColVal->value.pData = NULL; + code = tRealloc(&pColVal->value.pData, pColVal->value.nData); + if (code) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (pColVal->value.nData) { + memcpy(pColVal->value.pData, pVal, pColVal->value.nData); + } + } + + if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + for (; iCol < pMerger->pTSchema->numOfCols; ++iCol) { + pTColumn = &pMerger->pTSchema->columns[iCol]; + taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); + } + + pMerger->version = key.version; + return 0; + } else { + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); + + for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { + pTColumn = &pMerger->pTSchema->columns[iCol]; + if (pTSchema->columns[jCol].colId < pTColumn->colId) { + ++jCol; + --iCol; + continue; + } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { + continue; + } + + tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); + + if (key.version > pMerger->version) { + if (!COL_VAL_IS_NONE(pColVal)) { + if (IS_VAR_DATA_TYPE(pColVal->type)) { + SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); + if (!COL_VAL_IS_NULL(pColVal)) { + code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); + if (code) return code; + + pTColVal->value.nData = pColVal->value.nData; + if (pTColVal->value.nData) { + memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); + } + pTColVal->flag = 0; + } else { + tFree(pTColVal->value.pData); + taosArraySet(pMerger->pArray, iCol, pColVal); } - pTColVal->flag = 0; } else { - tFree(pTColVal->value.pData); taosArraySet(pMerger->pArray, iCol, pColVal); } - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); } - } - } else if (key.version < pMerger->version) { - SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); - if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - code = tRealloc(&tColVal->value.pData, pColVal->value.nData); - if (code) return code; + } else if (key.version < pMerger->version) { + SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); + if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { + if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { + code = tRealloc(&tColVal->value.pData, pColVal->value.nData); + if (code) return code; - tColVal->value.nData = pColVal->value.nData; - if (pColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + tColVal->value.nData = pColVal->value.nData; + if (pColVal->value.nData) { + memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); + } + tColVal->flag = 0; + } else { + taosArraySet(pMerger->pArray, iCol, pColVal); } - tColVal->flag = 0; - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); } + } else { + ASSERT(0 && "dup versions not allowed"); } - } else { - ASSERT(0 && "dup versions not allowed"); } - } - pMerger->version = key.version; - return code; + pMerger->version = key.version; + return code; + } } int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) { From 37d042a338264df27435c1b6dcc8c15287413e6a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 00:33:24 +0800 Subject: [PATCH 040/145] fix(query): fix error in add row into merger. --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 7d1cca37b5..0e2fb4b6aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -722,6 +722,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) if (taosArrayGetSize(pMerger->pArray) == 0) { // ts + jCol = 0; pTColumn = &pTSchema->columns[jCol++]; ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); From a73957aabadb33762ca72c9917336ffe2fb494f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 09:24:15 +0800 Subject: [PATCH 041/145] fix(query): fix error in add row into merger. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 85 +++++++++++--------------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index dd6913039c..0bba83622c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1954,7 +1954,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, TSDBROW* pRow, SIterInfo* pIter, int64_t key, SLastBlockReader* pLastBlockReader) { - SRowMerger merge = {0}; + SRowMerger* pMerger = &pReader->status.merger; SRow* pTSRow = NULL; SBlockData* pBlockData = &pReader->status.fileBlockData; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2013,7 +2013,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(&pReader->status.merger, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); @@ -2021,7 +2021,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &pReader->status.merger, &pReader->verRange, pReader->idStr); } if (minKey == k.ts) { @@ -2030,7 +2030,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } if (init) { - tsdbRowMergerAdd(&merge, pRow, pSchema); + tsdbRowMergerAdd(pMerger, pRow, pSchema); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); @@ -2057,7 +2057,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } code = doMergeRowsInBuf(pIter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, pReader); - if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { + if (code != TSDB_CODE_SUCCESS || pMerger->pTSchema == NULL) { return code; } } @@ -2065,7 +2065,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); @@ -2073,12 +2073,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->verRange, pReader->idStr); } if (minKey == key) { if (init) { - tsdbRowMergerAdd(&merge, &fRow, NULL); + tsdbRowMergerAdd(pMerger, &fRow, NULL); } else { init = true; int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); @@ -2090,7 +2090,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - int32_t code = tsdbRowMergerGetRow(&merge, &pTSRow); + int32_t code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2098,7 +2098,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); return code; } @@ -2106,12 +2106,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { + SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); bool copied = false; int32_t code = TSDB_CODE_SUCCESS; SRow* pTSRow = NULL; - SRowMerger merge = {0}; TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); @@ -2132,10 +2132,10 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(&merge, &fRow1, NULL); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2143,7 +2143,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2155,14 +2155,14 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); // merge with block data if ts == key if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2170,7 +2170,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2237,7 +2237,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { - SRowMerger merge = {0}; + SRowMerger* pMerger = &pReader->status.merger; SRow* pTSRow = NULL; int32_t code = TSDB_CODE_SUCCESS; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2310,7 +2310,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2321,24 +2321,24 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->verRange, pReader->idStr); } if (minKey == ik.ts) { if (init) { - tsdbRowMergerAdd(&merge, piRow, piSchema); + tsdbRowMergerAdd(pMerger, piRow, piSchema); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2352,14 +2352,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { if (init) { - if (merge.pTSchema == NULL) { - return code; - } - - tsdbRowMergerAdd(&merge, pRow, pSchema); + tsdbRowMergerAdd(pMerger, pRow, pSchema); } else { // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2372,7 +2368,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { if (minKey == k.ts) { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2385,10 +2381,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMergerAdd(&merge, piRow, piSchema); + tsdbRowMergerAdd(pMerger, piRow, piSchema); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, piRow, piSchema); + code = tsdbRowMergerAdd(pMerger, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2402,39 +2398,32 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&merge, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &merge, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->verRange, pReader->idStr); } if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } else { - if (merge.pTSchema == NULL) { - return code; - } - tsdbRowMergerAdd(&merge, &fRow, NULL); + tsdbRowMergerAdd(pMerger, &fRow, NULL); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); } } - if (merge.pTSchema == NULL) { - return code; - } - - code = tsdbRowMergerGetRow(&merge, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2442,7 +2431,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&merge); + tsdbRowMergerClear_rv(pMerger); return code; } From ae8a9d25c019b9b5711f66b05b2e4cf2d57be980 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 09:41:49 +0800 Subject: [PATCH 042/145] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index b2b04abff1..0de1d7de61 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -769,13 +769,13 @@ typedef struct SLDataIter { bool ignoreEarlierTs; } SLDataIter; +#define tMergeTreeGetRow(_t) ((_t)->pIter->rInfo.row) int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); -TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); SSttBlockLoadInfo *tCreateLastBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t numOfCols, int32_t numOfStt); diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index e354a50253..5a1b9d0ce4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -647,8 +647,6 @@ bool tMergeTreeNext(SMergeTree *pMTree) { return pMTree->pIter != NULL; } -TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; } - void tMergeTreeClose(SMergeTree *pMTree) { pMTree->pIter = NULL; if (pMTree->destroyLoadInfo) { From d57676c924870faf5dcc5be9a42d699b2d0cbf46 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 25 Apr 2023 09:52:16 +0800 Subject: [PATCH 043/145] fix: delete table error --- source/dnode/vnode/src/meta/metaTable.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 2b5bc36acb..96eec89127 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -936,8 +936,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { int tLen = 0; if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) { - version = ((SUidIdxVal *)tData)[0].version; - STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = version}; + STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version}; if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) { SDecoder tdc = {0}; SMetaEntry stbEntry = {0}; From e724bc15e03f9b5bc9cb06f7b3a35bdc39a6e6e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 09:54:00 +0800 Subject: [PATCH 044/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0bba83622c..a04100343a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2567,8 +2567,8 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan } static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { - TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - return TSDBROW_TS(&row); + TSDBROW* pRow = &tMergeTreeGetRow((&(pLastBlockReader)->mergeTree)); + return pRow->pBlockData->aTSKEY[pRow->iRow]; } static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } From 41bec8560a2587edabe2dd8a55841883a5a77b74 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 25 Apr 2023 09:58:28 +0800 Subject: [PATCH 045/145] opti:change push mgr to consume msg for subscribe --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 73c7075d51..8e35183ebb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -158,8 +158,8 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq->path); tqMetaClose(pTq); streamMetaClose(pTq->pStreamMeta); - taosMemoryFree(pTq); taosArrayDestroy(pTq->pPushArray); + taosMemoryFree(pTq); } void tqNotifyClose(STQ* pTq) { From 4c4e1486352638d0fe2df5763f4617aa1c1f00a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 10:19:37 +0800 Subject: [PATCH 046/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 45 ++++++++++++++------------ 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a04100343a..90bb391120 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -891,8 +891,7 @@ static void cleanupTableScanInfo(SSHashObj* pTableMap) { } } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum) { - int32_t numOfQTable = 0; +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { size_t sizeInDisk = 0; size_t numOfTables = taosArrayGetSize(pIndexList); @@ -952,7 +951,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN } if (taosArrayGetSize(pScanInfo->pBlockList) > 0) { - numOfQTable += 1; + taosArrayPush(pTableScanInfoList, &pScanInfo); } } @@ -963,8 +962,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN tsdbDebug( "load block of %ld tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, - pReader->idStr); + numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfLastFiles, + sizeInDisk / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; pReader->cost.headFileLoadTime += el; @@ -1443,7 +1442,7 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter, const char* idStr) return TSDB_CODE_SUCCESS; } -static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks) { +static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int32_t numOfBlocks, SArray* pTableList) { bool asc = ASCENDING_TRAVERSE(pReader->order); SBlockOrderSupporter sup = {0}; @@ -1452,7 +1451,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte pBlockIter->pTableMap = pReader->status.pTableMap; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)tSimpleHashGetSize(pReader->status.pTableMap); +// int32_t numOfTables = (int32_t)tSimpleHashGetSize(pReader->status.pTableMap); + int32_t numOfTables = taosArrayGetSize(pTableList); int64_t st = taosGetTimestampUs(); int32_t code = initBlockOrderSupporter(&sup, numOfTables); @@ -1464,16 +1464,18 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte void* ptr = NULL; int32_t iter = 0; - while (1) { - ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); - if (ptr == NULL) { - break; - } +// while (1) { +// ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); +// if (ptr == NULL) { +// break; +// } - STableBlockScanInfo* pTableScanInfo = *(STableBlockScanInfo**)ptr; - if (pTableScanInfo->pBlockList == NULL || taosArrayGetSize(pTableScanInfo->pBlockList) == 0) { - continue; - } + for(int32_t i = 0; i < numOfTables; ++i) { + STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); + ASSERT(pTableScanInfo->pBlockList != NULL && taosArrayGetSize(pTableScanInfo->pBlockList) > 0); +// if (pTableScanInfo->pBlockList == NULL || taosArrayGetSize(pTableScanInfo->pBlockList) == 0) { +// continue; +// } size_t num = taosArrayGetSize(pTableScanInfo->pBlockList); sup.numOfBlocksPerTable[sup.numOfTables] = num; @@ -2914,7 +2916,7 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) } } -static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { +static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { SReaderStatus* pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; pBlockNum->numOfLastFiles = 0; @@ -2942,13 +2944,14 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { } if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSttF > 0) { - code = doLoadFileBlock(pReader, pIndexList, pBlockNum); + code = doLoadFileBlock(pReader, pIndexList, pBlockNum, pTableList); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; } if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { + ASSERT(taosArrayGetSize(pTableList) > 0); break; } } @@ -3353,7 +3356,9 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBlockIter) { SBlockNumber num = {0}; - int32_t code = moveToNextFile(pReader, &num); + SArray* pTableList = taosArrayInit(40, POINTER_BYTES); + + int32_t code = moveToNextFile(pReader, &num, pTableList); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3366,7 +3371,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl // initialize the block iterator for a new fileset if (num.numOfBlocks > 0) { - code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); + code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks, pTableList); } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order); From 6467b99a52299ac076495f553eccd792c7c5cc07 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 10:23:21 +0800 Subject: [PATCH 047/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 90bb391120..8d4708925b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1461,8 +1461,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int32_t cnt = 0; - void* ptr = NULL; - int32_t iter = 0; +// void* ptr = NULL; +// int32_t iter = 0; // while (1) { // ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); @@ -2951,7 +2951,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { - ASSERT(taosArrayGetSize(pTableList) > 0); +// ASSERT(taosArrayGetSize(pTableList) > 0); break; } } From d8762b9bc956782904505638ce5af23549d16999 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 10:48:21 +0800 Subject: [PATCH 048/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 6 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 32 ++++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 5a1b9d0ce4..79f4a17f65 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -261,8 +261,10 @@ int32_t tLDataIterOpen(struct SLDataIter *pIter, SDataFReader *pReader, int32_t pIter->pReader = pReader; pIter->iStt = iStt; pIter->backward = backward; - pIter->verRange = *pRange; - pIter->timeWindow = *pTimeWindow; + pIter->verRange.minVer = pRange->minVer; + pIter->verRange.maxVer = pRange->maxVer; + pIter->timeWindow.skey = pTimeWindow->skey; + pIter->timeWindow.ekey = pTimeWindow->ekey; pIter->pBlockLoadInfo = pBlockLoadInfo; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8d4708925b..efd8ab1288 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -148,6 +148,7 @@ typedef struct STableUidList { typedef struct SReaderStatus { bool loadFromFile; // check file stage bool composedDataBlock; // the returned data block is a composed block or not + bool mapDataCleaned; // mapData has been cleaned up alreay or not SSHashObj* pTableMap; // SHash STableBlockScanInfo** pTableIter; // table iterator used in building in-memory buffer data blocks. STableUidList uidList; // check tables in uid order, to avoid the repeatly load of blocks in STT. @@ -875,7 +876,18 @@ _end: return code; } -static void cleanupTableScanInfo(SSHashObj* pTableMap) { +static void doCleanupTableScanInfo(STableBlockScanInfo* pScanInfo) { + // reset the index in last block when handing a new file + tMapDataClear(&pScanInfo->mapData); + taosArrayClear(pScanInfo->pBlockList); +} + +static void cleanupTableScanInfo(SReaderStatus* pStatus) { + if (pStatus->mapDataCleaned) { + return; + } + + SSHashObj* pTableMap = pStatus->pTableMap; STableBlockScanInfo** px = NULL; int32_t iter = 0; @@ -885,10 +897,10 @@ static void cleanupTableScanInfo(SSHashObj* pTableMap) { break; } - // reset the index in last block when handing a new file - tMapDataClear(&(*px)->mapData); - taosArrayClear((*px)->pBlockList); + doCleanupTableScanInfo(*px); } + + pStatus->mapDataCleaned = true; } static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum, SArray* pTableScanInfoList) { @@ -896,8 +908,10 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN size_t numOfTables = taosArrayGetSize(pIndexList); int64_t st = taosGetTimestampUs(); - cleanupTableScanInfo(pReader->status.pTableMap); + cleanupTableScanInfo(&pReader->status); + // set the flag for the new file + pReader->status.mapDataCleaned = false; for (int32_t i = 0; i < numOfTables; ++i) { SBlockIdx* pBlockIdx = taosArrayGet(pIndexList, i); STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); @@ -3022,8 +3036,12 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; - bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); - if (!hasVal) { + // reset the index in last block when handing a new file + doCleanupTableScanInfo(pScanInfo); + pStatus->mapDataCleaned = true; + + bool hasDataInLastFile = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + if (!hasDataInLastFile) { bool hasNexTable = moveToNextTable(pUidList, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; From f264f29bbda2f238d782259178f5f611794942f1 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Tue, 25 Apr 2023 11:17:49 +0800 Subject: [PATCH 049/145] fix: restore ci buildPath --- tests/parallel_test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/run.sh b/tests/parallel_test/run.sh index de343752c6..f05e0dfc83 100755 --- a/tests/parallel_test/run.sh +++ b/tests/parallel_test/run.sh @@ -303,7 +303,7 @@ function run_thread() { if [ ! -z "$corefile" ]; then echo -e "\e[34m corefiles: $corefile \e[0m" local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" + local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" # if [ $ent -ne 0 ]; then # remote_build_dir="${workdirs[index]}/{DEBUGPATH}/build" # fi From 05d16920734367eeb29173febd4716930e9477ad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 13:50:05 +0800 Subject: [PATCH 050/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index efd8ab1288..8e70badbc1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -18,6 +18,7 @@ #include "tsimplehash.h" #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) +#define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -108,6 +109,7 @@ typedef struct SLastBlockReader { uint64_t uid; SMergeTree mergeTree; SSttBlockLoadInfo* pInfo; + int64_t currentKey; } SLastBlockReader; typedef struct SFilesetIter { @@ -234,7 +236,6 @@ static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdb static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); -static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); @@ -1870,7 +1871,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = TSDBROW_KEY(&row); + TSDBKEY k = {.version = row.version, .ts = row.pBlockData->aTSKEY[row.iRow]}; + pLastBlockReader->currentKey = k.ts; pScanInfo->lastKeyInStt = k.ts; if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order, pVerRange)) { @@ -2582,10 +2584,9 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan return nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->verRange); } -static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { - TSDBROW* pRow = &tMergeTreeGetRow((&(pLastBlockReader)->mergeTree)); - return pRow->pBlockData->aTSKEY[pRow->iRow]; -} +//static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { +// return pLastBlockReader->currentKey; +//} static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } From 81e54541dec8e044665bbc6ef04ccdc0d531a0e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 14:24:24 +0800 Subject: [PATCH 051/145] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 50 ++++++++++++------------- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0de1d7de61..8ce40ac293 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -769,7 +769,7 @@ typedef struct SLDataIter { bool ignoreEarlierTs; } SLDataIter; -#define tMergeTreeGetRow(_t) ((_t)->pIter->rInfo.row) +#define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row)) int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo, bool destroyLoadInfo, const char *idStr, bool strictTimeRange, SLDataIter* pLDataIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 42f332c54a..65fc086f8d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -668,7 +668,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow, bool *pIgnoreEa state->state = SFSLASTNEXTROW_FILESET; goto _next_fileset; } - state->row = tMergeTreeGetRow(&state->mergeTree); + state->row = *tMergeTreeGetRow(&state->mergeTree); *ppRow = &state->row; if (TSDBROW_TS(&state->row) <= state->lastTs) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8e70badbc1..0ad9ae1361 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1870,8 +1870,8 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc return false; } - TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = {.version = row.version, .ts = row.pBlockData->aTSKEY[row.iRow]}; + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBKEY k = {.version = pRow->version, .ts = pRow->pBlockData->aTSKEY[pRow->iRow]}; pLastBlockReader->currentKey = k.ts; pScanInfo->lastKeyInStt = k.ts; @@ -2029,12 +2029,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&pReader->status.merger, &fRow1, NULL); + tsdbRowMergerAdd(&pReader->status.merger, fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2081,12 +2081,12 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(&pReader->status.merger, fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2130,12 +2130,12 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool copied = false; int32_t code = TSDB_CODE_SUCCESS; SRow* pTSRow = NULL; - TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", fRow.pBlockData, fRow.iRow, pLastBlockReader->uid, pReader->idStr); + TSDBROW *pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + code = tryCopyDistinctRowFromSttBlock(pRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); if (code) { return code; } @@ -2144,13 +2144,13 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, pMerger, &pReader->verRange, pReader->idStr); code = tsdbRowMergerGetRow(pMerger, &pTSRow); @@ -2168,7 +2168,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2227,8 +2227,8 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, pMerger, &pReader->verRange, pReader->idStr); @@ -2337,12 +2337,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2414,12 +2414,12 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { init = true; - code = tsdbRowMergerAdd(pMerger, &fRow1, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, pRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3827,8 +3827,8 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { - TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMergerAdd(pMerger, &fRow1, NULL); + TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, From 0b6e531dd38d2e25a673f906220cc822faff5021 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 14:56:20 +0800 Subject: [PATCH 052/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 55 ++++++++++++-------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0ad9ae1361..a2313b40f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -221,7 +221,7 @@ static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); -static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, +static bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange); static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, @@ -1871,11 +1871,13 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc } TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - TSDBKEY k = {.version = pRow->version, .ts = pRow->pBlockData->aTSKEY[pRow->iRow]}; - pLastBlockReader->currentKey = k.ts; - pScanInfo->lastKeyInStt = k.ts; + int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; + int64_t ver = pRow->version; - if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order, pVerRange)) { + pLastBlockReader->currentKey = key; + pScanInfo->lastKeyInStt = ver; + + if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { // the qualifed ts may equal to k.ts, only a greater version one. // here we need to fallback one step. return true; @@ -2543,8 +2545,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum return false; } - TSDBKEY k = {.ts = ts, .version = ver}; - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, &k, pReader->order, + if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->fileDelIndex, ts, ver, pReader->order, &pReader->verRange)) { return false; } @@ -2584,10 +2585,6 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan return nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->verRange); } -//static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { -// return pLastBlockReader->currentKey; -//} - static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* pDumpInfo) { @@ -3562,7 +3559,7 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } -bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order, SVersionRange* pVerRange) { +bool hasBeenDropped(const SArray* pDelList, int32_t* index, int64_t key, int64_t ver, int32_t order, SVersionRange* pVerRange) { if (pDelList == NULL) { return false; } @@ -3574,29 +3571,29 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 if (asc) { if (*index >= num - 1) { TSDBKEY* last = taosArrayGetLast(pDelList); - ASSERT(pKey->ts >= last->ts); + ASSERT(key >= last->ts); - if (pKey->ts > last->ts) { + if (key > last->ts) { return false; - } else if (pKey->ts == last->ts) { + } else if (key == last->ts) { TSDBKEY* prev = taosArrayGet(pDelList, num - 2); - return (prev->version >= pKey->version && prev->version <= pVerRange->maxVer && + return (prev->version >= ver && prev->version <= pVerRange->maxVer && prev->version >= pVerRange->minVer); } } else { TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pNext = taosArrayGet(pDelList, (*index) + 1); - if (pKey->ts < pCurrent->ts) { + if (key < pCurrent->ts) { return false; } - if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version && + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && pVerRange->maxVer >= pCurrent->version) { return true; } - while (pNext->ts <= pKey->ts && (*index) < num - 1) { + while (pNext->ts <= key && (*index) < num - 1) { (*index) += 1; if ((*index) < num - 1) { @@ -3608,7 +3605,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 continue; } - if (pCurrent->ts <= pKey->ts && pNext->ts >= pKey->ts && pCurrent->version >= pKey->version && + if (pCurrent->ts <= key && pNext->ts >= key && pCurrent->version >= ver && pVerRange->maxVer >= pCurrent->version) { return true; } @@ -3621,10 +3618,10 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 if (*index <= 0) { TSDBKEY* pFirst = taosArrayGet(pDelList, 0); - if (pKey->ts < pFirst->ts) { + if (key < pFirst->ts) { return false; - } else if (pKey->ts == pFirst->ts) { - return pFirst->version >= pKey->version; + } else if (key == pFirst->ts) { + return pFirst->version >= ver; } else { ASSERT(0); } @@ -3632,15 +3629,15 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 TSDBKEY* pCurrent = taosArrayGet(pDelList, *index); TSDBKEY* pPrev = taosArrayGet(pDelList, (*index) - 1); - if (pKey->ts > pCurrent->ts) { + if (key > pCurrent->ts) { return false; } - if (pPrev->ts <= pKey->ts && pCurrent->ts >= pKey->ts && pPrev->version >= pKey->version) { + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { return true; } - while (pPrev->ts >= pKey->ts && (*index) > 1) { + while (pPrev->ts >= key && (*index) > 1) { (*index) += step; if ((*index) >= 1) { @@ -3652,7 +3649,7 @@ bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32 continue; } - if (pPrev->ts <= pKey->ts && pCurrent->ts >= pKey->ts && pPrev->version >= pKey->version) { + if (pPrev->ts <= key && pCurrent->ts >= key && pPrev->version >= ver) { return true; } } @@ -3680,7 +3677,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p // it is a valid data version if ((key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer) && - (!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order, &pReader->verRange))) { + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->order, &pReader->verRange))) { return pRow; } @@ -3699,7 +3696,7 @@ TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* p } if (key.version <= pReader->verRange.maxVer && key.version >= pReader->verRange.minVer && - (!hasBeenDropped(pDelList, &pIter->index, &key, pReader->order, &pReader->verRange))) { + (!hasBeenDropped(pDelList, &pIter->index, key.ts, key.version, pReader->order, &pReader->verRange))) { return pRow; } } From 8dbc3b34572c71b2d939df5996b76a6da0a51cf9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 15:07:37 +0800 Subject: [PATCH 053/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a2313b40f4..b54ade12e5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -161,6 +161,7 @@ typedef struct SReaderStatus { SDataBlockIter blockIter; SLDataIter* pLDataIter; SRowMerger merger; + SColumnInfoData* pPrimaryTsCol; // primary time stamp output col info data } SReaderStatus; typedef struct SBlockInfoBuf { @@ -780,6 +781,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; + setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + code = tBlockDataCreate(&pReader->status.fileBlockData); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -792,7 +795,8 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); + ASSERT (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); + pReader->status.pPrimaryTsCol = taosArrayGet(pResBlock->pDataBlock, pSup->slotId[0]); tsdbInitReaderLock(pReader); @@ -4091,11 +4095,12 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t code = TSDB_CODE_SUCCESS; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - if (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID) { - SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); - ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; +// ASSERT (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID);// { +// SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); +// ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; + ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; i += 1; - } +// } SColVal cv = {0}; int32_t numOfInputCols = pBlockData->nColData; From bca5fc7893e8f6e539ef28a3f433594c0f987a49 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 15:28:54 +0800 Subject: [PATCH 054/145] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b54ade12e5..03616df21b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4478,6 +4478,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbUninitReaderLock(pReader); + taosMemoryFreeClear(pReader->status.pLDataIter); taosMemoryFree(pReader->status.uidList.tableUidList); SIOCostSummary* pCost = &pReader->cost; From 16fe48f770f6b8a101d36acb8fe6eb94e4bcb8fb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 16:09:58 +0800 Subject: [PATCH 055/145] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 03616df21b..0ad5a44919 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3378,14 +3378,16 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl SBlockNumber num = {0}; SArray* pTableList = taosArrayInit(40, POINTER_BYTES); - int32_t code = moveToNextFile(pReader, &num, pTableList); + int32_t code = moveToNextFile(pReader, &num, pTableList); if (code != TSDB_CODE_SUCCESS) { + taosArrayDestroy(pTableList); return code; } // all data files are consumed, try data in buffer if (num.numOfBlocks + num.numOfLastFiles == 0) { pReader->status.loadFromFile = false; + taosArrayDestroy(pTableList); return code; } @@ -3400,6 +3402,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); + taosArrayDestroy(pTableList); return code; } From aede1090ff2c6f7144de1d88f647bcda18174178 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 25 Apr 2023 16:23:30 +0800 Subject: [PATCH 056/145] fix: the latest schema may delete columns --- source/dnode/vnode/src/tsdb/tsdbUtil.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8e778da877..8cbbcb751c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1218,9 +1218,10 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int32_t iColumn = 1; STColumn *pTColumn = &pTSchema->columns[iColumn]; for (int32_t iCid = 0; iCid < nCid; iCid++) { - if (ASSERTS(pTColumn != NULL, "invalid input param")) { - code = TSDB_CODE_INVALID_PARA; - goto _exit; + + // aCid array (from taos client catalog) contains columns that does not exist in the pTSchema. the pTSchema is newer + if (pTColumn == NULL) { + continue; } while (pTColumn->colId < aCid[iCid]) { @@ -1229,9 +1230,8 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, pTColumn = &pTSchema->columns[iColumn]; } - if (ASSERTS(pTColumn->colId == aCid[iCid], "invalid input param")) { - code = TSDB_CODE_INVALID_PARA; - goto _exit; + if (pTColumn->colId != aCid[iCid]) { + continue; } tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, From d1e5d6e0f992d39986926a3e092f9cde1d67a7cf Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 25 Apr 2023 16:23:58 +0800 Subject: [PATCH 057/145] fix:pHandle->msg is not null if rebalance --- source/client/src/clientTmq.c | 5 +++-- source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tq/tqUtil.c | 11 +++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 9e60f8b04d..54e929c9a4 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1363,6 +1363,7 @@ CREATE_MSG_FAIL: typedef struct SVgroupSaveInfo { STqOffsetVal offset; int64_t numOfRows; + int32_t vgStatus; } SVgroupSaveInfo; static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, @@ -1398,7 +1399,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .currentOffset = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, - .vgStatus = TMQ_VG_STATUS__IDLE, + .vgStatus = pInfo != NULL ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, .vgSkipCnt = 0, .emptyBlockReceiveTs = 0, .numOfRows = numOfRows, @@ -1457,7 +1458,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows}; + SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows, .vgStatus = pVgCur->vgStatus}; taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8e35183ebb..0080feadbe 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1079,11 +1079,11 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); - tqDebug("vgId:%d start set submit for subscribe", vgId); taosWLockLatch(&pTq->lock); for(size_t i = 0; i < taosArrayGetSize(pTq->pPushArray); i++){ STqHandle* pHandle = (STqHandle*)taosArrayGetP(pTq->pPushArray, i); + tqDebug("vgId:%d start set submit for pHandle:%p", vgId, pHandle); if(ASSERT(pHandle->msg != NULL)){ tqError("pHandle->msg should not be null"); break; diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 3f92414c34..663dc8bbb9 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -181,17 +181,16 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, // code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); // lock taosWLockLatch(&pTq->lock); - if(ASSERT(pHandle->msg == NULL)){ - tqError("pHandle->msg should be null"); - taosWUnLockLatch(&pTq->lock); - goto end; +// tqDebug("data is over, register to handle:%p, msg:%p", pHandle, pHandle->msg); + if(pHandle->msg == NULL){ + pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg)); } - pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); pHandle->msg->contLen = pMsg->contLen; - tqError("data is over, register to handle:%p, pCont:%p, len:%d", pHandle, pHandle->msg->pCont, pHandle->msg->contLen); + tqDebug("data is over, register to handle:%p, pCont:%p, len:%d", pHandle, pHandle->msg->pCont, pHandle->msg->contLen); taosArrayPush(pTq->pPushArray, &pHandle); taosWUnLockLatch(&pTq->lock); tDeleteSMqDataRsp(&dataRsp); From ee93500619e91ff6e7851171c2b38ec9d13859f8 Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Apr 2023 17:03:58 +0800 Subject: [PATCH 058/145] fix: log file size over 4G report error on windows --- source/os/src/osFile.c | 3 ++- source/util/src/tlog.c | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index aab547223f..9833e3ba45 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -538,10 +538,11 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { return -1; } - struct stat fileStat; #ifdef WINDOWS + struct __stat64 fileStat; int32_t code = _fstat(pFile->fd, &fileStat); #else + struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); #endif if (code < 0) { diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 2a18f420cd..d415379f92 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -347,7 +347,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { char name[LOG_FILE_NAME_LEN + 50] = "\0"; int32_t logstat0_mtime, logstat1_mtime; - int32_t size; tsLogObj.maxLines = maxLines; tsLogObj.fileNum = maxFileNum; @@ -395,8 +394,7 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { printf("\nfailed to fstat log file:%s, reason:%s\n", fileName, strerror(errno)); return -1; } - size = (int32_t)filesize; - tsLogObj.lines = size / 60; + tsLogObj.lines = (int32_t)(filesize / 60); taosLSeekFile(tsLogObj.logHandle->pFile, 0, SEEK_END); From 97a5ebac1c5154458bf13a5e7b63a41cd555713d Mon Sep 17 00:00:00 2001 From: Alex Duan <417921451@qq.com> Date: Tue, 25 Apr 2023 17:10:12 +0800 Subject: [PATCH 059/145] fix: miss out _fstat64 --- source/os/src/osFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 9833e3ba45..dd670595f0 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -540,7 +540,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) { #ifdef WINDOWS struct __stat64 fileStat; - int32_t code = _fstat(pFile->fd, &fileStat); + int32_t code = _fstat64(pFile->fd, &fileStat); #else struct stat fileStat; int32_t code = fstat(pFile->fd, &fileStat); From 49728a235e83f58df0a995e6a4c88779d9980d9b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 25 Apr 2023 17:32:45 +0800 Subject: [PATCH 060/145] fix: more fix --- source/dnode/vnode/src/inc/tsdb.h | 5 +- source/dnode/vnode/src/tsdb/tsdbDataIter.c | 61 ++++++++++++++-------- source/dnode/vnode/src/tsdb/tsdbSnapshot.c | 26 +++++---- 3 files changed, 57 insertions(+), 35 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2a85b191a4..4d3111a9b7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -322,8 +322,9 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, #define TSDB_STT_FILE_DATA_ITER 2 #define TSDB_TOMB_FILE_DATA_ITER 3 -#define TSDB_FILTER_FLAG_BY_VERSION 0x1 -#define TSDB_FILTER_FLAG_BY_TABLEID 0x2 +#define TSDB_FILTER_FLAG_BY_VERSION 0x1 +#define TSDB_FILTER_FLAG_BY_TABLEID 0x2 +#define TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE 0x4 #define TSDB_RBTN_TO_DATA_ITER(pNode) ((STsdbDataIter2 *)(((char *)pNode) - offsetof(STsdbDataIter2, rbtn))) /* open */ diff --git a/source/dnode/vnode/src/tsdb/tsdbDataIter.c b/source/dnode/vnode/src/tsdb/tsdbDataIter.c index 3299a2f497..e27aec5b1b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataIter.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataIter.c @@ -14,6 +14,7 @@ */ #include "tsdb.h" +#include "vnodeInt.h" // STsdbDataIter2 /* open */ @@ -202,13 +203,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* for (;;) { while (pIter->dIter.iRow < pIter->dIter.bData.nRow) { if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { - if (pFilterInfo->tbid.uid == pIter->dIter.bData.uid) { - pIter->dIter.iRow = pIter->dIter.bData.nRow; - continue; - } - } - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { if (pIter->dIter.bData.aVersion[pIter->dIter.iRow] < pFilterInfo->sver || pIter->dIter.bData.aVersion[pIter->dIter.iRow] > pFilterInfo->ever) { @@ -232,13 +226,6 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* // filter if (pFilterInfo) { - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { - if (tTABLEIDCmprFn(&pFilterInfo->tbid, &pIter->rowInfo) == 0) { - pIter->dIter.iDataBlk = pIter->dIter.mDataBlk.nItem; - continue; - } - } - if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { if (pFilterInfo->sver > dataBlk.maxVer || pFilterInfo->ever < dataBlk.minVer) { pIter->dIter.iDataBlk++; @@ -262,13 +249,23 @@ static int32_t tsdbDataFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* if (pIter->dIter.iBlockIdx < taosArrayGetSize(pIter->dIter.aBlockIdx)) { SBlockIdx* pBlockIdx = taosArrayGet(pIter->dIter.aBlockIdx, pIter->dIter.iBlockIdx); - if (pFilterInfo && (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID)) { - int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid); - if (c == 0) { - pIter->dIter.iBlockIdx++; - continue; - } else if (c < 0) { - ASSERT(0); + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { + int32_t c = tTABLEIDCmprFn(pBlockIdx, &pFilterInfo->tbid); + if (c == 0) { + pIter->dIter.iBlockIdx++; + continue; + } else if (c < 0) { + ASSERT(0); + } + } + + if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) { + SMetaInfo info; + if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pBlockIdx->uid, &info, NULL)) { + pIter->dIter.iBlockIdx++; + continue; + } } } @@ -304,14 +301,24 @@ static int32_t tsdbSttFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* p for (;;) { while (pIter->sIter.iRow < pIter->sIter.bData.nRow) { if (pFilterInfo) { + int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow]; if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_TABLEID) { - int64_t uid = pIter->sIter.bData.uid ? pIter->sIter.bData.uid : pIter->sIter.bData.aUid[pIter->sIter.iRow]; if (pFilterInfo->tbid.uid == uid) { pIter->sIter.iRow++; continue; } } + if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) { + if (pIter->rowInfo.uid != uid) { + SMetaInfo info; + if (metaGetInfo(pIter->sIter.pReader->pTsdb->pVnode->pMeta, uid, &info, NULL)) { + pIter->sIter.iRow++; + continue; + } + } + } + if (pFilterInfo->flag & TSDB_FILTER_FLAG_BY_VERSION) { if (pFilterInfo->sver > pIter->sIter.bData.aVersion[pIter->sIter.iRow] || pFilterInfo->ever < pIter->sIter.bData.aVersion[pIter->sIter.iRow]) { @@ -395,6 +402,16 @@ static int32_t tsdbTombFileDataIterNext(STsdbDataIter2* pIter, STsdbFilterInfo* if (pIter->tIter.iDelIdx < taosArrayGetSize(pIter->tIter.aDelIdx)) { SDelIdx* pDelIdx = taosArrayGet(pIter->tIter.aDelIdx, pIter->tIter.iDelIdx); + if (pFilterInfo) { + if (pFilterInfo->flag & TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE) { + SMetaInfo info; + if (metaGetInfo(pIter->dIter.pReader->pTsdb->pVnode->pMeta, pDelIdx->uid, &info, NULL)) { + pIter->tIter.iDelIdx++; + continue; + } + } + } + code = tsdbReadDelData(pIter->tIter.pReader, pDelIdx, pIter->tIter.aDelData); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index aed863d194..dfea125cc1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -70,10 +70,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) { if (pReader->pIter) { // iter to next with filter info (sver, ever) - code = tsdbDataIterNext2(pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag - .sver = pReader->sver, - .ever = pReader->ever}); + code = tsdbDataIterNext2( + pReader->pIter, + &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag + .sver = pReader->sver, + .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { @@ -94,10 +95,11 @@ static int32_t tsdbSnapReadFileDataStart(STsdbSnapReader* pReader) { if (pReader->pIter) { // iter to valid row - code = tsdbDataIterNext2(pReader->pIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag - .sver = pReader->sver, - .ever = pReader->ever}); + code = tsdbDataIterNext2( + pReader->pIter, + &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag + .sver = pReader->sver, + .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); if (pReader->pIter->rowInfo.suid || pReader->pIter->rowInfo.uid) { @@ -139,7 +141,8 @@ static int32_t tsdbSnapReadNextRow(STsdbSnapReader* pReader, SRowInfo** ppRowInf int32_t lino = 0; if (pReader->pIter) { - code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, // flag + code = tsdbDataIterNext2(pReader->pIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | + TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, // flag .sver = pReader->sver, .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); @@ -346,8 +349,9 @@ static int32_t tsdbSnapReadNextTombData(STsdbSnapReader* pReader, SDelInfo** ppD int32_t lino = 0; code = tsdbDataIterNext2( - pReader->pTIter, - &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION, .sver = pReader->sver, .ever = pReader->ever}); + pReader->pTIter, &(STsdbFilterInfo){.flag = TSDB_FILTER_FLAG_BY_VERSION | TSDB_FILTER_FLAG_IGNORE_DROPPED_TABLE, + .sver = pReader->sver, + .ever = pReader->ever}); TSDB_CHECK_CODE(code, lino, _exit); if (ppDelInfo) { From e584bb3ea17e272135a5cbfdb8b13b94749d991c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 25 Apr 2023 17:39:28 +0800 Subject: [PATCH 061/145] feat: subtable level privilege --- include/libs/nodes/querynodes.h | 2 + source/client/src/clientMain.c | 32 +++--- source/libs/catalog/src/ctgUtil.c | 6 +- source/libs/nodes/src/nodesUtilFuncs.c | 2 + source/libs/parser/src/parAuthenticator.c | 11 +- source/libs/parser/src/parInsertSql.c | 114 ++++++++++++++++++--- source/libs/scalar/src/scalar.c | 26 ++--- tests/parallel_test/cases.task | 1 + tests/script/tsim/user/privilege_table.sim | 34 ++++-- tests/script/win-test-file | 1 + 10 files changed, 177 insertions(+), 52 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 480912a8cf..9569cfe055 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -379,6 +379,8 @@ typedef struct SVnodeModifyOpStmt { SName usingTableName; const char* pBoundCols; struct STableMeta* pTableMeta; + SNode* pTagCond; + SArray* pTableTag; SHashObj* pVgroupsHashObj; SHashObj* pTableBlockHashObj; // SHashObj SHashObj* pSubTableHashObj; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 54ae1ab4b3..2ebc8e7379 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -108,7 +108,7 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha if (pass == NULL) { pass = TSDB_DEFAULT_PASS; } - + STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); if (pObj) { int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); @@ -359,11 +359,11 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) case TSDB_DATA_TYPE_NCHAR: { int32_t charLen = varDataLen((char *)row[i] - VARSTR_HEADER_SIZE); if (fields[i].type == TSDB_DATA_TYPE_BINARY) { - if(ASSERT(charLen <= fields[i].bytes && charLen >= 0)){ + if (ASSERT(charLen <= fields[i].bytes && charLen >= 0)) { tscError("taos_print_row error binary. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); } } else { - if(ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)){ + if (ASSERT(charLen <= fields[i].bytes * TSDB_NCHAR_SIZE && charLen >= 0)) { tscError("taos_print_row error. charLen:%d, fields[i].bytes:%d", charLen, fields[i].bytes); } } @@ -705,16 +705,16 @@ int taos_get_current_db(TAOS *taos, char *database, int len, int *required) { int code = TSDB_CODE_SUCCESS; taosThreadMutexLock(&pTscObj->mutex); - if(database == NULL || len <= 0){ - if(required != NULL) *required = strlen(pTscObj->db) + 1; + if (database == NULL || len <= 0) { + if (required != NULL) *required = strlen(pTscObj->db) + 1; terrno = TSDB_CODE_INVALID_PARA; code = -1; - }else if(len < strlen(pTscObj->db) + 1){ + } else if (len < strlen(pTscObj->db) + 1) { tstrncpy(database, pTscObj->db, len); - if(required) *required = strlen(pTscObj->db) + 1; + if (required) *required = strlen(pTscObj->db) + 1; terrno = TSDB_CODE_INVALID_PARA; code = -1; - }else{ + } else { strcpy(database, pTscObj->db); code = 0; } @@ -741,6 +741,7 @@ static void destoryCatalogReq(SCatalogReq *pCatalogReq) { taosArrayDestroy(pCatalogReq->pUser); taosArrayDestroy(pCatalogReq->pTableIndex); taosArrayDestroy(pCatalogReq->pTableCfg); + taosArrayDestroy(pCatalogReq->pTableTag); taosMemoryFree(pCatalogReq); } @@ -976,7 +977,7 @@ void doAsyncQuery(SRequestObj *pRequest, bool updateMetaForce) { if (TSDB_CODE_SUCCESS == code) { pRequest->stmtType = pRequest->pQuery->pRoot->type; code = phaseAsyncQuery(pWrapper); - } + } if (TSDB_CODE_SUCCESS != code) { tscError("0x%" PRIx64 " error happens, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, tstrerror(code), @@ -1044,11 +1045,11 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { } void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - if(ASSERT(res != NULL && fp != NULL)){ + if (ASSERT(res != NULL && fp != NULL)) { tscError("taos_fetch_rows_a invalid paras"); return; } - if(ASSERT(TD_RES_QUERY(res))){ + if (ASSERT(TD_RES_QUERY(res))) { tscError("taos_fetch_rows_a res is NULL"); return; } @@ -1094,11 +1095,11 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { - if(ASSERT(res != NULL && fp != NULL)){ + if (ASSERT(res != NULL && fp != NULL)) { tscError("taos_fetch_rows_a invalid paras"); return; } - if(ASSERT(TD_RES_QUERY(res))){ + if (ASSERT(TD_RES_QUERY(res))) { tscError("taos_fetch_rows_a res is NULL"); return; } @@ -1113,11 +1114,11 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { } const void *taos_get_raw_block(TAOS_RES *res) { - if(ASSERT(res != NULL)){ + if (ASSERT(res != NULL)) { tscError("taos_fetch_rows_a invalid paras"); return NULL; } - if(ASSERT(TD_RES_QUERY(res))){ + if (ASSERT(TD_RES_QUERY(res))) { tscError("taos_fetch_rows_a res is NULL"); return NULL; } @@ -1275,7 +1276,6 @@ _return: return code; } - int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (NULL == taos) { terrno = TSDB_CODE_TSC_DISCONNECTED; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index a42a189e75..9274c1ef92 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -170,6 +170,9 @@ void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pTableCfg); pData->pTableCfg = NULL; + taosArrayDestroy(pData->pTableTag); + pData->pTableTag = NULL; + taosMemoryFreeClear(pData->pSvrVer); } @@ -530,7 +533,7 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) { } case CTG_TASK_GET_USER: { if (*pRes) { - SUserAuthRes* pAuth = (SUserAuthRes *)*pRes; + SUserAuthRes* pAuth = (SUserAuthRes*)*pRes; nodesDestroyNode(pAuth->pCond); taosMemoryFreeClear(*pRes); } @@ -711,6 +714,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { SCtgTbTagCtx* taskCtx = (SCtgTbTagCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); taosMemoryFreeClear(taskCtx->pVgInfo); + taosMemoryFreeClear(taskCtx); break; } case CTG_TASK_GET_DB_VGROUP: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 422a196c50..f71eef7969 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -827,6 +827,8 @@ void nodesDestroyNode(SNode* pNode) { SVnodeModifyOpStmt* pStmt = (SVnodeModifyOpStmt*)pNode; destroyVgDataBlockArray(pStmt->pDataBlocks); taosMemoryFreeClear(pStmt->pTableMeta); + nodesDestroyNode(pStmt->pTagCond); + taosArrayDestroy(pStmt->pTableTag); taosHashCleanup(pStmt->pVgroupsHashObj); taosHashCleanup(pStmt->pSubTableHashObj); taosHashCleanup(pStmt->pTableNameHashObj); diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index d182f5bd73..1586d8128b 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -91,17 +91,22 @@ static int32_t mergeStableTagCond(SNode** pWhere, SNode* pTagCond) { } static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) { + SNode* pTagCondCopy = nodesCloneNode(pTagCond); + if (NULL == pTagCondCopy) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (NULL == *pWhere) { - *pWhere = pTagCond; + *pWhere = pTagCondCopy; return TSDB_CODE_SUCCESS; } if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pWhere) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pWhere)->condType) { - return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCond); + return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy); } - return mergeStableTagCond(pWhere, pTagCond); + return mergeStableTagCond(pWhere, pTagCondCopy); } static EDealRes authSelectImpl(SNode* pNode, void* pContext) { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index eb2efd573d..02de9f227d 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -53,6 +53,7 @@ typedef struct SInsertParseContext { bool missCache; bool usingDuplicateTable; bool forceUpdate; + bool needTableTagVal; } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -577,28 +578,39 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { if (NULL == pValue) { return TSDB_CODE_OUT_OF_MEMORY; } - pValue->node.resType.type = pVal->type; + + pValue->node.resType = ((SColumnNode*)*pNode)->node.resType; + nodesDestroyNode(*pNode); + *pNode = (SNode*)pValue; + switch (pVal->type) { case TSDB_DATA_TYPE_BOOL: pValue->datum.b = *(int8_t*)(&pVal->i64); + *(bool*)&pValue->typeData = pValue->datum.b; break; case TSDB_DATA_TYPE_TINYINT: pValue->datum.i = *(int8_t*)(&pVal->i64); + *(int8_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_SMALLINT: pValue->datum.i = *(int16_t*)(&pVal->i64); + *(int16_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_INT: pValue->datum.i = *(int32_t*)(&pVal->i64); + *(int32_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_BIGINT: pValue->datum.i = pVal->i64; + pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_FLOAT: pValue->datum.d = *(float*)(&pVal->i64); + *(float*)&pValue->typeData = pValue->datum.d; break; case TSDB_DATA_TYPE_DOUBLE: pValue->datum.d = *(double*)(&pVal->i64); + *(double*)&pValue->typeData = pValue->datum.d; break; case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_NCHAR: @@ -611,18 +623,23 @@ static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { break; case TSDB_DATA_TYPE_TIMESTAMP: pValue->datum.i = pVal->i64; + pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_UTINYINT: pValue->datum.i = *(uint8_t*)(&pVal->i64); + *(uint8_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_USMALLINT: pValue->datum.i = *(uint16_t*)(&pVal->i64); + *(uint16_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_UINT: pValue->datum.i = *(uint32_t*)(&pVal->i64); + *(uint32_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_UBIGINT: pValue->datum.i = *(uint64_t*)(&pVal->i64); + *(uint64_t*)&pValue->typeData = pValue->datum.i; break; case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_VARBINARY: @@ -667,16 +684,15 @@ static int32_t checkTagCondResult(SNode* pResult) { : TSDB_CODE_PAR_PERMISSION_DENIED; } -int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode* pCond) { - int32_t code = setTagVal(pTagVals, pTagName, pCond); - SNode* pNew = NULL; +static int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode** pCond) { + int32_t code = setTagVal(pTagVals, pTagName, *pCond); if (TSDB_CODE_SUCCESS == code) { - code = scalarCalculateConstants(pCond, &pNew); + code = scalarCalculateConstants(*pCond, pCond); } if (TSDB_CODE_SUCCESS == code) { - code = checkTagCondResult(pNew); + code = checkTagCondResult(*pCond); } - nodesDestroyNode(pNew); + NODES_DESTORY_NODE(*pCond); return code; } @@ -716,6 +732,10 @@ static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt } } + if (TSDB_CODE_SUCCESS == code && NULL != pStmt->pTagCond) { + code = checkSubtablePrivilege(pTagVals, pTagName, &pStmt->pTagCond); + } + if (TSDB_CODE_SUCCESS == code && !isParseBindParam && !isJson) { code = tTagNew(pTagVals, 1, false, &pTag); } @@ -843,7 +863,7 @@ static void setUserAuthInfo(SParseContext* pCxt, SName* pTbName, SUserAuthInfo* pInfo->type = AUTH_TYPE_WRITE; } -static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) { +static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) { int32_t code = TSDB_CODE_SUCCESS; SUserAuthInfo authInfo = {0}; setUserAuthInfo(pCxt, pTbName, &authInfo); @@ -863,11 +883,28 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) *pMissCache = true; } else if (!authRes.pass) { code = TSDB_CODE_PAR_PERMISSION_DENIED; + } else if (NULL != authRes.pCond) { + *pTagCond = authRes.pCond; } } return code; } +static int32_t checkAuthForTable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, bool* pNeedTableTagVal) { + SNode* pTagCond = NULL; + int32_t code = checkAuth(pCxt, pTbName, pMissCache, &pTagCond); + if (TSDB_CODE_SUCCESS == code) { + *pNeedTableTagVal = ((*pMissCache) || (NULL != pTagCond)); + *pMissCache = (NULL != pTagCond); + } + nodesDestroyNode(pTagCond); + return code; +} + +static int32_t checkAuthForStable(SParseContext* pCxt, SName* pTbName, bool* pMissCache, SNode** pTagCond) { + return checkAuth(pCxt, pTbName, pMissCache, pTagCond); +} + static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isStb, STableMeta** pTableMeta, bool* pMissCache) { SParseContext* pComCxt = pCxt->pComCxt; @@ -970,7 +1007,7 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStm return TSDB_CODE_SUCCESS; } - int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); + int32_t code = checkAuthForTable(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pCxt->needTableTagVal); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); } @@ -993,7 +1030,7 @@ static int32_t getUsingTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStmt return TSDB_CODE_SUCCESS; } - int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache); + int32_t code = checkAuthForStable(pCxt->pComCxt, &pStmt->usingTableName, &pCxt->missCache, &pStmt->pTagCond); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTableMeta(pCxt, &pStmt->usingTableName, true, &pStmt->pTableMeta, &pCxt->missCache); } @@ -1606,6 +1643,8 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod static void resetEnvPreTable(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { insDestroyBoundColInfo(&pCxt->tags); taosMemoryFreeClear(pStmt->pTableMeta); + nodesDestroyNode(pStmt->pTagCond); + taosArrayDestroy(pStmt->pTableTag); tdDestroySVCreateTbReq(pStmt->pCreateTblReq); taosMemoryFreeClear(pStmt->pCreateTblReq); pCxt->missCache = false; @@ -1780,14 +1819,18 @@ static int32_t createInsertQuery(SInsertParseContext* pCxt, SQuery** pOutput) { return code; } -static int32_t checkAuthFromMetaData(const SArray* pUsers) { +static int32_t checkAuthFromMetaData(const SArray* pUsers, SNode** pTagCond) { if (1 != taosArrayGetSize(pUsers)) { return TSDB_CODE_FAILED; } SMetaRes* pRes = taosArrayGet(pUsers, 0); if (TSDB_CODE_SUCCESS == pRes->code) { - return (*(bool*)pRes->pRes) ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED; + SUserAuthRes* pAuth = pRes->pRes; + if (NULL != pAuth->pCond) { + *pTagCond = nodesCloneNode(pAuth->pCond); + } + return pAuth->pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED; } return pRes->code; } @@ -1826,9 +1869,40 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifyOpS sizeof(SVgroupInfo)); } +static int32_t buildTagNameFromMeta(STableMeta* pMeta, SArray** pTagName) { + *pTagName = taosArrayInit(pMeta->tableInfo.numOfTags, TSDB_COL_NAME_LEN); + if (NULL == *pTagName) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SSchema* pSchema = getTableTagSchema(pMeta); + for (int32_t i = 0; i < pMeta->tableInfo.numOfTags; ++i) { + taosArrayPush(*pTagName, pSchema[i].name); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t checkSubtablePrivilegeForTable(const SArray* pTables, SVnodeModifyOpStmt* pStmt) { + if (1 != taosArrayGetSize(pTables)) { + return TSDB_CODE_FAILED; + } + + SMetaRes* pRes = taosArrayGet(pTables, 0); + if (TSDB_CODE_SUCCESS != pRes->code) { + return pRes->code; + } + + SArray* pTagName = NULL; + int32_t code = buildTagNameFromMeta(pStmt->pTableMeta, &pTagName); + if (TSDB_CODE_SUCCESS == code) { + code = checkSubtablePrivilege((SArray*)pRes->pRes, pTagName, &pStmt->pTagCond); + } + taosArrayDestroy(pTagName); + return code; +} + static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, SVnodeModifyOpStmt* pStmt, bool isStb) { - int32_t code = checkAuthFromMetaData(pMetaData->pUser); + int32_t code = checkAuthFromMetaData(pMetaData->pUser, &pStmt->pTagCond); if (TSDB_CODE_SUCCESS == code) { code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); } @@ -1841,6 +1915,9 @@ static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMeta if (TSDB_CODE_SUCCESS == code) { code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); } + if (TSDB_CODE_SUCCESS == code && !isStb && NULL != pStmt->pTagCond) { + code = checkSubtablePrivilegeForTable(pMetaData->pTableTag, pStmt); + } return code; } @@ -1860,6 +1937,8 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { pCatalogReq->pTableHash = NULL; taosArrayDestroy(pCatalogReq->pUser); pCatalogReq->pUser = NULL; + taosArrayDestroy(pCatalogReq->pTableTag); + pCatalogReq->pTableTag = NULL; } static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, @@ -2033,8 +2112,15 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** return TSDB_CODE_SUCCESS; } +static int32_t buildInsertTableTagReq(SName* pName, SArray** pTables) { return buildInsertTableReq(pName, pTables); } + static int32_t buildInsertCatalogReq(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SCatalogReq* pCatalogReq) { - int32_t code = buildInsertUserAuthReq(pCxt->pComCxt->pUser, &pStmt->targetTableName, &pCatalogReq->pUser); + int32_t code = buildInsertUserAuthReq( + pCxt->pComCxt->pUser, (0 == pStmt->usingTableName.type ? &pStmt->targetTableName : &pStmt->usingTableName), + &pCatalogReq->pUser); + if (TSDB_CODE_SUCCESS == code && pCxt->needTableTagVal) { + code = buildInsertTableTagReq(&pStmt->targetTableName, &pCatalogReq->pTableTag); + } if (TSDB_CODE_SUCCESS == code) { if (0 == pStmt->usingTableName.type) { code = buildInsertDbReq(&pStmt->targetTableName, &pCatalogReq->pTableMeta); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 0521076d23..d9295656e8 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -53,6 +53,7 @@ int32_t sclCreateColumnInfoData(SDataType *pType, int32_t numOfRows, SScalarPara int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows, true); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; + colDataDestroy(pColumnData); taosMemoryFree(pColumnData); return terrno; } @@ -1061,17 +1062,20 @@ int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) { if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) { if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) { - SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft)); + SCL_ERR_JRET( + sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode *)node->pLeft)); } } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) { if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) { if (SCL_IS_VAR_VALUE_NODE(node->pRight)) { - SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), + (SValueNode *)node->pRight)); } else if (QUERY_NODE_NODE_LIST == node->pRight->type) { - SNode* pNode; - FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) { + SNode *pNode; + FOREACH(pNode, ((SNodeListNode *)node->pRight)->pNodeList) { if (SCL_IS_VAR_VALUE_NODE(pNode)) { - SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode)); + SCL_ERR_JRET( + sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode *)pNode)); } } } @@ -1086,8 +1090,6 @@ _return: return DEAL_RES_ERROR; } - - int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) { int32_t code = 0; @@ -1096,19 +1098,20 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) { } if (SCL_IS_VAR_VALUE_NODE(node->pCase)) { - SNode* pNode; + SNode *pNode; FOREACH(pNode, node->pWhenThenList) { SExprNode *pExpr = (SExprNode *)((SWhenThenNode *)pNode)->pWhen; if (TSDB_DATA_TYPE_TIMESTAMP == pExpr->resType.type) { - SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode*)node->pCase)); + SCL_ERR_JRET(sclConvertToTsValueNode(pExpr->resType.precision, (SValueNode *)node->pCase)); break; } } } else if (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pCase)->resType.type) { - SNode* pNode; + SNode *pNode; FOREACH(pNode, node->pWhenThenList) { if (SCL_IS_VAR_VALUE_NODE(((SWhenThenNode *)pNode)->pWhen)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, (SValueNode*)((SWhenThenNode *)pNode)->pWhen)); + SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pCase)->resType.precision, + (SValueNode *)((SWhenThenNode *)pNode)->pWhen)); } } } @@ -1271,7 +1274,6 @@ EDealRes sclRewriteLogic(SNode **pNode, SScalarCtx *ctx) { return DEAL_RES_CONTINUE; } - EDealRes sclRewriteOperator(SNode **pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index dda4ec3e84..8ea13e741c 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -722,6 +722,7 @@ ,,y,script,./test.sh -f tsim/user/privilege_db.sim ,,y,script,./test.sh -f tsim/user/privilege_sysinfo.sim ,,y,script,./test.sh -f tsim/user/privilege_topic.sim +,,y,script,./test.sh -f tsim/user/privilege_table.sim ,,y,script,./test.sh -f tsim/db/alter_option.sim ,,y,script,./test.sh -f tsim/db/alter_replica_31.sim ,,y,script,./test.sh -f tsim/db/basic1.sim diff --git a/tests/script/tsim/user/privilege_table.sim b/tests/script/tsim/user/privilege_table.sim index 5256cdd21d..05f91ff5b0 100644 --- a/tests/script/tsim/user/privilege_table.sim +++ b/tests/script/tsim/user/privilege_table.sim @@ -96,7 +96,9 @@ sql connect wxy sql reset query cache; sql_error select tbname, * from test.st1; sql insert into test.st1s1 values(now, 10); +sql insert into test.st1s3 using test.st1 tags(1, 'dachang') values(now, 100); sql_error insert into test.st1s2 values(now, 20); +sql_error insert into test.st1s4 using test.st1 tags(3, 'dachang') values(now, 300); sql_error select * from test.st2; sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); @@ -112,7 +114,7 @@ sql connect wxy sql reset query cache; sql select * from test.st1; -if $rows != 5 then +if $rows != 6 then return -1 endi sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -133,7 +135,7 @@ sql connect wxy sql reset query cache; sql select * from test.st1; -if $rows != 2 then +if $rows != 6 then return -1 endi sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -153,8 +155,9 @@ sql grant read on test.st1 with id = 1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; -if $rows != 1 then +if $rows != 4 then return -1 endi sql_error insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -174,8 +177,9 @@ sql grant write on test.st1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; -if $rows != 2 then +if $rows != 6 then return -1 endi sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); @@ -195,10 +199,17 @@ sql grant write on test.st1 with id = 1 to wxy; sql close sql connect wxy -sql select * from test.st1; +sql reset query cache; +sql select * from test.st1; +if $rows != 8 then + return -1 +endi sql insert into test.st1s1 values(now, 10); sql_error insert into test.st1s2 values(now, 20); -sql select * from test.st2; +sql select * from test.st2; +if $rows != 2 then + return -1 +endi sql_error insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); print =============== case 11: database write privilege and table unauthorized @@ -212,6 +223,7 @@ sql grant write on test.* to wxy; sql close sql connect wxy +sql reset query cache; sql_error select * from test.st1; sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; @@ -226,7 +238,11 @@ sql grant read on test.st1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; +if $rows != 11 then + return -1 +endi sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); @@ -241,7 +257,11 @@ sql grant read on test.st1 with id = 1 to wxy; sql close sql connect wxy +sql reset query cache; sql select * from test.st1; +if $rows != 8 then + return -1 +endi sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; sql insert into test.st2s1 values(now, 10) test.st2s2 values(now, 20); @@ -256,6 +276,7 @@ sql grant write on test.st1 to wxy; sql close sql connect wxy +sql reset query cache; sql_error select * from test.st1; sql insert into test.st1s1 values(now, 10) test.st1s2 values(now, 20); sql_error select * from test.st2; @@ -271,6 +292,7 @@ sql grant write on test.st1 with id = 1 to wxy; sql close sql connect wxy +sql reset query cache; sql_error select * from test.st1; sql insert into test.st1s1 values(now, 10); sql_error insert into test.st1s2 values(now, 20); diff --git a/tests/script/win-test-file b/tests/script/win-test-file index 2d5a1b3108..3b358993fa 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -3,6 +3,7 @@ ./test.sh -f tsim/user/privilege_db.sim ./test.sh -f tsim/user/privilege_sysinfo.sim ./test.sh -f tsim/user/privilege_topic.sim +./test.sh -f tsim/user/privilege_table.sim ./test.sh -f tsim/db/alter_option.sim rem ./test.sh -f tsim/db/alter_replica_13.sim ./test.sh -f tsim/db/alter_replica_31.sim From 7b7a483afc64230822519cadec80d4acfe231ee1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 25 Apr 2023 18:43:48 +0800 Subject: [PATCH 062/145] fix(query): add tmp variable to hold the TSDBROW value. --- source/dnode/mnode/impl/src/mndMain.c | 6 +----- source/dnode/vnode/src/tsdb/tsdbRead.c | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 5c20887cf5..92ff550895 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -124,11 +124,7 @@ static void mndCalMqRebalance(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); if (pReq != NULL) { - SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_TIMER, - .pCont = pReq, - .contLen = contLen, - }; + SRpcMsg rpcMsg = { .msgType = TDMT_MND_TMQ_TIMER, .pCont = pReq, .contLen = contLen }; tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 0ad5a44919..9922312bef 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1876,10 +1876,10 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); int64_t key = pRow->pBlockData->aTSKEY[pRow->iRow]; - int64_t ver = pRow->version; + int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; pLastBlockReader->currentKey = key; - pScanInfo->lastKeyInStt = ver; + pScanInfo->lastKeyInStt = key; if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { // the qualifed ts may equal to k.ts, only a greater version one. @@ -2132,16 +2132,21 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, bool mergeBlockData) { SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); - bool copied = false; - int32_t code = TSDB_CODE_SUCCESS; - SRow* pTSRow = NULL; - TSDBROW *pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + bool copied = false; + int32_t code = TSDB_CODE_SUCCESS; + SRow* pTSRow = NULL; + TSDBROW* pRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + + // create local variable to hold the row value + TSDBROW fRow = {.iRow = pRow->iRow, .type = TSDBROW_COL_FMT, .pBlockData = pRow->pBlockData}; + tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, pReader->idStr); // only last block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { - code = tryCopyDistinctRowFromSttBlock(pRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); + code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); if (code) { return code; } @@ -2150,7 +2155,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2174,7 +2179,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pReader->pSchema); + code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } From e9f4c769d8d753c18f79f7d3c9d077683fc57e12 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Tue, 25 Apr 2023 19:20:42 +0800 Subject: [PATCH 063/145] enhance: packaging keeper in deb rpm main (#21071) * enhance: community package include taoskeeper * enhance: community rpm package add taoskeeper --- packaging/deb/DEBIAN/preinst | 37 ++++++++++++++++++++++++++++++++++ packaging/deb/DEBIAN/prerm | 1 + packaging/deb/makedeb.sh | 26 ++++++++++++++++++++++++ packaging/rpm/makerpm.sh | 24 +++++++++++++++++++++- packaging/rpm/tdengine.spec | 21 +++++++++++++++++-- packaging/tools/post.sh | 39 ++++++++++++++++++++++++++---------- packaging/tools/preun.sh | 17 +++++++++++++++- 7 files changed, 150 insertions(+), 15 deletions(-) diff --git a/packaging/deb/DEBIAN/preinst b/packaging/deb/DEBIAN/preinst index 8a1a7d4d81..904a946e20 100644 --- a/packaging/deb/DEBIAN/preinst +++ b/packaging/deb/DEBIAN/preinst @@ -26,6 +26,38 @@ if pidof taosd &> /dev/null; then sleep 1 fi +# Stop adapter service if running +if pidof taosadapter &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taosadapter || : + elif $(which service &> /dev/null); then + ${csudo}service taosadapter stop || : + else + pid=$(ps -ef | grep "taosadapter" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taosadapter service success!" + sleep 1 +fi + +# Stop keeper service if running +if pidof taoskeeper &> /dev/null; then + if pidof systemd &> /dev/null; then + ${csudo}systemctl stop taoskeeper || : + elif $(which service &> /dev/null); then + ${csudo}service taoskeeper stop || : + else + pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi + fi + echo "Stop taoskeeper service success!" + sleep 1 +fi + # if taos.cfg already softlink, remove it cfg_install_dir="/etc/taos" install_main_dir="/usr/local/taos" @@ -41,6 +73,11 @@ if [ -f "${install_main_dir}/taosadapter.service" ]; then ${csudo}rm -f ${install_main_dir}/cfg/taosadapter.service || : fi +if [ -f "${install_main_dir}/taoskeeper.toml" ]; then + ${csudo}rm -f ${install_main_dir}/cfg/taoskeeper.toml || : +fi + + # there can not libtaos.so*, otherwise ln -s error ${csudo}rm -f ${install_main_dir}/driver/libtaos.* || : [ -f ${install_main_dir}/driver/libtaosws.so ] && ${csudo}rm -f ${install_main_dir}/driver/libtaosws.so || : diff --git a/packaging/deb/DEBIAN/prerm b/packaging/deb/DEBIAN/prerm index 65f261db2c..0d63115a04 100644 --- a/packaging/deb/DEBIAN/prerm +++ b/packaging/deb/DEBIAN/prerm @@ -32,6 +32,7 @@ else ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 94a24a4148..9f49cf345a 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -44,8 +44,31 @@ mkdir -p ${pkg_dir}${install_home_path}/include #mkdir -p ${pkg_dir}${install_home_path}/init.d mkdir -p ${pkg_dir}${install_home_path}/script +# download taoskeeper and build +if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then + arch=amd64 +elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then + arch=386 +elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then + arch=arm +elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then + arch=arm64 +else + arch=$cpuType +fi + +echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper" +echo "$top_dir=${top_dir}" +taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper` +echo "taoskeeper_binary: ${taoskeeper_binary}" + +# copy config files +cp $(dirname ${taoskeeper_binary})/config/taoskeeper.toml ${pkg_dir}${install_home_path}/cfg +cp $(dirname ${taoskeeper_binary})/taoskeeper.service ${pkg_dir}${install_home_path}/cfg + cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg cp ${compile_dir}/../packaging/cfg/taosd.service ${pkg_dir}${install_home_path}/cfg + if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then cp ${compile_dir}/test/cfg/taosadapter.toml ${pkg_dir}${install_home_path}/cfg || : fi @@ -53,6 +76,7 @@ if [ -f "${compile_dir}/test/cfg/taosadapter.service" ]; then cp ${compile_dir}/test/cfg/taosadapter.service ${pkg_dir}${install_home_path}/cfg || : fi +cp ${taoskeeper_binary} ${pkg_dir}${install_home_path}/bin #cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_path}/init.d cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script @@ -143,6 +167,7 @@ else exit 1 fi +rm -rf ${pkg_dir}/build-taoskeeper # make deb package dpkg -b ${pkg_dir} $debname echo "make deb package success!" @@ -150,4 +175,5 @@ echo "make deb package success!" cp ${pkg_dir}/*.deb ${output_dir} # clean temp dir + rm -rf ${pkg_dir} diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh index 4ac67ec754..9cf00364aa 100755 --- a/packaging/rpm/makerpm.sh +++ b/packaging/rpm/makerpm.sh @@ -35,14 +35,16 @@ function cp_rpm_package() { local cur_dir cd $1 cur_dir=$(pwd) - + echo "cp_rpm_package cd: ${cur_dir}" for dirlist in "$(ls ${cur_dir})"; do if test -d ${dirlist}; then cd ${dirlist} + echo 'cp_rpm_package ${cur_dir}/${dirlist}' cp_rpm_package ${cur_dir}/${dirlist} cd .. fi if test -e ${dirlist}; then + echo "${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm" cp ${cur_dir}/${dirlist} ${output_dir}/TDengine-${tdengine_ver}.rpm fi done @@ -54,6 +56,25 @@ fi ${csudo}mkdir -p ${pkg_dir} cd ${pkg_dir} +# download taoskeeper and build +if [ "$cpuType" = "x64" ] || [ "$cpuType" = "x86_64" ] || [ "$cpuType" = "amd64" ]; then + arch=amd64 +elif [ "$cpuType" = "x32" ] || [ "$cpuType" = "i386" ] || [ "$cpuType" = "i686" ]; then + arch=386 +elif [ "$cpuType" = "arm" ] || [ "$cpuType" = "aarch32" ]; then + arch=arm +elif [ "$cpuType" = "arm64" ] || [ "$cpuType" = "aarch64" ]; then + arch=arm64 +else + arch=$cpuType +fi + +cd ${top_dir} +echo "${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r ${arch} -e taoskeeper" +taoskeeper_binary=`${top_dir}/../enterprise/packaging/build_taoskeeper.sh -r $arch -e taoskeeper` +echo "taoskeeper_binary: ${taoskeeper_binary}" +cd ${package_dir} + ${csudo}mkdir -p BUILD BUILDROOT RPMS SOURCES SPECS SRPMS ${csudo}rpmbuild --define="_version ${tdengine_ver}" --define="_topdir ${pkg_dir}" --define="_compiledir ${compile_dir}" -bb ${spec_file} @@ -85,3 +106,4 @@ mv ${output_dir}/TDengine-${tdengine_ver}.rpm ${output_dir}/${rpmname} cd .. ${csudo}rm -rf ${pkg_dir} +rm -rf ${top_dir}/build-taoskeeper \ No newline at end of file diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index c21063e6a4..52d5335003 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -3,6 +3,7 @@ %define cfg_install_dir /etc/taos %define __strip /bin/true %global __python /usr/bin/python3 +%global _build_id_links none Name: tdengine Version: %{_version} @@ -62,6 +63,15 @@ fi if [ -f %{_compiledir}/test/cfg/taosadapter.service ]; then cp %{_compiledir}/test/cfg/taosadapter.service %{buildroot}%{homepath}/cfg fi + +if [ -f %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml ]; then + cp %{_compiledir}/../build-taoskeeper/config/taoskeeper.toml %{buildroot}%{homepath}/cfg ||: +fi + +if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper.service ]; then + cp %{_compiledir}/../build-taoskeeper/taoskeeper.service %{buildroot}%{homepath}/cfg ||: +fi + #cp %{_compiledir}/../packaging/rpm/taosd %{buildroot}%{homepath}/init.d cp %{_compiledir}/../packaging/tools/post.sh %{buildroot}%{homepath}/script cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/script @@ -73,8 +83,12 @@ cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/udfd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosBenchmark %{buildroot}%{homepath}/bin +if [ -f %{_compiledir}/../build-taoskeeper/taoskeeper ]; then + cp %{_compiledir}/../build-taoskeeper/taoskeeper %{buildroot}%{homepath}/bin +fi + if [ -f %{_compiledir}/build/bin/taosadapter ]; then - cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin ||: + cp %{_compiledir}/build/bin/taosadapter %{buildroot}%{homepath}/bin fi cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver [ -f %{_compiledir}/build/lib/${wslibfile} ] && cp %{_compiledir}/build/lib/${wslibfile} %{buildroot}%{homepath}/driver ||: @@ -119,7 +133,9 @@ if [ -f %{_compiledir}/build/bin/jemalloc-config ]; then cp %{_compiledir}/build/lib/pkgconfig/jemalloc.pc %{buildroot}%{homepath}/jemalloc/lib/pkgconfig fi fi - +ls -al %{buildroot}%{homepath}/bin +tree -L 5 +echo "==============================copying files done" #Scripts executed before installation %pre if [ -f /var/lib/taos/dnode/dnodeCfg.json ]; then @@ -196,6 +212,7 @@ if [ $1 -eq 0 ];then ${csudo}rm -f ${bin_link_dir}/taosd || : ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosadapter || : + ${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${cfg_link_dir}/* || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 3a013ade2c..fc392c9684 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -436,7 +436,7 @@ function local_fqdn_check() { function install_taosadapter_config() { if [ ! -f "${cfg_install_dir}/taosadapter.toml" ]; then - [ ! -d %{cfg_install_dir} ] && + [ ! -d ${cfg_install_dir} ] && ${csudo}${csudo}mkdir -p ${cfg_install_dir} [ -f ${cfg_dir}/taosadapter.toml ] && ${csudo}cp ${cfg_dir}/taosadapter.toml ${cfg_install_dir} [ -f ${cfg_install_dir}/taosadapter.toml ] && @@ -451,19 +451,26 @@ function install_taosadapter_config() { } function install_taoskeeper_config() { - if [ ! -f "${cfg_install_dir}/keeper.toml" ]; then - [ ! -d %{cfg_install_dir} ] && - ${csudo}${csudo}mkdir -p ${cfg_install_dir} - [ -f ${cfg_dir}/keeper.toml ] && ${csudo}cp ${cfg_dir}/keeper.toml ${cfg_install_dir} - [ -f ${cfg_install_dir}/keeper.toml ] && - ${csudo}chmod 644 ${cfg_install_dir}/keeper.toml + # if new environment without taoskeeper + if [[ ! -f "${cfg_install_dir}/keeper.toml" ]] && [[ ! -f "${cfg_install_dir}/taoskeeper.toml" ]]; then + [ ! -d ${cfg_install_dir} ] && ${csudo}${csudo}mkdir -p ${cfg_install_dir} + [ -f ${cfg_dir}/taoskeeper.toml ] && ${csudo}cp ${cfg_dir}/taoskeeper.toml ${cfg_install_dir} + [ -f ${cfg_install_dir}/taoskeeper.toml ] && + ${csudo}chmod 644 ${cfg_install_dir}/taoskeeper.toml + fi + # if old machine with taoskeeper.toml file + if [ -f ${cfg_install_dir}/taoskeeper.toml ]; then + ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new fi - [ -f ${cfg_dir}/keeper.toml ] && - ${csudo}mv ${cfg_dir}/keeper.toml ${cfg_dir}/keeper.toml.new + if [ -f ${cfg_install_dir}/keeper.toml ]; then + echo "The file keeper.toml will be renamed to taoskeeper.toml" + ${csudo}mv ${cfg_install_dir}/keeper.toml ${cfg_install_dir}/taoskeeper.toml + ${csudo}mv ${cfg_dir}/taoskeeper.toml ${cfg_dir}/taoskeeper.toml.new + fi - [ -f ${cfg_install_dir}/keeper.toml ] && - ${csudo}ln -s ${cfg_install_dir}/keeper.toml ${cfg_dir} + [ -f ${cfg_install_dir}/taoskeeper.toml ] && + ${csudo}ln -s ${cfg_install_dir}/taoskeeper.toml ${cfg_dir} } function install_config() { @@ -655,6 +662,15 @@ function install_taosadapter_service() { fi } +function install_taoskeeper_service() { + if ((${service_mod}==0)); then + [ -f ${script_dir}/../cfg/taoskeeper.service ] &&\ + ${csudo}cp ${script_dir}/../cfg/taoskeeper.service \ + ${service_config_dir}/ || : + ${csudo}systemctl daemon-reload + fi +} + function install_service() { log_print "start install service" if [ "$osType" != "Darwin" ]; then @@ -732,6 +748,7 @@ function install_TDengine() { install_taosadapter_config install_taoskeeper_config install_taosadapter_service + install_taoskeeper_service install_service install_app diff --git a/packaging/tools/preun.sh b/packaging/tools/preun.sh index 0e96c71d5d..68f6b53c45 100755 --- a/packaging/tools/preun.sh +++ b/packaging/tools/preun.sh @@ -17,7 +17,7 @@ cfg_link_dir="/usr/local/taos/cfg" service_config_dir="/etc/systemd/system" taos_service_name="taosd" - +taoskeeper_service_name="taoskeeper" csudo="" if command -v sudo > /dev/null; then csudo="sudo " @@ -57,6 +57,13 @@ function kill_taosd() { fi } +function kill_taoskeeper() { + pid=$(ps -ef | grep "taoskeeper" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + function clean_service_on_systemd() { taosadapter_service_config="${service_config_dir}/taosadapter.service" if systemctl is-active --quiet taosadapter; then @@ -76,6 +83,12 @@ function clean_service_on_systemd() { [ -f ${taosadapter_service_config} ] && ${csudo}rm -f ${taosadapter_service_config} + taoskeeper_service_config="${service_config_dir}/${taoskeeper_service_name}.service" + if systemctl is-active --quiet ${taoskeeper_service_name}; then + echo "TDengine taoskeeper is running, stopping it..." + ${csudo}systemctl stop ${taoskeeper_service_name} &> /dev/null || echo &> /dev/null + fi + [ -f ${taoskeeper_service_config} ] && ${csudo}rm -f ${taoskeeper_service_config} } function clean_service_on_sysvinit() { @@ -111,6 +124,7 @@ function clean_service() { # must manual stop taosd kill_taosadapter kill_taosd + kill_taoskeeper fi } @@ -124,6 +138,7 @@ ${csudo}rm -f ${bin_link_dir}/taosadapter || : ${csudo}rm -f ${bin_link_dir}/taosBenchmark || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/set_core || : +${csudo}rm -f ${bin_link_dir}/taoskeeper || : ${csudo}rm -f ${cfg_link_dir}/*.new || : ${csudo}rm -f ${inc_link_dir}/taos.h || : ${csudo}rm -f ${inc_link_dir}/taosdef.h || : From 2a45ef88b00db98d307bf7357c15ef559aef7b50 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 00:40:33 +0800 Subject: [PATCH 064/145] fix(query): fix memory leak. --- source/client/src/clientTmq.c | 6 +----- source/dnode/vnode/src/tsdb/tsdbRead.c | 14 +++++++------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 16a4f55840..9292be83e9 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1664,11 +1664,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p return handleErrorBeforePoll(pVg, pTmq); } - sendInfo->msgInfo = (SDataBuf){ - .pData = msg, - .len = msgSize, - .handle = NULL, - }; + sendInfo->msgInfo = (SDataBuf){ .pData = msg, .len = msgSize, .handle = NULL }; sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 9922312bef..ad28b5c9e6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -789,12 +789,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); - if (pReader->status.pLDataIter == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto _end; - } - ASSERT (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); pReader->status.pPrimaryTsCol = taosArrayGet(pResBlock->pDataBlock, pSup->slotId[0]); @@ -4388,6 +4382,12 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL goto _err; } + pReader->status.pLDataIter = taosMemoryCalloc(pVnode->config.sttTrigger, sizeof(SLDataIter)); + if (pReader->status.pLDataIter == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pReader->suspended = true; if (countOnly) { @@ -4487,7 +4487,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbUninitReaderLock(pReader); taosMemoryFreeClear(pReader->status.pLDataIter); - taosMemoryFree(pReader->status.uidList.tableUidList); + taosMemoryFreeClear(pReader->status.uidList.tableUidList); SIOCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; From 8e2edd6ebf1b6613b6d95c0db5cc9373a39328fd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 10:00:00 +0800 Subject: [PATCH 065/145] fix(query): fix null ptr access. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ad28b5c9e6..e643e8f1a4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -790,7 +790,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } ASSERT (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); - pReader->status.pPrimaryTsCol = taosArrayGet(pResBlock->pDataBlock, pSup->slotId[0]); + pReader->status.pPrimaryTsCol = taosArrayGet(pReader->pResBlock->pDataBlock, pSup->slotId[0]); tsdbInitReaderLock(pReader); From 59725d1d87b6077497427024b5ded4ce16d7f1f6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 12:02:43 +0800 Subject: [PATCH 066/145] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 14 +-- source/dnode/vnode/src/tsdb/tsdbRead.c | 70 ++++------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 154 +------------------------ 3 files changed, 33 insertions(+), 205 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 8ce40ac293..86cd4f6a6b 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -123,15 +123,13 @@ int32_t tsdbRowCmprFn(const void *p1, const void *p2); int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema); void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); -// SRowMerger -int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); -int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tsdbRowMergerClear(SRowMerger *pMerger); -int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); -int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema); -void tsdbRowMergerClear_rv(SRowMerger* pMerger); -void tsdbRowMergerCleanup_rv(SRowMerger* pMerger); +// SRowMerger +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pSchema); +int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); +void tsdbRowMergerClear(SRowMerger *pMerger); +void tsdbRowMergerCleanup(SRowMerger *pMerger); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e643e8f1a4..79dc9543ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1464,7 +1464,6 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte pBlockIter->pTableMap = pReader->status.pTableMap; // access data blocks according to the offset of each block in asc/desc order. -// int32_t numOfTables = (int32_t)tSimpleHashGetSize(pReader->status.pTableMap); int32_t numOfTables = taosArrayGetSize(pTableList); int64_t st = taosGetTimestampUs(); @@ -1474,21 +1473,10 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int32_t cnt = 0; -// void* ptr = NULL; -// int32_t iter = 0; -// while (1) { -// ptr = tSimpleHashIterate(pReader->status.pTableMap, ptr, &iter); -// if (ptr == NULL) { -// break; -// } - - for(int32_t i = 0; i < numOfTables; ++i) { + for (int32_t i = 0; i < numOfTables; ++i) { STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, i); ASSERT(pTableScanInfo->pBlockList != NULL && taosArrayGetSize(pTableScanInfo->pBlockList) > 0); -// if (pTableScanInfo->pBlockList == NULL || taosArrayGetSize(pTableScanInfo->pBlockList) == 0) { -// continue; -// } size_t num = taosArrayGetSize(pTableScanInfo->pBlockList); sup.numOfBlocksPerTable[sup.numOfTables] = num; @@ -1876,8 +1864,6 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc pScanInfo->lastKeyInStt = key; if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->lastBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { - // the qualifed ts may equal to k.ts, only a greater version one. - // here we need to fallback one step. return true; } } @@ -1939,7 +1925,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* return NULL; } - code = tsdbRowMergerInit_rv(&pReader->status.merger, pReader->pSchema); + code = tsdbRowMergerInit(&pReader->status.merger, pReader->pSchema); if (code != 0) { terrno = code; return NULL; @@ -2021,7 +2007,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; // todo check if pReader->pSchema is null or not - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2031,15 +2017,15 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW* fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMergerAdd(&pReader->status.merger, fRow1, NULL); + tsdbRowMergerAdd(pMerger, fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } } - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, &pReader->status.merger, &pReader->verRange, pReader->idStr); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLast, pMerger, &pReader->verRange, pReader->idStr); } if (minKey == k.ts) { @@ -2051,7 +2037,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(pMerger, pRow, pSchema); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2069,7 +2055,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return terrno; } - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, pRow, pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2086,7 +2072,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(pMerger, fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2099,7 +2085,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(pMerger, &fRow, NULL); } else { init = true; - int32_t code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2116,7 +2102,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear_rv(pMerger); + tsdbRowMergerClear(pMerger); return code; } @@ -2149,7 +2135,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2166,14 +2152,14 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear_rv(pMerger); + tsdbRowMergerClear(pMerger); if (code != TSDB_CODE_SUCCESS) { return code; } } } else { // not merge block data - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2193,7 +2179,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear_rv(pMerger); + tsdbRowMergerClear(pMerger); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2223,7 +2209,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { SRow* pTSRow = NULL; - SRowMerger* pMerger = &pReader->status.merger; + SRowMerger* pMerger = pMerger; int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { @@ -2245,7 +2231,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear_rv(pMerger); + tsdbRowMergerClear(pMerger); return code; } else { return TSDB_CODE_SUCCESS; @@ -2454,7 +2440,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear_rv(pMerger); + tsdbRowMergerClear(pMerger); return code; } @@ -2627,7 +2613,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear_rv(&pReader->status.merger); + tsdbRowMergerClear(&pReader->status.merger); return code; } } @@ -3913,7 +3899,7 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, } pResRow->type = TSDBROW_ROW_FMT; - tsdbRowMergerClear_rv(&pReader->status.merger); + tsdbRowMergerClear(&pReader->status.merger); *freeTSRow = true; return TSDB_CODE_SUCCESS; @@ -3972,7 +3958,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } int32_t code = tsdbRowMergerGetRow(pMerger, pTSRow); - tsdbRowMergerClear_rv(pMerger); + tsdbRowMergerClear(pMerger); return code; } @@ -4097,12 +4083,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t code = TSDB_CODE_SUCCESS; SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; -// ASSERT (pReader->suppInfo.colId[i] == PRIMARYKEY_TIMESTAMP_COL_ID);// { -// SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, pSupInfo->slotId[i]); -// ((int64_t*)pColData->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; ((int64_t*)pReader->status.pPrimaryTsCol->pData)[outputRowIndex] = pBlockData->aTSKEY[rowIndex]; - i += 1; -// } + i += 1; SColVal cv = {0}; int32_t numOfInputCols = pBlockData->nColData; @@ -4355,7 +4337,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL } if (pReader->pSchema != NULL) { - tsdbRowMergerInit_rv(&pReader->status.merger, pReader->pSchema); + tsdbRowMergerInit(&pReader->status.merger, pReader->pSchema); } pReader->pSchemaMap = tSimpleHashInit(8, taosFastHash); @@ -4515,7 +4497,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFree(pReader->idStr); - tsdbRowMergerCleanup_rv(&pReader->status.merger); + tsdbRowMergerCleanup(&pReader->status.merger); taosMemoryFree(pReader->pSchema); tSimpleHashCleanup(pReader->pSchemaMap); @@ -4562,7 +4544,6 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { } pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); - // pInfo->lastKey = ts; } } else { // resetDataBlockScanInfo excluding lastKey @@ -4585,7 +4566,6 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { } pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); - // pInfo->lastKey = ts; } pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; @@ -5134,8 +5114,6 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa const int32_t numOfBuckets = 20.0; - // find the start data block in file - // find the start data block in file tsdbAcquireReader(pReader); if (pReader->suspended) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 0e2fb4b6aa..d8b0fc7c45 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -637,78 +637,6 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { } // SRowMerger ====================================================== - -int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { - int32_t code = 0; - TSDBKEY key = TSDBROW_KEY(pRow); - SColVal *pColVal = &(SColVal){0}; - STColumn *pTColumn; - int32_t iCol, jCol = 0; - - if (NULL == pResTSchema) { - pResTSchema = pTSchema; - } - - pMerger->pTSchema = pResTSchema; - pMerger->version = key.version; - - pMerger->pArray = taosArrayInit(pResTSchema->numOfCols, sizeof(SColVal)); - if (pMerger->pArray == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // ts - pTColumn = &pTSchema->columns[jCol++]; - - ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); - if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - // other - for (iCol = 1; jCol < pTSchema->numOfCols && iCol < pResTSchema->numOfCols; ++iCol) { - pTColumn = &pResTSchema->columns[iCol]; - if (pTSchema->columns[jCol].colId < pTColumn->colId) { - ++jCol; - --iCol; - continue; - } else if (pTSchema->columns[jCol].colId > pTColumn->colId) { - taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); - continue; - } - - tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); - if ((!COL_VAL_IS_NONE(pColVal)) && (!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - uint8_t *pVal = pColVal->value.pData; - - pColVal->value.pData = NULL; - code = tRealloc(&pColVal->value.pData, pColVal->value.nData); - if (code) goto _exit; - - if (pColVal->value.nData) { - memcpy(pColVal->value.pData, pVal, pColVal->value.nData); - } - } - - if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - } - - for (; iCol < pResTSchema->numOfCols; ++iCol) { - pTColumn = &pResTSchema->columns[iCol]; - taosArrayPush(pMerger->pArray, &COL_VAL_NONE(pTColumn->colId, pTColumn->type)); - } - -_exit: - return code; -} - int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -836,7 +764,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) } } -int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) { +int32_t tsdbRowMergerInit(SRowMerger* pMerger, STSchema *pSchema) { pMerger->pTSchema = pSchema; pMerger->pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); if (pMerger->pArray == NULL) { @@ -846,7 +774,7 @@ int32_t tsdbRowMergerInit_rv(SRowMerger* pMerger, STSchema *pSchema) { } } -void tsdbRowMergerClear_rv(SRowMerger* pMerger) { +void tsdbRowMergerClear(SRowMerger* pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); if (IS_VAR_DATA_TYPE(pTColVal->type)) { @@ -857,7 +785,7 @@ void tsdbRowMergerClear_rv(SRowMerger* pMerger) { taosArrayClear(pMerger->pArray); } -void tsdbRowMergerCleanup_rv(SRowMerger* pMerger) { +void tsdbRowMergerCleanup(SRowMerger* pMerger) { int32_t numOfCols = taosArrayGetSize(pMerger->pArray); for (int32_t iCol = 1; iCol < numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -869,82 +797,6 @@ void tsdbRowMergerCleanup_rv(SRowMerger* pMerger) { taosArrayDestroy(pMerger->pArray); } -void tsdbRowMergerClear(SRowMerger *pMerger) { - for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { - SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); - if (IS_VAR_DATA_TYPE(pTColVal->type)) { - tFree(pTColVal->value.pData); - } - } - - taosArrayDestroy(pMerger->pArray); -} -/* -int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { - int32_t code = 0; - TSDBKEY key = TSDBROW_KEY(pRow); - SColVal *pColVal = &(SColVal){0}; - - ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); - - for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { - tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal); - - if (key.version > pMerger->version) { - if (!COL_VAL_IS_NONE(pColVal)) { - if (IS_VAR_DATA_TYPE(pColVal->type)) { - SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); - if (!COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pTColVal->value.pData, pColVal->value.nData); - if (code) goto _exit; - - pTColVal->value.nData = pColVal->value.nData; - if (pTColVal->value.nData) { - memcpy(pTColVal->value.pData, pColVal->value.pData, pTColVal->value.nData); - } - pTColVal->flag = 0; - } else { - tFree(pTColVal->value.pData); - pTColVal->value.pData = NULL; - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } - } else if (key.version < pMerger->version) { - SColVal *tColVal = (SColVal *)taosArrayGet(pMerger->pArray, iCol); - if (COL_VAL_IS_NONE(tColVal) && !COL_VAL_IS_NONE(pColVal)) { - if (IS_VAR_DATA_TYPE(pColVal->type)) { - if (!COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&tColVal->value.pData, pColVal->value.nData); - if (code) goto _exit; - - tColVal->value.nData = pColVal->value.nData; - if (tColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, tColVal->value.nData); - } - tColVal->flag = 0; - } else { - tFree(tColVal->value.pData); - tColVal->value.pData = NULL; - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } - } else { - ASSERT(0); - } - } - - pMerger->version = key.version; - -_exit: - return code; -} -*/ int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } From 3e80f57fc73c606324ea79f421252dc8c45be3a6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 12:09:05 +0800 Subject: [PATCH 067/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 17 ++++++++++++++--- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 -- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 79dc9543ba..e797c8df15 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -780,7 +780,6 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd } pSup->tsColAgg.colId = PRIMARYKEY_TIMESTAMP_COL_ID; - setColumnIdSlotList(pSup, pCond->colList, pCond->pSlotList, pCond->numOfCols); code = tBlockDataCreate(&pReader->status.fileBlockData); @@ -789,8 +788,21 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - ASSERT (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID); + + if (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr); + terrno = TSDB_CODE_INVALID_PARA; + goto _end; + } + pReader->status.pPrimaryTsCol = taosArrayGet(pReader->pResBlock->pDataBlock, pSup->slotId[0]); + int32_t type = pReader->status.pPrimaryTsCol->info.type; + if (type != TSDB_DATA_TYPE_TIMESTAMP) { + tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, + pReader->idStr); + terrno = TSDB_CODE_INVALID_PARA; + goto _end; + } tsdbInitReaderLock(pReader); @@ -2952,7 +2964,6 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr } if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { -// ASSERT(taosArrayGetSize(pTableList) > 0); break; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index d8b0fc7c45..d4ff92f7d2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1051,8 +1051,6 @@ int32_t tBlockDataCreate(SBlockData *pBlockData) { pBlockData->aTSKEY = NULL; pBlockData->nColData = 0; pBlockData->aColData = NULL; - -_exit: return code; } From 5db5205e31733ca2a991144575f4bcb885279054 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 26 Apr 2023 13:41:16 +0800 Subject: [PATCH 068/145] fix: schema/tag version update to int32 --- include/libs/catalog/catalog.h | 4 ++-- include/libs/qcom/query.h | 4 ++-- source/libs/executor/src/tsort.c | 4 ++-- source/libs/qcom/src/querymsg.c | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 429e7ffa73..d7084cfac4 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -124,8 +124,8 @@ typedef struct SSTableVersion { char stbName[TSDB_TABLE_NAME_LEN]; uint64_t dbId; uint64_t suid; - int16_t sversion; - int16_t tversion; + int32_t sversion; + int32_t tversion; int32_t smaVer; } SSTableVersion; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index cfc6ef2025..3841210076 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -116,8 +116,8 @@ typedef struct STableMeta { // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta // info - int16_t sversion; - int16_t tversion; + int32_t sversion; + int32_t tversion; STableComInfo tableInfo; SSchema schema[]; } STableMeta; diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6c8e581b3f..d973b5bbf3 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -155,7 +155,7 @@ void tsortDestroySortHandle(SSortHandle* pSortHandle) { int64_t fetchUs = 0, fetchNum = 0; tsortClearOrderdSource(pSortHandle->pOrderedSource, &fetchUs, &fetchNum); - qError("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr); + qDebug("all source fetch time: %" PRId64 "us num:%" PRId64 " %s", fetchUs, fetchNum, pSortHandle->idStr); taosArrayDestroy(pSortHandle->pOrderedSource); taosMemoryFreeClear(pSortHandle); @@ -316,7 +316,7 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 } int64_t et = taosGetTimestampUs(); - qError("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); + qDebug("init for merge sort completed, elapsed time:%.2f ms, %s", (et - st) / 1000.0, pHandle->idStr); } return code; diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index b62a3e4932..01b136d5e0 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -407,7 +407,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta * pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } - qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d" PRIx64 + qDebug("table %s uid %" PRIx64 " meta returned, type %d vgId:%d db %s stb %s suid %" PRIx64 " sver %d tver %d" " tagNum %d colNum %d precision %d rowSize %d", msg->tbName, pTableMeta->uid, pTableMeta->tableType, pTableMeta->vgId, msg->dbFName, msg->stbName, pTableMeta->suid, pTableMeta->sversion, pTableMeta->tversion, pTableMeta->tableInfo.numOfTags, From 8be78ddee2c15faf979f1dd4d4980f743ea483ad Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 26 Apr 2023 14:23:25 +0800 Subject: [PATCH 069/145] fix:[TD-23881] check the max row's length in schemaless --- source/client/src/clientSml.c | 45 ++++++++++++++++++++++++++++++- source/client/src/clientSmlJson.c | 2 +- utils/test/c/sml_test.c | 27 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index cac559b0c1..57458ff8f7 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -649,6 +649,17 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO field->bytes = getBytes(kv->type, kv->length); } } + + int32_t maxLen = isTag ? TSDB_MAX_TAGS_LEN : TSDB_MAX_BYTES_PER_ROW; + int32_t len = 0; + for (int j = 0; j < taosArrayGetSize(results); ++j) { + SField *field = taosArrayGet(results, j); + len += field->bytes; + } + if(len > maxLen){ + return TSDB_CODE_TSC_INVALID_VALUE; + } + return TSDB_CODE_SUCCESS; } @@ -781,11 +792,15 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = smlBuildFieldsList(info, NULL, NULL, sTableData->tags, pTags, 0, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag1 failed. %s", info->id, pName.tname); + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); goto end; } code = smlBuildFieldsList(info, NULL, NULL, sTableData->cols, pColumns, 0, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col1 failed. %s", info->id, pName.tname); + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); goto end; } code = smlSendMetaMsg(info, &pName, pColumns, pTags, NULL, SCHEMA_ACTION_CREATE_STABLE); @@ -837,6 +852,23 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { pTableMeta->tableInfo.numOfColumns, true); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList tag2 failed. %s", info->id, pName.tname); + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + goto end; + } + + if (taosArrayGetSize(pTags) + pTableMeta->tableInfo.numOfColumns > TSDB_MAX_COLUMNS) { + uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); + code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + goto end; + } + if (taosArrayGetSize(pTags) > TSDB_MAX_TAGS) { + uError("SML:0x%" PRIx64 " too many tags than 128", info->id); + code = TSDB_CODE_PAR_INVALID_TAGS_NUM; + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); goto end; } @@ -891,6 +923,16 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { pTableMeta->tableInfo.numOfColumns, false); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " smlBuildFieldsList col2 failed. %s", info->id, pName.tname); + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); + goto end; + } + + if (taosArrayGetSize(pColumns) + pTableMeta->tableInfo.numOfTags > TSDB_MAX_COLUMNS) { + uError("SML:0x%" PRIx64 " too many columns than 4096", info->id); + code = TSDB_CODE_PAR_TOO_MANY_COLUMNS; + taosArrayDestroy(pColumns); + taosArrayDestroy(pTags); goto end; } @@ -1514,7 +1556,8 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL do { code = smlModifyDBSchemas(info); - if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA) break; + if (code == 0 || code == TSDB_CODE_SML_INVALID_DATA || code == TSDB_CODE_PAR_TOO_MANY_COLUMNS + || code == TSDB_CODE_PAR_INVALID_TAGS_NUM) break; taosMsleep(100); uInfo("SML:0x%" PRIx64 " smlModifyDBSchemas retry code:%s, times:%d", info->id, tstrerror(code), retryNum); } while (retryNum++ < taosHashGetSize(info->superTables) * MAX_RETRY_TIMES); diff --git a/source/client/src/clientSmlJson.c b/source/client/src/clientSmlJson.c index 9fd98e33b7..b0ae316031 100644 --- a/source/client/src/clientSmlJson.c +++ b/source/client/src/clientSmlJson.c @@ -575,7 +575,7 @@ static int32_t smlConvertJSONString(SSmlKv *pVal, char *typeStr, cJSON *value) { uError("OTD:invalid type(%s) for JSON String", typeStr); return TSDB_CODE_TSC_INVALID_JSON_TYPE; } - pVal->length = (int16_t)strlen(value->valuestring); + pVal->length = strlen(value->valuestring); if (pVal->type == TSDB_DATA_TYPE_BINARY && pVal->length > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { return TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN; diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 21964403ea..76dc66169b 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1132,6 +1132,31 @@ int sml_td22900_Test() { return code; } +int sml_td23881_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = + taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_23881 PRECISION 'ns'"); + taos_free_result(pRes); + + const char *sql[] = { + "lujixfvqor,t0=t c0=f,c1=\"tybxdylsdctesxshtzygycwgicbegupwlocvpijzdybrklsjwtuzclrsqwfoglqzvwywspekqvybdcuzcvsehisgjycmknffusjnptxmwlfbqsusxvgawcrhivmkscabawtjjyszyriojydlxxostnyisxnrtbtwrnficxxmcvgjgxoxccoqtiwowielknelifchfgecighcczyoqszpwvvqyvuzjuyuddcqtuzaifihymwttlzzlcxaobwgfyqtocekdcuolqwylsxgznorxzieznrxrrvmvtvkcfksahsnklyavsytmmmgjzarqkkzbdiocrupuycahrwdhaqzsgbfqxdwyisispodrzkhwgdodimxjihyrhprinfppknqezmgkphxmfunqshcudxrmyyrntkkkmwkdomcgvavdjekcwjkqcoimcdlzonoxwnfyeduqbombbzwexgbgkurbcbmeyxvrmmedsxyrhnqvbgimhamcrxsqkelnvrogarhvjfisinzkiurxycsaxgouaelgwjpuunftbhblrtnzkwmqfkbfnbawiqpjpacdadatotqshxpbvaoioxasbivpevgphgnurxsstxivpgcdxbkaipbnsejqpahsoeodxxazjozchjygksssebnwsjacdbkdpvcblagewuviwfnyblcrililpwnartqcqsdhwtmmgrbxswlsaskpaafosiptvhlgcgmvwsqtkdphqscwwxilvdsiwptzhnzgyfoejsahswmndszkrlatitxskfywgpxafsspoqfzzwogarceudyokprfcoaphqfkfslwmlgpyjuivietnhkofoibjwhvdbwbcckutqbejoovwlsyuflcsjznwnovjjchhgczwjesczcpfwimhnabyqlzihocraosqiaenutvvdynartocvxhjszdtvyqdnejbufpbzyyrskhdupjojkoheccpfbaoldyvgeayrjvkudbiqkieazsmpkfqunlzxasjnnqczkithgcolctdxoefuwgwdbusiqunwcuttzbnwzypwaahawexjpysznwnjyrkyigirmqeszzlgkwzxkrjkgdrprxzzwtzylwsxycgkqwsuxdvllkvxdpobccuxnnmdzqjmvlcyxoljgsztgmytrvgqeewmcpchmpexnobhakpykkftrzxqllgvhpdfyyucpshmosbocudxbttciqzzechzvtjigxjlzbfblfeztamfxmdbxvrhunmpfiavoryoitzsfeemzxpadgodsoydhalzhnghicpniorhcptwzfryucpefokezviorspzkfenyrbdbfkdnixkfijytqrxtvaivfqyxqikladjhxnjndammqvizckinppmwslyyhwkfyyjrjjnpefhesbdkpgjehochlvhskgacmbunndjijonfndvpcyxpuuvwjyfmaejnmakvrfpvmjusomnfzooplswmlbhrjoardxxbaemwgtyufyqpgzsfrxkstoohkshspnbgqgsvmlwbsivfpfwzwijthmvothigzrvhkqlmkwzwvpgedveqiedgoshddorrndjykqowfrohdulxvktatylnuqfqdidodrfxfslemggzmwfqapmgnndqvagtxffetpgpxijzhdpletiwurdqowjmntuvikvkzphvaqdaqznlhtmvunfmlpetysqcpzyjpmjjgnbxollkmwiuhpyxuyejctqdelupexjqpvnowudzpftjpggdtdrxkqgxpzxugqzbgsfkywfvbqzzpcokkdzslsmiwvwbfaijrfldicyhmxdxsuycnbqzejzkvlztysdohdcxiekckmkmfwlnpxlyiamcqrkqrdomoryqekzvvymppjtwubqdatfplwzhvlfhyqiinzqxhyhnymccbogggxyrezckjfqjkhnlopdkmqilfpjshuszdyudyhlgqlngaageodtpyzhdgmasrvgzogiuyhgsjrfzfvkwfpdbadqmugigiepzrqgzibmoumfmrxuuvbaaihunaypgdeaswzdbaqoubehpmfqbklyodcvbgpppputmldxnjgcbpcjjknvtmxfstrgxbylgkucbuqjtfswlgjgtdgrzcgjjonziolmceutclufnesxchwoecperlpwdwpicmspwavxsjunoipxxdfbyipsiojmpdvqygmqsysrsqvhjdunsimkvkvvconhtnnnqhhbpwndjrtapxqbchkrinehqtorytxlhbjtjqkitbfkdjhnthevuxnlokkqnnaztaarvfmsfqcepxgacwomwvckhsbejbioxwmojnooyukswstlwqkasiohatkdmfozncjdawzgaroybvvlcsjiiojxuedjzgwouookoxwfnksvoislpzllcitwwpiafeqcayktzurnoztdgjuwhjljszreoljjowttsteitrwmujqdaidhlcxfzdefkvbkmsmuyhozphcyuxwmpyipxfxdpdfojaksqhefqwhpqcplgmomuhezvcwcrdaksurrbqirlvgjjezmmooflmdepohrijqfwbiqxpepnexbpbdwvvzqgwzquvocsalethkqetdgnexughqqnbnytkmiwrainnqlyuwuljkffsjaommyftidtyamffyqedxfvacnvrrckqaxlwyvjvvaztjqxmtuiiltrrlntefluclbrkytjstoeznorpomhwtluojbnzqrzksaedosuslwdfftvcilagbrrxweldblmeclqaskclradfldhyufwqbiitwlmtccaxmswhjpzmfccqcxzyoneiudmxndivuteqkapjpnqsnxksgkorxwkjwkrrbkplrmgqfcazkjiprhfmxuxshdkldkfmefctapwpynozskxpxrnsluwuzdvhprgfpjfddnzncdtawkjbnktjxogaxhpcymxugrdvlkrfilgsvcglklihoiziarzuawuggrqpuhhnpjuijllckkruslupluzhggccebqqxktgfxjrnwamwopzqqsuvjlzyqnmudfqfxvacuoiwwhmawooaognltuowfmbslbtiyxoddglgszlpdlkwabfbiwdvxkmyvnktpiscceiniuvohafyudpxpqfowbwcuglyzbrceljxdszzozpyrmuoewmanlbfzibnmoyrtowsgqdglikzvvqrzjjoulopqnfvhvhdqynamxlcnbscjvkogjwqigpsxkkpwzejwnwkxtqjpibfrropketlkxolxhpdsjkvcduxcfregcogjimqshjhqamtcwefdzgkelhojnfmpcnafvhdpkgsmzmatuzivrsaowkaohdeeybbkzljnxcazvbobginldgseyquzwqyxjccvagnvofatcfhpfwmrdvjfqdovsxhryrpksxcnbypntvucrhovaupbibvqozvjfmhyrayvqizncbusqfhuhxaffshymjfubrohpetnretdvshlxmurqvdtymawuqyxbhqtfkebypnutldzelwyhbexsvznlyzeswxdymisetpybygvsegmxiribznlypjrnrlheofgavvlfxdaeyepwelpkhascutjaskrgkzddwhyhexegguqwpyxtuusystfsthwhaabheqxrvqxoshuairbfxhxyzoxslwydstlebiansxnjlkcdondfnillshxtagbvufpnlzhlfpnlxbrtuuesiouktwfjpnkzwypnperyakqrmbuvjdzfmayjsnekpiozozkhqwusqucqeqduhnnezagclocdfcisfdobjafrtdusmcdwtzhrdylbdvaudyolxjhnnxbqzyzlvesifftcczwyvxvpxjgnrddwozhazbhgnabssztudplvhhjxshnhpryiekxurnegvklesafkvuifzubljkhsrfufabtrykfdwexjiqpwkcvxbfhcvozvxzjubvqzgaxjfrtgnbkectdbbozqqkfymwcrbxfelargrnqggseceslnmeoyskruxivvpslxyzccvbxejayvtrhpykjyleoevwpzdetyjncwntfjyqddyqgbrvdtgoyngpsdvtiqwcyugvvjvltuhuwrxthcnrmjemvfluccerjylureacftsfsmbexudajyyehohdqteffywxstkwtkodyvbpxddjbmrwiccaxisidmvmgiligukllqokyhateqcqkwsvvzyeejaqwsmyxcsaavcvhshdoidcpuioqeqwjxzpceqjypmlnosupcbglvddxwsvmnzjnwvktpdujotjftekcnxiwyvmucqskubkgcmcpfsixynqdgxgjrjdxdhrrgluqjqunbbepizkchmqmrlioonlhxgluldiijonwryoffyfxbwrdiretxeedlaeznyfcplfxucxwaygjgjzpvtbdpljsuqdqlcoohgqfqtxrbkqkdjezuczuvuwjdhabuvpzlctjwzlqkbrmglffkdxxhhfpruhsbnxtfwnlulhenjbzddnipxkvewdakueehpkllugcmkoyzvgryvbddlbmudmhaznghbpzthkzvowqtubhkvihdsqayytztpqytdcjarkgnflapjqcpeoffakggnupyebaqxvpwmqwwincpmnjtyprjhyjhwxcyulgsmcxuihpjfltultigzifvsqdtgagwkyniqycftqzebioccfragqulcqlpzwdcmygehpqzbxvlbunohlfnjfpbwxwrqglyldhzsxmndaqqctxrzugomxkyivwlrrhkkfhmkijndrqdqppdtnvsudkfnbetzdelkvezaedxelrqekktarjrwxdxvpjwksavwbkflmrwdsanigmgdillfrxhkfmbivlvetiqcfmkqqqrdhsyvtvsiijkjbadizacjwsifcbutsikrqhhyctanumwdxflvcpwrdccufmufmlzomxtsvhvawoypxijkdgefwdutiipyoudnevhqiwvtmmjchuvrjtmbdbscbulfijmhpeilyfmardjiprckybweyccrenqkvvrniimwtsdleorfrcrgfbczqsxckciprbobdclcnpeibzqcluvyutijtwzqlrwjxspzqleeatzmqtsdqnuqjtqhacbbvkwtbteudvtmqgjagiqkjmdhuvfxwkucgpwkqxpsewgpvfqgubkiqgfcpvwlhpmwtjahqdhjhmevgdoikopiqtrfbpdmbeyzxdzubnwdgwqdtxcvdqijmpuidwplifmemhzfnegwkjcoszyewvwwqzjldgsdlkecasrthnmpfmrzcvgqgiyvgbnjqrvcwpdfrtkozwncxbnfgnjjuzgbqpugsefpbdxltadeuyrwcejklyjggsifvnetzwvdqbhlakbtemunafezlkgzzeziuxziwyjxciicokcsxxzefiosdfxmvcyvvsjtnqedncbpngkqwyhygdwcmjdnrrtpazdydudewobwyzfvejbutmktcnrlirjjmzegnraradtinrrbusbzisrfrostexwoeusqrxbuhhdycneorvrwxdixhjktnhssqicujibyfpdnluulpqmxranemqtmhclmcaznagekocqttjalztonnvqdxxajzblpizdlphcafbubavzqlhuryumvxsjjnubsffwduxddypvgvtdmqozgrhwvrwmdpjvdbexqugrrpxpvayqbccxgbvhqejmzuuobtkwopledyuqxlmhhawvppkjbyrrtjxitoytspwqomweunqplvagndtgvahwxkfxbeusydbbfurgtwtgjnryljupomgksprjqravlvprcnhoaagpcwapsfoucokrlpmcjtazxqspgyzbcaearbkptcyhkbecpmmjkzyxwdixbebqkgurdqgybvsmzsenxgljpxcfjmvrrrxbnvlsgdwvjmsjvhttdnrgowrcgggzldxirxluxrjlbueqlqrnmvajosdwyjiwwbmqteiwmifrsrlfpvxuoxykpukqfsgxdxunnbbiuoivyczqaylyrltkbsucrwxoosghizxztiqlbkywasxkyqhoesuwdvwzwjfnznfrkbjmlntuirfboqcpkhqfcfjdkoroydcnpwtlymvbwvdwlqqazprpjpwruvtmaoazyeddxqobxvqimqapjswmviuotvkvalmgjdtrnupiloxihdgcbujrqqvcljsksbxanffwhcxtvzxpwvhoctthkidvolnvfrpmolngrsxbbqoqizmjcvzveuarlbrcgxwwqrdefefjhjqmmgwdkipvsriiiubfhiffcpkdlbondnzmelvlhwlqlpbhrmugkeaonhfrvcimninyzauigftcesiihtumuxwuktbbulqqjnkxgkdwbslfdafwdentobwafxyhyhbasmiibaxjoyavplbnqciuvwpzgckfrwuvhbgwfcwzxzqcmtytrfelbvgfdnxjsuzeeiuqfkmrfmudxktyvupucnlfvnmkxscslxhzbpecbaiqjnbydrfwrgptegptfzkayhltrisahwqtudwhwjotnhxsvkgviuciagqvlllumknjtjcarlfccaslbrqpsoihlupkphgpgwgzodtksgrgexcaybrltrlqkzadpoinhkmzoapdxflyexwnsgknmxuupnhijxltmwdzjwpsspotmkfothqebgmfiafxdytvwuwptyrjepyjihxxnfdnuyxrcdzppoxvjjiujlbjtjuusdtejkpuiompzkhrqvskcjnhehskjybmqaryryzzddjgvwbwvozxaxurfoxnpsfcmyemeisgtfrhrbvxdqdmkklluemluklxhfmsxspzznlurzcuswdfsdmgssadjlemornqupbwbukizzvlufwxogmgluwdksnevsvpqzutevsyidudtphvphqtdaqsemnpcpicopkxmgazmdqfnxjxbuunohrvmknumchplkvvlbqgzpquwlmlcxiuevgnepdfvklwplahzgsscbghfjagcrqjlpjobiqdzhqswtiojsrhbdjmwljowfzebncvdxnsvqgkzrygetamrwbupghasymtiwkmlndvhkckazssyjlymkjotwtbdfbwecfrsilltrpkaeiszklwqcbkxepmenazonihsqdtvxzeojlbtlprmsfzxnxzdguhygtyiyqnhfkussqxbkrvstdkxnovtmdvcjmaugeizqknamjnpuwhalxqclygvdnecomtlzzymlbctwfufdvkqzedoeeuabdqnumkmfqgvdqtexmpeyiikqycqxtanlnqvbeunaowyqgjmdafxsopblxmimafeyvzmxmeqhhngoxygmhakwisvlnhcqzurxlitwplasuqzllrplahahvwwkqfxktigjanzhmfdtwsdydwbvbwwimmxszzwlroucukuzannqtrrgqztkgepipxilxxjcrwfdrixjdmadhqyhukeexiwjmqswhgegjvawsnntgtjncspitspgvipxvxsqvofjfgkzkbhxvnqthikgdsqyzswprfyxsxiozwjfgnziurlclpoucwpxoqgaqipygykyeltpkfvotvnustvzlalcgbqsoojzjfyomfwmepdwnfdtpyhyjfzqgjokyujudzkfrzsnzpuerxlffzdzadgcslpivtaunevdzbynyixhhzanumhopvlsvutnuytqbsyaqwfaovpqluvcqowmjlrmollasvmorzqptldwpavcdmvkmpnmkgljoiukaaqndhalataljocaowlwreswkcjsiujeoewojbigpsozmfenvzxmissyerxiglumbkgnnzwzoxkbzwgsxvhzlkqsfkeqfwnajkkymyunsoyukwuxfwicmzoadtvhesvqmzxwkuceshbkrljhqipjlsejdjycejkbobkkwggbfvmatdkqllkzkzbeeqpxctdvqjfejpnflyyqgbkveistvtpsyosexusayaawcblcwlfcuqjhjxvtmhzbccdjtorivyxvbzlrpzpoljnzowtvyevkbwvdwljynbjxahqxzhxhmaklckqoemeudjidvkclsaqxhavadwdkitdbuqjneaqvyzcyqilixeglczhobrlknjyyquxxnlxxiohmtrqkcgenwxxgglumjzyygrozfvvxsuautmunkraggxhihaafxghyyexwfwbqmqzynzvmygeakzmqnegvwcwrkjplpqfawkazkykmapleonxqukjqjkvpopfrujizaoawdxlcqhyzkyjwucbgsflvvnvfroybnabigpipyghsiyotdfeqrusgghzswuwvntlzqlgzsmqllevdcysuelxbyiyfofxyclmafrupqszsshakoskgtcsoecvaoduezmqczqxwjyaildgxuomhrzfbqqxfnkztjrvaajkcgxmwqvmrfrzojazdkjtsmulrzgrspwdtgvqkhxddkkxhkcasxlbejnsbdesakmrvhyaziigrfunuwjcrtwllvxqosgubtheowoukcejnkeywktloeilohtposjbocsvntqlkwislefrszsxnzjvtvlabigcifkpgsinbzofkapdzjgobytxlxpxnqtkzlqywetksrjfzsnxhshudmwxztaksskeqxqzakqebpdxhcjfrqjwmwlptgvvbhubkkrpvblzalfhvjaitreulgolcfbtiftwraluhhdceietxdhcicvedqcsbnpnsrpskfrrembdbhfxwumwrwiuvqelqfkypdihofvkgivahaaeyqljivvsykhcqzwyzpdtrwiimjiudnmqgbcgdvcqnyzmfyrktjeosjojhykwwqgvbawnobchrglwujfsddhztyhrxfxpaudksdhiagxebgsvzhkazcprmrdoxeryeudnufozhnlskvmbdwxqxlpmpzwujkinxhuyabvvrpdqafwpliohxydkqykejmqkihjmtykgptjwwmraiczztxtvatzarvlgpczgrbuahywkcmlcusrqyhcpssdkmsecemqfejiwoftvtchqllrdrztfjnmfitbbdoptpbjkrquqsusdfkhnywrymtlymlshqqwnnmyvmquoskahqopzrkioakiqsyhejutaqcrmqjcxgbqciajlqoeffuwvxmvhaxfuwoumxhgnccvzxbhrxembmohjsqlxzzhrynjnkobimzktxvxlqacglzoborokeazhvmkcgzdyqmaajdzhlfgobussodslwqfcpsmyhblkjszhyyhtdkxlsiloldpclulpbhiftyuxxhwbahusttnescewxsbtwjoclaudpunyenljxdguzygdwpnbfjfqoulfduyrtreinewgcsxfwrnohbfkqrmqloqyddprwedzqopudlzyurpzryaltuybttczggepmplazssuwshxasmleqivtgqhtioifmwqihnpcuxhdxaxwibybphxdfgbsawzwwawowtbeljudcyhrsdttgynrbcqvaojovpexxnevtdtsudukjbofusthzxjsnmjstenvnjxqektukpndxjfuojzmachfodeykybxbmdkuprqkkzuyvhhbwlvlwhbtunslevzaccdqxcibtwiuujwfvwowgiubudppeenmcdnhthnkmymzdbyianzgsilcidlefernnpdaottybwysyclmyrgfyjjbglifbfbzsrrqeyebtbdiwrqjjmmrcuzoxuafigderwgcinmivdfmgvrzwettesjpkaptfyogfhbntzwywycgvelrixpgaclbcwabghcldjflycvtaijslaharpcosojqyrkmzkrxtofigaolnhbsrwosntcngeuxbmrdodtnaouwnmcozrpjfbltqguikjzbzmjdwmginbmbpkbiglcxurpcdkmbhvwicxyonpznbvaxhfzqqkosdzdbkftywxxfwfcgzqibsqwrpzqdntrobcfsjxplgzceplxjugfvgfvyrasurwjyqzhzapydjtquusohlpsmwektnxyvmgzcnymwiuqzorxvlvjkuxwqwxqbhcujlzggzypymjqpsfxtruijklkaaviufnygwomnthsvgzjheqoloftemnoqhwxrhrheqdmgejbyszlieitydasqrsbknrtqhnnwpabflhvfluitruebsdgxipjimeqqegixngbiyepzgjokkaadamrpdtleoakijpccpsvpyhkwyubcpbjxratjstbwnzztbyenvudidekaefzftaefraotafbnujgcgihojrfwkbqkppzusxpqhzmbsjuyrmjxvcydwjiedddgeypqytrevbvwayymnqbjzyjvxxlojwkdmpgfwgrlqgkwbltuejvjpaqcowfumrqaxrrzlbharznnrbxdcyrtufmkoxkjkyqisdyxpxqcsaembumiaeofuxurtfqwhiomofwxlqbedlbmmcloqivmjthdnenggqipizewoqmjqyjcnellxdnlnwlaxsruifwpajjcqkwapktjyzlqxhymngnthawemhkrnjcnghbdaoynvdtxmjdvrbyhteqqjlxtupjbolrlfedyfkrzguompuuthhkmkjdfupmnuyrehaeylxhlgjwprexkybaflumitudfilawpmksysktnhxbdrvpdsdxklstgejkeibrjlzjfmaguoibjmrzwjkfbmpbzzpmgwiafrizxkyvqwtfodmkwcbryrubkwduiaavxzfibnjigllqudqvhoxjliiwveuwxfzbsqflxuncxxzezxxqkxbnzaudjckdixzwckkfdfwxzkwqjujwurodouzomdysynsbdwgzvuaqhpusqyqsbiriibtuoyakddpcswwbrascztkqwkxtvnpafmxdwzjqodebtxxayokgoshoxdyqxlxyrscseasvvtdhxfeztiriqapzhcvvvuzpqncippbwhlbxokpyxaixcuwbahemlpdnayzjafupsqzhdbekhqungusybissgzkhpjeaiocubqemhkngdukobnozozoaagbwdqrluhwsxbfkbxbxmyrqsgeqswhdaxiummendqttvvwpoflpuubgazqmxehbmmchwqnbdlskqzbbnfllqteukizmskrbfapgbdwfrktwhqxlrzmuklepritlyrfmoyuzcrsqoegbryexhhpqxlufehzafotnxhcemqyfqekuixozczikejsyeevgplfoopqlprnfpricfwdvaeojlhagvctecqirsiyppjqsaosmrwholuxpxezirvhddlrlgmlukyjvziwskxcypxuatmmaywjreemktplsijkzblagncodfwcymmkfohtteypcorifflnwqckcutohaljtdrthvzkfuwpoxjtlpsoxkmfwdiyzkkdbjuanbtdxpfefbjkqonpgrtfwhulkssbotqgvndxpjtvexwafnaiczgtwlrtxrlsntxjzcesplwafenddrouaezrbanevpdkewojsyxpwsnmtyfrnqxrnekeselocmrdyknvvsfilayqjryqkfuwgcvpjjuveunfmwqfnwdrbyyyrzrtovwjeimcvotwrlxpjnzcwsppkysormxdlrohypvgszexmvcslcmqkzoodlxuezndemosvxrktpvxzyuymbvqdekzzellkonvicekxeoffsqpwwggoktypqmrwdkrnxheoxpsfdhvczrhmihlekjdvkvdnikawfhxuconhfqyahpddryuimdyshipxemekvnozyrvieezytfecydfixiydrveaamcdqsofbicyrnfzrgqsjdcpgkejrvdqgttjqvkqmqiesemvuafyowkeqbgxtdqfhrirofaefbogdhnwacxxogaobeaoglrofiryzhxmndgywzpdrrgimhptsrpskznohtiiijdnitaczajewlrichkvkzijkjhzrurzaqbdedpdcftarwsqzodkizjzltojvsiqnuuzmwnircgpodazxnumryyeqzxrwobzpzeauxnpdmabkhkoxvqjspnqublgunntcpoqsvrncgkojnmzjrifcmncqxeuvkyxmevbgmmakxscrmhwrbxmsbksadqnzvasrikfvhgryofnxzqzzknenhwdrlmdykumnsufqreywpcsfoeuxdifyivguqrrzyjknqzlwzwcmfehfcfkkoqfyfholtnpglpwodmwfhtwyehkdguhygzkbqcnjaltsgqdsvlzfkytttyuuvduxbssglbhgxzqdgturhaxoeaeigbrmzcpdtrvoeivmvtkiqrjiskdkdpqlualqjgpjxwwgtuglikhhrxcoqwtkzrmwcarodawtszzjnlwykzulclkoqaexvyhlfhsxreqdzjvbvznhwdcncfjscamlqseieufjvfiwixivgshyrvjyqlslusujnjosbkqcyrkhwqgmgfvkzgeugrsvtiklkrvjqjwcyrjemomtgewdfbdmnundgsatqdbftjhvvcvndmtmnswajowwcxiuhmtojthqkrzuvfmvbxdbvrxjbwijfuxkepzhjpdajilscqjgfxoldotaronmpkhcczehntcushirzzbhygnhzaryvehviqczxkckjbwqxubimjfjkwmsuobpwhcgywqxbwlgsasvonjpxwmyyfgasvsomgjgiavenjcbwwhqhbhzplxzqpyguecjyicsjkpcwwfjylqnvtdqoikkjbppaljzsnoxxgfxzvdygutlxnavblxalobhsufoyusyeiggxplniqrzcbbmunyqbfudjmhsjuqfpucseihoixjrhokgnscoeuhrrecfwjrdojflmqnsosailywobgqxiatgdgehlqooxrospcmxscsnfoidijdzjfudprkgkdolzmyccpseewqznnmzuxxtuafuvgykzielxhfhoxmyoewfhqlwwhczwioankvdugnapxtxnohqpclcftahzbpmzfwmtpdzqwasyjjfmalihvbkptxthuqnivalmqnurkxsjepppepcprbeczbrasanjlohbyxbkmrjvwexqxwhpbmnkjathbaygihnzhgxuixdledopbsgzwjmeocwgunzrkcgavowuevsutbajoiiytozuxxzphnqkkbgmqjqdwrqaqxkiamujoiypzfhqgmflwjrwjlbnkvzuwzwbhltzcxhxkedcfallyiywiowvpwdvpgdauwiaqfefbgxyxcfhszdoohwzdoqfgdzhdjfwtcsejtmoqaxkhdeagyvccplrjqtjxahlixxwtgdpbkwovccrhpmtqejliklpldljipzzxuwjlokvzhbsaarsqlyazsxxrixugbnijxifccfmteizbfhdzqsabjebwpkxyfooockqthtodjjjxepgyicpdnrndopdzkoucytywunnjsweavxoxonyovnqneqqqpwyilsobiiuegvnmwhpgsmsjymcrgrnucsqsdgeuwjbaironzfbfjnboiqcuhoqyqkdtorctjdjfiqbsjbjdrhjwaynbhgoigoprxhcwdrrspcbjjykqeydvrtctdqcdgbklfmbjyhxtajpvmefrefmlcdzbnllscvoqjjhpfwgagbjyjxoznbajpcuzbuxaolbvtaqkefujjxbedgbyrgxnsybsgwkuyukjsdfyxdgvusklfitokuetjlsshvkpvvfzxtmstjiftlpglhdjinjfoluokejsmmegioundlngqhlehmxjvobmxpcqhgljmpnwmluutgaryhjyjzkcgodxympmftevxdusledjaczzsviwttygxaswfgqzkkdhdgsphteqcycvjvvukchshbgsumvfgzagjkqfckqmpzvykuhcbbxqhmkkznjmimmopjgitgsddmjaogwsoublwaqpyhxlanprmnufltgaaklurhbelomdpusskwahiaonzqnlkpftiiycbnsxhpajqxsbcesedzpcdeowotgjrxwzlqllhgnudbudxlpupvzajnqngwsbkkgnqjcfgdwnypawuyuqoatdaxveziizabyktndzkgojxqaunonhupgndhxpwayzrkvqwzxriaskzmidnukxeunpulkouhwulnsnauwmubddzyqhbqsbnlhfuvdihmacylzenwuwcrlqtummcjrkqbfgbezgejcvpgurdmdtdkzzvviffxzndmxmmvjnljbnzjrtybdlacgsyzcrhqovlovladilnntzawsyxhmwyzldklxwzninbxxjlkbhifwgzpjxlkfzqblhrgwmgronvbttikwicibcteexbwggsipkubtqbhxemtifqrnfdaewbrgywfcpqfdacwhcsshgyrvrkddvghbvwtuwacqwswyaqcrqzyvlnjnzkrilimqzcwmhllzlindulxmckdhtneslpnlodzqzfxlragjtbngazbgrkskclgjljxfbrlhtsrukiqtkxigmrkuikydmqlbcibnkqtgbenlivlflmbyectljucdmuvbozumpopmkflsplzkbwtwcsarhqixsbgsqkhangirikdilewqdumsxekthzjgalyrzqystkzzebvviqcxdfzfecupgrhcrpulqdvtwqzjinecsdwambidunlxisrhnhmqtlaojxuhrgspolfhggbspywcbyflaagtgiliowspwkcgycfpchukewdulmwsdwrwzetilsomqtckejnutxssjyporppnaexvarmnhgsvwiodtrnbgivzzbzfppjebldtxthwsutydfqirqouxmxocpgircdztvhauayykwbauwozxknpfbddyvxsjfutbkdsaovcjkdpkfeknfsspeuydaxdkkfbxohsnmeudggxcxolojxgcpjqmowkhxgwgnxwrhmqfguapakhlerfyoqhwwawawcpgbuilcmdobzmwgwbeqqquejetwvnsziarxdvouugmetjjmhfcvnaetyvjbzmwwpkddjsxeqbdczecrlnzazjjddlduhbrmfhpkelmkhyvmadrkgxcvwyveyyagfmvtrpwdfrmefxotdrduliitysextojhsirlsiagqjbbvcctkcdmkcbthcsnfuxyreeffixdlowerrgrxtovtrhugafbgruayddylerfxnqnjpijvcyxlrqkmhufhwqsqaagrziaoldkqcwdallejbkurbtwzwvzhfbtggfezjubvcjkfipsnoowfwvsnezakllfwwhwrgagpstsddaxkivbmzndgshouzgrzbpqwuaquqjyobdaqliyuufjwivdtdqzutofvwinukwcqhmrjlytvchgdahvdqvnauiykevdyilclhanlrlnfgbvqjifzqmftnxcitirzyvgjjsmyjbxtqwliqoprikkfexvuiaqmonzjcmpdhcnutfunsftbicgifdynnpmaaexxxixfrtszbneujarczqaglzhfaqfbjikhfcogfcduigyolvcrqusassrlsnebmyqlspqvkxbfkqftbwtrmosvafmahhxvdummjsbhcweotqgmdzkxsvczhiummtzyadnnmvpbzubazitewofuievjzpwkyqwxrhoevbyvuosbclquuxatofrwfqmadptmgjuovcrlhrhmyafwibmrermfqkscompeexmtlannjqazpsruhwbezgfxkbpxqghhhtoudcesesfujdcmjecjadzmxrnpbwywhbjrsvzkymybqrnxrokzyorkpsssemgjvalzdavgupfybiyqbftvwscwwqsynmvxmnoyinqihrtjqkihafybaszfztohtombfgaarblacoaelaopvhqyomgotovmbyixzpjdjocleizxkocdyivgfmerqahdjzsiousuxhnuojgqkbjdykcnpeosgpzvwwqkdtukpattwbwepxruqtcxrxuoojmtzwozldfabgbshkztljrrrrdkscgvqbtqpeewxrbplbsrowzlyezeuhgjqmcfozjyinygjifdfyjvorhltqrzvoemboybyimtwpddtloajtroimjpggoxhfszrkwihedutonowdgudhnzulrdsafdorgxcymdzztzwvvtyvfxsqyivnzhcamystgvmussyfkzwincofzfkpijmsekofeizotnqchvgllmvmsptqqrryodudxtns\",c2=\"gbktdxiyhtcyabdwusjotcpfcyjkhejmztqkqchngrygrhxhdqhqicpfgtjmforxsfnqhhiqicphpjyuqyeximildnqsledxeulvlgkizpmvyixjueiqkaeldfiovsycazzafrukyabxrcpdxwgglfazjwztrjwinwjnbcwzyxwidkxhjqudnbvemeoawptqwmljfraonjagujeindnxmluxspoudowoxmhiignoewukyokbjbraszjjensivupwzlcizqwstksjxaxadmxeowgyomgacadhvmageboakqvrgwkerzgeukpbnfzxsqezybseggdsxrnfnewfmfoxarffcjmqywcorhuroaqjydgdgrvfaewgmuxosbrufejhllqccmpxqwpmdkkcibttkbxjmkwjfblawheaeprhzcirqvhguxxqkhvqylapygexkgpwxhmteuzpkfnaughjirionsiwijfhpoucpoxhnkopqslwfgzlvirzhiuqkruwubckmnxumkbnrdnnukssafbcebbhcjdmbcmmlkeaitvjrvfksyapfoipytswfolzpmabsbeyuzoruthachwmsegydxhnmnkdidejmbkxbsvteleqjepaxqdyxuiqnkfrknuqgybmmvmkyrcakpxubgwyvrtzvsevidmabieqfymoynzjjyywfqghkcliyzasleomfjqrrlgcgqegawhphsafasdjwrkqneugafrefqhyadpfnqnojvfqpmcoxavjzmnffpcewqyfjlsfuvflemmrxoddfbqprpwvujdxjoebzgsymzvsbclynhasoyuaypsurlqtacoulhjfzbbarulvvrutuhrbmdjztqubooccojgunlwwdyprkluiepsfnhrsfvfozyljimkjcewubwhjomnurvolstadcllfnascxzocwaxnmknqifysapbvnxquahdywzzowpfxzrpgccawmctajzwhtaqrwkwijyiussyinntujacpaswozwqjdrskcelljbwcdrmnqxbxrpvnbikwjhyesounmzsxnhwnrgnvplcdijxoojfzstospkdisijkueaozwrgxyvbbaakycrnjzytmtalinmhnzgpcjxwauicyyxbiorkaarduvmoomvosnhekswdkzeqdxaiogucqgnumbceuedzcjruemmfbtprsuwovvrtukslmrvuzqmuzwwvyshegnjonabvjqgxccjyvxpuzzivdstpxbrjelfomtbfxlshfchhkaauwdivewrbpcebjfnnjattfzirfwkjhotjjmyretffruhevpdpvptbhinbemqucewwrpdgcecwyphkyclteijeejuetoqkevzotovydvjdlzazouwwsdbmbfemvwmczepdybryaaegxctwroegrslmazoyqwirgrtrukpdfvaccnntktwnbifuvyvqmkzxvpizgupihuxhuuzticuettmtwbyxplrrikzwhdrhrdsxqoithfjcsvattazdxbsfzalshaugxdblpphgihebsnjgxsgwjwzokvebrffcqaxujvodtrhvjgfchpdjliyilxzoxyykvcnzteopkgbaqfjblicmpnooinkkctnvyvtvhjdrvyzdlczfrhbkxkprvlvmysltaehphbddcdrsiirkbsaokuvuvupbkrbzmtowsirkkiqbtpftrnggkzztfcbvuufwktplnyisdhorxtbhfzoonansxktktejqulorjqvtuizehhnnptcrnqbrzrtwiqkfvsarsnjwsbaofxggmstebenchnhyhwysgbvljpsfmofrmfqyxzluodrxapxtzyrwjjvxgxwqbkgontxnvpnatlraevbyzegdoybnxaxowweehyszoetpalfztwfpngyqiyfybulllicmzdtekpgyekhqfgontemyfhlkkqugjgeznydroqwfdxkimsakuicxoxyzuksqaqmajivttvrpgarsxqmjntvkozuyfcftjeysrjvbpzhtyazeflayjbgogpajzmptodqlkxubwdlpjsnkrwazvcdtifahvnglztfewnkgzrruhyjgbbqtdelpaxdpjjfmywegkzramunnngcdbyrzgruwffuxmtdwffcdymawvmdisttmdkhknnsfceqpzfyuaxltnggajcklephspnhxakwcsyzzzvwftuwkcqrijbbqvzcyefettcrlqfqacicoprvuvxeqecbgprfuqyincldclygvjfzbwsqdrqncyfzzfnunevnksiqeplmqbsibhmmgbrkpzfdimuofytoscprgokemdczkascacsyygxuxgqnpiwvrzzqreidzlvxvhotzdywfrrldxlumvzgvvnjbmadzflvdnbzsgmrkppyduzvlkvnefybimiadjdumfokwvvnmqyjvdejuhnpngsaggoqdwsrlmwpppvemuxxpdcfrriuwjxmfzryemksacnnbmuyacmnwxyjswqlnyixqldrbshavvrripyfxlrdjkqcnbbojrtkbgvesgujlooksacqhfdbamgetdplovnhljjqmjahegsagdfkpmjxravsmoyhchwejjeztnkixyrgxxswuthwsnqgumuvslqfhlxtuzawchvvshwxziyepyqtwvwevgljorevgczohpaeyaeckepxyvuwgaxppcyaqnrjlwqzlngpydoqvbyrgrwnpnwwnnyutsfioumwdnmhyajcndbhxmieioqnebweycevjvaqbnenxhxbvforbjokucjlyhecrhgosfkpwhowjuqnacirzyjsnjtghtcddupwdhlyadgmtoxlcxxsvlbfljmbwgjiallhqnmajwqueasywvhdtwhcrqeqmzsdnxpzjzjsrbnmlssmxeaaalbghrfamooumrlhjgiabpcanzlsxcxkzdgvujpttteynfumwnmpyeesoqtfyicvmrjvzkuusdlkevmepbivpxhjwnmsumnatvsdfjgiktkjyhwmsexhrxxspedhblfffkhrwbqwxxrzkwpkvdqevpqxlnobztmhlecywidjthkutjxnjvgdxfdaqyvqzkcxbrfyapkvmirijqejqeplqutogohvhuezkqnykusrkjmrfcqgvbnjhiifygztnxqwytdccgivbwqmcpcznxamdvycgjvujkndtqyvtmtluvohbaesurssnbwkfzbhbvlbpbvvpdscouskdzscofsdcjzrxiyibiedwjkulvfxvbllsksbuhcpypsuiufvfqglwlhbptewsmjaufmomdzqfyhgrdlaeugtziyfddvasrcgtkmfixeeeinnfpwdxocmxmyrhywiraisgalvgbvcoqxmzrdgcrhjkofeoggtvnjjgcuufgnzzvcqbnquhdybsbpeinisiunegdwrowcbtjhgwsouozqxzzwgllzbouwggocywzctxfwpcmdvzwrkznjaptgqfgxbsbohbxetazinumeohfyundmkpwigovlhodunwczozpvfqcqvimiagfyqoqsjzwtjuxpclttlipgwguhjbtqkudwvpbyspmxbakrumxfhjgpbkgchnddkphqdjtjaevjppiunemhrbkdvhnqiosgtkzmgcbxsbbsfqkquttvbvlbzuwsxyynpqicqmbzgbdighddklpaoxfheffsjfdcecdtjseyxtuzslkazqcbaexwvvpnggshwirgnmkomhreobzrphlumzeedsrhkrlbmzmooxkxumhissxywzhvymtfonhjkumsuswbewfllwbtsqhnyyhwsftsveqxttufnnfpjatlgmplhnyazzawcpbicdcbcgcljhivcjhueodayzhucsxkolwtnkxrlwqecdrgdxxlevacapfajtpmkznatwsdnhqheppevoxllwtqrnuqsnwxwpggjwnqwymgptmnaytczgnodjnpywxzzbcdembzwghrusvzsljnrucxmenylqaigquexutijtyxonmubkumyufdgsnrehofhuctdhqsivymlndqmwvgnesclizqeeefakpkzoimqdkzsbgweixhxxkjskupobzjbgucynyslqrklnyiqckymtcpfxahaudxhhtqrwnnqmwytgcxcomavrlypydodumwshrnrqinttrtnjzkqklifgpqdypddnbtjcmddgubvxhiuvcmcvavzomgprqlaqylyldosabcxijleklcrlrfhlikegkjtrocvjwqdgfkazqyjyjxxytwzsmhdvllfwfateyxqcodmxtmwnsqovilydnschxvrlifvzkacjrsilbjpbijvloqugatmrfkagcekyhesdmkqhhhfvaebpqdsmambainjyikirvbdxhsvazkvgnhomtfuzyucvqwkokbvmrrmutkcafeyfaqtnrvoumixsukutnvpwtqbyqpuhfbrtzkncfiiazcuhnoqzfiolfztacpvfmpbzpxtamirdhhlujmtilanjnekgjgqeikxrksksakapsjqtmjgvnhkjlepqjykdrgykumnaffvygsfrczhyqflbrprlzhowbomkqbaspqwjqcddryedcdlylguzehqazzuwidmudyrxcwnwnmxwelpjwfpwgjjfnqjunmaxjvkkizcbogxfcykrvuflwcoxfyjhiibuvxjuwwlxpmexdtsuixnrkfiwokxyqpgxllhlqiixsubitvzabebevgjhxuimagjbmptueteinygkjuxtphuoelhratsvjlrrxkrntnzvjtnmllljvrqkfnwprhorjmbddctcldwizqishugevemzthppahnrjhmjdzouqkehpeaiowatxndeaxiwoglhmndlbtlueinzfxajxxrreofsdpmcmlkpxkdcghsqquqqbnknwiownrstliohvjfqebogbjxzwdlgqyeqigrunelgyjbheuorbqkqtjcmikowuhtroljbzuohuohkyedpuvzjfhvzabeyxvnzkcfznjwdhyufofayoaeixfpcsvdzrpjolmcuyvjyjiwijxdgosvbmdjevdsoomjnubaoezcwcdimbbswxocyhigoxxciutvakbqmjrikgalxoggvetffzlxrpgahiaxujjibtexexurnoykmlorvtmupdqsbuzmhtosiyvxzzklgdvdodytpokeruozszybpjtkkjmcrzqfxanscdxjfyocviywmllgyctdylvittmunxidmyligvsalttzosvqoqwnemxurywwabtrmbbdkpazayqjtgvgfinfocepqidlwivfsufffknabzjekeefowowwakghguphpdetpwkwhnrspciybwrmektwwyyrdqfffzncusdfvaueoynvywvuwyubtaqzwprtcsklngyeqwpskyhahgpjrkygzevcsmjoodokineycadbeqpkqynjaevvxbsvnvpsspwgbotahlwvthaaisbchydtkiqqtpvkzwafdgjnxhcvsgekcabgejasoczoljyhfegyurirjggckoxuvlyjapmgsxltfhtaybhitcuknzdntzilpnyoowscrkivqzrdxhijpcwvulgwxolobendvtkrwdeywgmigbyvjgfmtwvabvrlqygyceawwgbktnrkodkuwiyermlxjlmowbjgjssodtwxiftoqfglfeyruvxqwwpmvfdyhqmluqlryfsvsdjomosaersveqtsbwnnljrlqpgbulkghhfoksxvemneujjdiszejmzzvwuuvbsoflwfiquonkhtwtifybhllhsgencyocoaoqdgeyffuwkmfixreakanytytqctdulbptidsjlqakpbhfpiypkzwolpvlyjxiwgzpaavbdasadubwuvinxnpvdsjzssotrmzvxmnnifausqjqvimsrykasgbkxvlixwqicvboxfrwkhxyzeyzcqwngiojucwrifyegtzcdjpphxjgkfjjvspphydffwaszaxntvuhubtzorsqmukttbqdiowvuwrlttlhnnodqablvjxevcegmqozxnktqqpvapmfxjswzgojfkbswtxqyjaaozmdyuthlhdxxtodzgfmkzhplinppaacumbpmwhujyqozmomnzcvjznuqexphqbyspxkmkyflwbpzanupzwmtwmewvrjzigkdoseeimtchqlzuslymvkqyzkvukjoyybqrvugblteyejozaiierunlwwwydtmeeggdxzugifvpwqchljcennboypxfxxcsllleszolngabpqcbwitrzwwskhguheeaojynxojxmvcwpcbxhpdxgaqpotilndchklvwjorelviuvzcmtomfpexbhbdzdoaxrennltjijfvcsbatuuudfbpxhbossijadggzurudirohwaekbudsbwqdqgmehltmmsdiiisjtnszpxtjzqfhlxoigqxrrtnjkbdvohqncmxritaiujvqpwomlzwllrrslowfnnbgmlccvtctsavavdgiuukmtkdpsmengisfkkiudxlrgpgeyknokeokgfwoinwxsywhphiwoafncgfudulmntwmgrkbcajfhftqkbxoskmeieadearjxlcvcysgclntgdxfzunbeanmtoqzofibzldefzzcgptwlsrfaborgiogbkqhdkujdzxglthwxvbpgcqcqjhdzhwpzepcsxscfqnucuzsfvnxmmbftzheclwotsbdltfaqbgwlgsvgwuoxbkmnpkusvgsjspqvtcoatlulygejttbqobcxavznncxgvqedqsytdxoptekfxzvxznghhxzkwlfyeuueoaykhqqocoryawouwcsyjpgonetqzkkuuosngpkrmcbovgifzzunyndjgugudxsgwutmzzygpbnrtssndrjurstewmkbcrrqznrgvanrogjksfqjksseqtuvjiejfwhviwthforlrvnbzznptdqvbqwraqtfhvbwdrdhoriheyjwicnarnojglmibvgzgxjmfpvwnpsisevmqhbatydjcbroknfdnnaxwfyugylhkeaieahvztldzvkaxgsoilegvsymdyaedryubdcryjbdshbqqchmpzfkmnjqtkzxuhnsvjszjosacmpswarfeolplhagcjippynbitkshlfhbospcysbeapflfsboqebxgucyrdqorkgifvbjtoivlyskzghhesmnxgyqnjpnivacszmrunydymoahxlqksxxqxokbywvxztetcbfsenpdwhnwcipdcaqrjllnofhnsppltqdkhbbqkyqtctcyogqqhnwvbpczlyzwlzkoqabfhhyewhehngsmmuwnjpkqghxbchagpmklenlpsdsobozanbjbmkimtzcwfmvigvscgpqvolyeanqsllectfsezmdrdcfwfgewnqsgjrvxpwlieojmwcwpudxvrnzelenzzqzdvljtouwspqxwrfwhojmvihuqazugwjoxmkprysrjjxoeihgvdohuppnweujlbuloleidasqjcumyylkrmvopxsayhkabbapawseytzaasbeathakuynlnppjikdmsrsxsmsjlsnkzjrwxmipssryivnlysrbydvxarryfqtmwwocctdkhbstcmzvnyvbdddpxmkaaozohciqhudtlqmrtladugjnjnqqrwqpbvieamavmrofyfpqgjiapzbibhdlaltijoyzfssyvcvuuwxabsszfrnrjrkyujtullavdlrwfgusajwordjrfprzzypmubvyzqwdtkcovyoyvikbyofcufxtauzaczjoovcuanqjnkhuvkhxepwebqvpwpixotkxvscmluplqrscyvodoqqizfoxmuvvtyrazsokdlliryyseqksmdowblnhrendcghvstojiysaithzshwnwqhwyczycyclkuqlzlgtxlroigdtdtjjcdqgthnobwkdgrctypxloodprvucnjccfaxrmhrizlnhmvgpjqphphlmokocagxtkzoyujaosgegnaqmosmlzigggwnplxfinccydwrlubnfcbuktuvzqzigzrikaggwybrceobeipldcqoypkieakmrtfgoxwaductesiumadblvtmxvwbyadsundkldbzinegbbumqsglqxzyvlzpustistittgpmmdplroqyhlrgyqiuttcjhuvllryhddujgmegmllykicujijrpkhjswhdpfnuqeimuchoenbhrkjbudmgmoecdtpjuzmgufrifrepetvhgkpskyoqdxzldghanzzeyvdoiwtxjwyiiitwjlnqnemwawtvryrbsiofnjnhvxqeimygfqsgzdyfzkfygqmjogjdkzlygrerzrzdhlzjvpkjehzzhpknmmkchwaxivdtlnuqhcyewisxnbfxwieliaopxerlispaduvdwsbipaehwbhhmlelcjhtvjljzsubusmuaqltcgbzztyuvkhfchueietozbdsgfblcxqbcssivkmbpvhpblrhbwwrmgspfxzivsyhubfrggasfvllymqiziscoohfdykcackgufyyyvqmiifmqudgzpxwghdeqtzzpecjgqdvuazegszacameezlrwazcfiafiqebsqiieolazctwplazyqsadrrqdmygxeoizzrpzamkuqjcoonwaxqpyqbqvoxyweptoruqootamklssmornmzeywgkppmppxujtnaddvswlojhgnxcbejrveltbkztgjffdeeeimmnyompgoowwtpefdgxvgahuhxoyydoheafxxisyozamuplzjovgbfmmtpbzukjinrtgqfqmbolvcqamgntdqhnsruppqywqacmytzotbkralagxzmewrklgjvvsaymhgaqhocdpilhqfdydyyalenricsweystosryxehtpkxyvflhdqtkyoxpaigbkrlpfdhqowfopjwnhepwknkmrrvfgnfjzfqdrzduulteuzajexildqdfntpqmghdrqanaduukvunfwyylxhfryrvyevdrkjeemuhiwcibkrozmwztfhnconwxmuvmeddjubbnsbvfzdvvjwpuablrddlrluhuavupbvgcpdrmdmyxhfomdnvlejzfvkscagxksahkqoxdtnuowlwbjkshhvpltppcmxuojcydnbmtncnfqeghgigldzmgzupufwbxlehdazyidwbfdsqkyduazvlkwwpiceswgndilkkteggbwqltistitqcfoaaxjmwgqhurysxydcrehdcwjrogqjkflelakffwjyxfrzektjepbgoravydgbgrpilkreppmdkwhteolfiwzpubwqfjmswnoyfgzktzdcfvugglxrpactfpjhwokdcqadfjnfcrkhzkkcyusgxuhaakzvponmccjjdjpegzsezakjscbhzfloxrxlznzogzawlrermoqnnmyxmqhwgcigqdavizcsiwmvdmrowiltbrlcaunwwypftpsctprgxeswfuppwmzuwypsanajafvilpyhgpvfnptxcgowyvbiycbusgxwurgxzxpjtytflxrfncwroybadfvpqjexxuhlsldaxcrwmkhljrdlduifzhilkdvdfguyygsoptfvyjoczbcnexuqacqbfxpuhixicoyfomfzcykifksrhqkcsxqguftsciikixsgkhjkfgyppjfafdcyzcbguhaqtunqadtqnptczmdqsfpccngyxoxjldqekslchvbqunjvqsqmzuyqllcjitedfjilatiwrabvowjqfpwdhuoeuprwzojzdqnlogooauexucthykvxcijqfgrbyjgoefnfscvvffomkvzellewgvyzbooodgbctfwdliylikxxzkxsxeehdvvdnvwjdykbiktdsusbfxbwglmjoumifnccarbmfinhtponkjxdvnejuzpnkkkiwidbbqyctobgccxvoecluamqhcdcwnsfcflzgozzhbigzzqoakyoczwrfnfrteayiumxuqugojlfvlenglofhqepfnzaxvvhcwkmuyhphevaphskyzrrujjnpvukozybauovsqnikhpidacpqpkrwjtsxiridtgebgmudphncdaufagzcbzvqnhqiueamhdhhfcyxlbesnaqbjzjgiufxpycfpgmnhqepbbyxnnyldtyictqgqdggrazhaanuosgcqqilnyqpmlhwwywabeoradrggsmhsgmfgcakrxsysoatxoagfedftrdwowvhzekvakflhyuwovomiithqjaythpfyacdpujlaroqstboqkmxqjxcjoreltkcmebqrhkvgltiwzqgzqejzyhdansagxtpxoqmrjgdqhqqogvcahqdvltifwrqnfpgnapqgmjlmgyoonghjcsaevtrowlypvgmkmfkhpocufpuvqkufxgleeyhzvpvfhvdwpdajtgplagmxsocjsxdpwygtigvuywavzihujwednsfzqafictknjivdhhhqiaktrjaeyyhipepcgcwmnsdgvukbctrdwwrbnhcrsnavksrkshefhzxkgzwoqbpvkwduebjxvpwfnufdvtogqsjpfwzutwlmwfxxgxdyiuwutwcpiomszwyodfttmedmojhwbfqrglcsznsigmwwxldrtlaertjcxdkmgabtjhqkniygughprqsgpkvkihstencpxfwtbzoncicycykfupewqluqhpwsdkkwlkkdbjfdhhctzdggxoxzeoetxulhqldjhgnvgmkaiwfvyhpzblzwrotlukarptvemurteqccygzqwanxldploevkvcvguebyoaqzsnfzkcuncwmsjwnringmkfdvkwiomfhorooxtainftcarojfgtsxydljtdvodndjvultedottiymcjkmckewmyefralfybkduxelvhfitqogcmkphostnlirofutkxxoyfszslfjwsdwhbtgeigyxwoowudwoqddmjgrmoaqcsrfjeevjqvmpzhfyhgmeqeqvzbuktmvvejcnhulwijpomtmzesenebjwghajcrpbalfcdamjaymjbhbolemcdatvdfwjhvowfcnnchchnsnzxrakzmbirmedutuecbhtawnzohdtreikwwcjbnrpijcteprpnvemyyfejkuntgtbmjglubdciggtponyfsmkoltpbhwppmushyjoqbiqqtxaitydioxaabnysiskegrvodubrtqsziufxirlfkzcqglwnsedrajknrdqthyqhrreichephybmhxjsmlickwumijnxqmakdzdihxeyrywoamwammpjigcfllkjqiawradzeidrgtbcegntiuqzfnhyegweefodjrffqsptpvevrkoahbdbufudnryplzxchtxlzturwozzselulupdbawirsfqemspwbhxlfzvvpeausfmgvfszsemonsoiwtvjcmiabcemymwxonqikpvftvphjrbojyabaujyggycgrvbqgvtcfnmvkcllenkmnnvsijitjvpgkhsuxonahpohyhxagalhqrfktefhweothwfigrymrfycvurblbxcovnulmoyaumbzzcrxodnatwloohhhqdhmbejqtdlsheymcnhbhihsxywijtptzqkphxnxijvoxqmqjcrednrpsovejsfppbseyvepmqmzuayjitjxnpbeixqtxkbyojgbkcoyyhatvylwnmkoulzrkprnfeaanplfikygbzaqfvynqiwbnktushixejwnqoafjzsvsxvbtfhbqnulpgibsmbttavotetdvoitrygmgatkmcrlprmrbmuttwjymngfzeiqyurjaqerlgfebbyxvhfklpvcfaapiduybsfrmqtzhcbhorpicoskvcenejqhqbvemaoqpisiqsqnzrwjhnwecrmbnkjvhymkzlsfjgzpbdtxcunkzfikluelrdmgsnudvsdrzotqxfqedfxnupvzugyqdndqsxzigkrzygflvcmjkgcxpaqmjcheirjdhkdqgjbfxxoeunnzmfolmdglfxmtamxllsjiqcwdsvpmrqxcwkigaxyikwqovqzjrndikrnbqkrsacpropabacutdzdgiiulskkdncyejuhkimwzknjcrwsekvlllfvzwwfmcufiuwlqckwvvxsbgwljtfmmcdhsuoruclebftsednfeywqvscyenwsnhngfdiblncskochybucvlubhwrblwrxeztlvqypzcijlpdowpdeajmqacbnndsbozkbjxymycbqsqcvvalusqqfuckkshaivpwjammevufcipcqqapazczhrnanlxehfwapdqtvvpumpxffoinnrardkjwslkblcwqmkqbbvpdgukqkqxabjubaotuzihmubotrycgscdkxqavdqaniwcreznkkstemywdryvfyzdrepxujcqpuazsbtpylqwtbxlrdrzprovhefcutrlwakwdncbbprmmmihchjpqqckcwudgfbnuoysfpkjxsmekfgcmjsrpilhdjgpnjfbhvpcmuyrgnufpiilpaehfmhiarakvhmtgjgvhxlhzzwzbldastgxepywscoawrnktkcridestqxlvmcgjkhjepstfctdzavfrhthmqatnjhkgwphymojuwxvrtzytengjnzkqcbjyklgzceaybwigpabfdjnomddwrwpqkxqrqtdzgcperndtfucshdrjccbeqrszbiqkqxpyjlarvyuozvsalzazlmkealqdakcbdkxjrvsrulykhcsurbjznufdnvxgafuvnljqyvzwgosbienmpsydbakqleautgapvpzrxfrkzigboxndrbusaeacxlhrgpjcxompwobmrfjwacpbadyfdudvxyfobvcdykaghxtofcvfptvutmdyorynyrsvakkkzyzdmdpikjsqviuskpttuezzvumrwaiyitfqevjxyovewlvqbtyyylbumazzfexbexljasfadeuwftznkjldjbildslxauisimcuarhoiyufdcrhgnjtuiaxhezehqvizgvtvzkyxmddxmeazxuadrschkfebbsnohnzhgdxuhnhytqlmeymmukpmrxqubmfdouhlanemtqxihqgxgwmywlrdlpasmvlecruinomrgjicnjcwhmdrneyddxqocnpoaqpdehoatssxuutojzwvgtmgdqewuuoeqdxefznohmumllnszffopxrelmadgkfixpvzhdlmvmehwtxezdqagoqmkjpzpmnlmosmbhgcakchmfqawcinwdtimxlikxcswbsddeyzyeuwspnwnjwsktvgkrfzwcbeuhkzhigjegchyyvtynltamfdowzzrsnlzkciidemkqhuwmfielywgvqftwyhwptgajusjwzbnwcdxsfjnbvryhkbwjnucnazaziwhybrsyvfzgbstqsldjjnjsncyhefpwfsviozwfovsffrwydjdmbyyshkspmgqjohsziorvjhavziblqnakebmcewqtsmapimbrxxkrjzrdommdjepmatibehxqfefkkvsflsxmmjrpzxefwnivmdxxckpwcznnqpihfalttvvxetyjhpglvatjrcvrcppnzcflasedekycezwhuobcqolwucksewxrlzrzlnwmbgyrihthtoqmvvmyemvrvhzsmtjsaxuqblopbljdqcyvagmdxlvfhaqjznrrmunyenanuoaonulihoqlcdksuufelnpppwnnhglxojejltxqzegicxehtcwlyoyeyclsikswclkftxymwcmremslaiqvfivzvojbxiwwlvxulhmuovblllybtggsdzppogoggcedszbtganflldfmujjpagtzfpnzafwrohwlgemfitgqptdzebugkcaicvujngtanekcjpgdfpyyeoputzmpaawrmkqbdqcfkaidxkdacvggutkmpgzosxqflkuzekimbxkcwhiucnwhlxmoulwotvbedutlnffvkdcodqptqlxpbwfzubgkcidrrfwmlhkhphipuvahhzeofdycjcniuttcyvfqruvsskjvwwstofoztwgrsbhmbwvuwufxcvbtkcnhjirbfnkvwfadxaorvrbwkhyohjszzeepbeooqrggujrnzbaufwekttvhlpgiteazpolhlhfxvxbaqzymzluulaxwbypbrornvvtojlobdnchebpnzeruixchhmsdjghhtjuklbxefddpjoxaxurxtvsprflewwdctybnoehsptmnophswidlxrkedtyxexsymbwnaystyzvrfxngplhijdkrudbrsivrcblxxzvselbsxzgpcudwpvqbjgbwkhzoembhswweocsfybvtuapmbhahlboyurnduztbusvwohusuqgqfjboqcxynnrwhqjakvuaoqkilkrrrgcaiatcuixhhahseelfyynfqjysbtxmltlpnawqelgzhtpoilbpypbkdbitkpluroyebqdrlzttsvadwqfkzmmlufkkcsidiyumicadnkqqwbpmftzamgmnhlrcrakjdhwafmasphbvuauqtoafhnskqdxguverbqhmhxsmegjgktqxudowzzyfmbqpxkzhrfvtmyqkssohwuymhcxkvilpkwoizmbsxtjywieqdssyjhhqwyfyxcnreyslelviylnvnlqytiwbqjbzatjvzfcjtwbvjzqvkkgdnjgegzjzdxxrwpqoxnymbslggzgbynecfsdfxbblksuootiygyykrgeesnmtaqgntigxmniyqjrivtgdfeqslesuaouewxjvaipowsapfefhpwyaerlfffnaasjuzvtixozeusbdcnpdzuesihoscbxhxxdewtmbdoqfwwmopsfrtzoijesrbxkyilzosakjtawazbytonefsjsfqimzvccwifgbdbsdvruyovoorepilrpsyilmzumaxazmzluqojtlqwgovaasiwuzoqkaxjyloipraxmjczhyajqicgganlzfepxrvmtnvuwzvoilgeuhliovpkntmmqplrqmesswsuflgffvsbviqinyrqbpfppskafusdxjjppzrrmzqmjvqkzbjxmewbvgrdlnzeoahumrhgekhcvrqosmtpwnfitlncplcolgxntugqofqddvtidzrlilcnyqxztepwwrttgprogocskiqkcwrhxbmyvghegndvkmcxxsrulumzemjfnlwljnzedhmheqlzsnrkhshztvypzkdbrqcsangkpmowlcfkkpzifalmmtexcgvhgiszstssshgbwjxgiepbmoohydhgvcxmeiyodiykfwqkkaubxmwlnclqpzpdouxwcfwcwrlunxhfpvrjvrugfjkmbvsyxfdqksvcsuhzzvnsjakwmutqlqcmzbzydxrvqwdtvohfdisdcsmiwutcsdvureydnplozpexwzfwhwlttdrgerbbirwpfrzpfsyytvminrllmyuphkoocbdgzxsnykantqjjcwywvnarhxtvdvharsgwyovcmttolujvxpwaidlwqtmvrhsmsmnvfutxohhralolautjgbluawhosbfkqrcbtclzomdrstzttetmbbiqvzetpyayjcvwjodkjcjdciubkvbbsycwxhebvxqigvktizeczjunvahzflturkmxnjnpllwhbyrtyjdvimwsehggpjnzzfhgiaswnhdrnhexibsraczwlluvaskhblqcfbuuyvwkrmcsjbcbwsbvkadcyccdzkqismnhsimirljyfsxeujsnuucgyelxnthwgrmicpulkuuyjiyeluclpizhkmzwnxkrvlijdcuoottlejvfvyhakmakkfwnsyuuheumqwcwqqcmszzskwclpirxhzaiwlpkxxfbmzalbntywytrgjfrfwjcngldhswclqsjklikordnsnupkzjdqfmjnyqpayrvzdnwfhqzumulaiwepjsquhmjnsmfpaiwmjsblntoiyqsvfessdausjqpyrjmbvqncowmfhqgmgkihfoeegammurbvlgutbeetykxpdlliekpvpzkrulyfrrxzwvwdaekzsibiupqxlbaunyzvqgtifephsqgkhfigfehrphxurydzokdvegitu\",c3=\"vxkcjddsuawtepxvxchtwskwqqrzennujhnrxpmleuoxfubjnzabiyveoihsgcbhiqbmjpokzfgmkmjeybeyozwukyynalxvgapnamoyqxrxpdlhiznbdzffeaulesrlmiekrawnpoyshayrrrgewfduxvanlrswkbnkbszccgxenvexylholxyvzaxgqhagdlnrothoyjmtjskgfbarbuvxsfvdinfndslzixkxkbutsoosptobakowrddzdhpqmbvqbloqunxpxvouiqaovcvrsnicluesufmzfnxceanmqaqvssakzkicxpzjwkzqmxkrqikirudmburkapifwfanwaiokpywaqtwmwkqdzmpbxrhutcitbpnukiklzkpzvgxwggvefosqcaakfwiganvlmdksshduidasgpocjeiobivplbjgmmyvxiilzrwhrqaiqvqbctlbnfkzysgtcqbzlbaufpohpwnsutpzlqlwveszxzrhtsqjueimkxwdtuqkjnguztdjxqnphxhodhskmmpbltwmpsjkgckllxwualstldwowvsadfszirrgdpkqckwgtqllznxiikohmtbhyqjmkkevpujfdjwdbhqqdppbbgesdnydtgrljdzxjvkurdjadeqsbpwkgbwmdtbpxbxmufjlesditdqvrmsujwzgjrvlthxpcnrmntkcrxgkjdistlatctmotlccvvsnkdtrzmvhgaiemefmsfjmvnugwwgcgttschxrvfldrjgccqdwadlwqeyjmpqqkmtxoxpbuxkshybgkbbhbuivpkgbzxyxyndqnxvlmbzxcotvstpjasvbcpxxtbownvgubdahqvljzubbbzwbxnavzhdocqxexrataojmgneaijuduroomfspimmflgwebprtjafcduirtvwobmzchrsavexxcqjnjzcmffsghyiqvqxtjcupawpnxvugrutjwihnedwvhyinyplvdakcirfgshorpukohazfaamacxpweiozjqvmbnuqznizwoxgpbqdexngztmhtsqrkwshutycisvaxzdklyjwbchhgnoppcrhkzxbtfvloxqhbzmyybxnrhdihtuxttajsyaaawlihinggcjvnvqscpcxzwynnbtcdqcuckdpndgzzaszwfnfritafwdjezhduewbtasjyvpjvltyqplxgzaqhedhyjkhsyofnzuaiqfpdahtgvtdopsegehvcmnhjytnbenuzbdpjlvvjrncfijkyanjmvgehmkdatyrmeszhkifcciwiftuqiosxkvijulcljkqoxkknqfgvzdmjtwkvcvhdzrzkbbygyvmkhhwyfvfztcminkdvetbikiumnslkdtuxrmljwbuadareqofjfkwiusxvsugfuqvwegjqjdswcipjkmwdrqhphuxahhfowmbyxdmldfehbpbbdgyuxqfrrbleevczrgqrnvxdzthuhcbcqokenixaldwyzmphvztflzfcjryvkojymlpvhyjwbpzjdvdkyrayyegidmlpvzahldjzdcfsdjkuigpetdyfujzbhkdrjauujpteotkpmjiunhripbwjkkuvncsxwjfzitmlklrhpmdguknivyxkgaxwylzbpqwsfnwdeyzhqvyovtaffwlamstodjybnbxzjpxmmdwxdmdveviqiymgrxwowsarcblphbjqxeunubzugxlsbcnipleumluhnkbtipmmojthzuwmttehasfrwmwwgxhlodyezvzhivflnjxfxijrsjvpjgmflqluwrtteeprceakeenhvwswoxpxdtveenwxdesymdofphsitykfdvahmgwevwjemfczvxsbjgpbymvikxndigoabcysnaoyvmspmtqeqsuevmvdmbronpfatcvdgmqfewzyjldebvhokgftvwgkplofgokofvdhvjkrmcjrgpaxexovatfngmxyhwxaeykjwloywdbjfyacawchfrmlzpqwtkayrkefqesionvnvdjloherriulnewgxsnbxijscoagslpsqqjymmakutogzsqnadcfbxlflaexcevswygpuvlhviahkkfxqaigsusmjbjsxailxavobljjaggacntcvxqltybhvdlxjtywfdszkltxpyklkcpjgpzqoxibjgcriastqktgzypvmuhmhjcgpmyimuajpdtpjnadnicmqmptcgancejqwpqusiswzlgpljiaytrhlygysyysmbkgzqvzugenreqdlwsjigzgzfqbfsrwhmidjuskcleyrampuxxlvpmmvjttxsesguyuyeptaqkopndlhifeawacdkdnkcytvqjsfllztmetciiwiwvwvcfaauvhobphjgykwxirbfvcnzybjaosekztvvsfoizcdacffruglgehlpbdakagvinbnqjjkkqnallyllcdfnchdtibkhfhdzpxrpyinxzvxukjqzszchdadjodqcsqbpwrrzofggrjxoukruiyticstltwyrepqnggocnwvronxzsmacikdzhmfwoejvpqcallhtbkclzihsfptttmnyozwziniblwaypbmfwwvhodeqabfaqksutyhhyvghwrdgmvenrfxebswqkdxzlipxrrjlrvkzmzcbwbwbpmppuygtuddnvtpwvxzebuhcfqnytbdllnvbbsjsvavlnqregweqnpsmxrzluepfkhqmrnslbtvnetmezclcjcbcjjowyccwdqrnsczmaqyyikhcxcrpdxzfeojplatokdbebyqfapvrqafirtlujdyweuxerrnqhzsxlkfditeqfvsmgsayudxxrpgncsaaboslauuwwiaudlirjzzzqgxdefuuhoauyqxiyoeqltafenkilqinmvnobaswvumelcnqebwdzcsamahxpnjejiehqhkahqiijqhidizhlbcvesmmqcolscsgjxkugytveczeqkuyzqeieyfqkxrxsrgetollnjohzazbpetkbpptkemvszcgbhofiprsgeenxuplbkwcvrwebbcmvnsoflbrjhuebbotbwxqlbfaccqjeglvysuwlmifnrqhydcyotwbxyhobgvcmtxcbbddahdxjqyzvnxrkjemcrpmdpbbsiyptxwhfuqvgnnazfkwawfxlwmowzvbrkuexzaqciflhmitxdolplxuwirertdqpomnnuyihvdshrgagnitzjubzeztnlivavdcxutmhoorhkvgburehikmjnjanogepcmpydipkbuioluwkbimktrfdqlnqdaaffuivujwdbpnabujflbzktlnygvsybmjedtcpnhcsjaegryryhumoagqbjwjaswccngzecdprtdaqnrwfrgfhbskqwrackdbiwowuojibenlnygzacobehavuvyswmginattxunmyabciagzblprwmzmsoqfekfjeniacqkfgdgbmvifpzjudlwufomuoqghukxitmxedarciefmzusymbxhxlfwkkhvkdmymtldkjyrbkytmqackabcojvztpoaspqhhnobqybwbxauywldgbugzycpikulerotyfdmcsxdmdmuiujsxpnppendgoyqpzyfoxjrpuhrphjlbussdavuzibdrjzxpieqxvpzlzmwosxehdhsvmbgejlxhsfspnvplstzkepsuocdccvgohxutkagtiumagzvtycsmxtezhpcvfgjwgxdiznntcykukzabwypejptvmbqkwxueyrmkxsbccjglfniejuomvbxyrajkmvphrixvovnhowpsfeaqgetvxulqdnbkaeodzaktdcvzxsedpokfttkiizuvcsrovoefbuivrudyxeltbfcgpvtllgqezqtxxwbnehlrarkfzylempzrffggibtfjoesxdpvdnuygszsgdtspclhtcmxptcsxcvcczvgkmcbsxxxjvphyliyavjobhgfkrvdapbtccjwhisqcszbtkxhidgkutseylkelkjbohdnttjjrvbwlnvydlztonqfffnvnltlkhzammbmigojosqebauzjshzlmvrjlounkuieepaccbgkvsgmhwkkxhrjrpxvaicybwawqepppmtksbbvmftbxzygrjxckllfczxhwvffbyrfofcyqccmwemgmgtotelgfceanpbqgtvdfdxllwaopnywreezllawwtyndscvlnazpdtpsjbedsdxxctoknhyovyukjjbciqfyhmqrpwndsotzndgqljulgvmsvpjeveuuugbfkuzftmrrplatnqedxfugiqiygpkkwsezgkkysazggzxjvaqevvmrfhexkwnszfdiqcqtfyhyuhngbxikejqdumrixgrhvzpojekglendurvsypgvdplczqhgvzxzalaqvppxioxxzztqhiqztoeghqwbtxajeabainpbjusjwoclubwsawbzkomsluvzzgrqwwrancoijcbfzsqlshhfzgvemnqsayluybkhixbvpaffheanfkykswlupntvqindckgkvbzcasqmhgfshlohrieqcwfjrwdrdnxmwczotqetaddnliazgkykfldloknouqcsrzuerkaubfisparhqnsxqdsmafhrvqsuwqlvbnmrymdgcafzmrslzmtihnnjkmnfkhqmiksjehwrvnmgepjndcqxybqzelmecerbkjnwfhmigxrrbqtyqqhgqjicffnlfmwtwzsxdlhxihxndslxffzakjezffvsdzgfkgszbuuvapttgctncmisgszmhcvihbkaduggmyedenkfgxrqotdgbqxxlyhlzfntcdqmtbxxfohupdgeaggmavwrrarkltzevgbugbxckvopibdiyldafvultvkcbxvpiiqzemscdobfjflpftkvnzbbvmmhqktbyeuvuqgxghqrulqbqvaisgesmymqlotumpadyltchbfrrmvlzrooxdpybebeekjpkbqrzvurysderyqdhgpmvhxpuwcukrnnnpgnyhnhgomqtrxwwtnlzojyelrhsebbhxsnwuoikpndiyplzvkzbuuutnjdkohhouslkashpiuwzjggojdnwcgcceeacrazajorlqjhbadoplqnmksxcjbyfvquvqsrocnrjrjtfxrxdlskuizpukmcekgrofrliiofepmtfjypneantxykrzaepambrhnfqryxfzqxquvcerdbabtawpopxbluwmborofprhrsmevvrsxrrvtrfxvadqxucbzsidocydxzbwtjxvplxuvyvwqadeskvtvamtehvwnmrsadpgehrcfikjfykpumshvmhdtwmoxcwziqdheydcizganzrjebjaztybbrrakkalkkmxzrjzwreogplkvevtwmktraylkhdtxrflczlqevctcmvzlarvohmbrsavdrklgynbzdyigejsjnxcgvhjlkshmlethafjnoxljtcvpahqwzinozuympfjyznaaqsqmivpoxuvvjmwrctmuwywjtbaveycywbhxmqtuqiswvwexmgvcoqumdmwufxufbairrxdnhdihkbphnspyiktvlffybnqycfgygcjafkabvmdfgknqthvgzkjhialvfrcfobnsfdohajaryoijwhvurqcwqzzxzdyymylpymjjedlrysazyjccqzdhsbwkoqkbxxqpkuwtfjvdgsfetbknckqfqrdlhlqnpbrmodhmuckuvxevbglgcpnxwsjmumlulddlgxrawmpdxpdrjttorhvwuectmugccxbcrmhgrrzjvipcjhzhyzgbebtnkatzfezkbojngoyeskwyztdwwdxqznaehzyybrwelqujxjkyyouturrkfimgasncpjmacyohgyuxnwlkiknwsqucklpyjeugsmufsbwvghklvijkssyeidpbgrbmelczojkusujxltqsymlijbczmqyovupdnxoqpsdllwabjpyrajpanpvzepkrlkhqxomddbrchdytvufjibjltixzxwjapscvxnnmgesepxpovfymvjudyzxysxeotamqbmrwhppdmqgqikswijioojhnrsobriwbchirovqrdvlehftmeeprdsqckigvxfzrnobviaepvkxwudhfpetwyhhvncvkaliiwzeoqxeqgbcrsxnfoocmhnxyhqhcpqievjoleuknldklzvxoynasifhcktxurtkujvgdhrzgxutvlccqqqnbuoatkkgqacpdmunowfwjalxqofhykycngqwdsdlijzmgscixqesunjedntwphjzgcbobgntrwqhjylsdjaascopthxoctpeftbgrgpezwnkwgrhfnkysuglbhypkprkqjczztulaepiesffwsclwexunewctbglkroevghlhmwgjqctsxdvuzkkjpnlzvvqkbuswkxmzqsosmefkpwleoczaacmulrbaqzsliwucuviwabxnmqsbzcwajakkoqbgekqngkakaziqqsvjgwyhcdzregrygipegvpwkwxgpivytpjfqcwhuopxjfxcjeynipnyeltaqljuehuzgyppdtnkwnmvjhsadjtiwgdchmlwamxwwlqyetgsvxtcwnpmznadeuwnvwkfizzpnrajekkfjsiksvuqnazoozgycvlkuavgdhzhetebgamnezbxqxgjkqfegcdpkhnrxruscxaklsrgejozxsvrzjennpzkykbelympqxpfvjcnnjxsvlpcpjbubnfjpzagwhhriqwujollcvvzkrauonugebcktrvyehqefzvcixeuiiibwrdwzpmpcxcxaxaywgywrhcfdztsnpyirdqfgobttxxilqqqqrlkcbwarnkntjpmwhbpyoljrptkfabhocadjezwqgpsfnmowssuzvpfkvbtyfbhkmpqduhukoegkwhfsqsrvijuvchsysrxwhrlshzmjnjzcoeywnrqwthroxzpuyzgxjntlaxbvjvknfbkkuvrzibevzqjlpxkvfjhyawphhwqzuaimueuezndrwcdnkjajvruofmynyybuuvapsopqrthiizcyrmmgsvryxjkcvhetsubdcklnrsoicuwtffohmlyvcsyzfjeuadlnimyhphumveetazwawozhnthquaaofdtmczubamxtskandzqlwjvhkpyacsxakymhjdvsjfosvbyolfxhfsarqiwxykuasndnlhcamifbyfzavniwizqqpimpsxpjefvnvbzdbsnsmxdvjujhnhuhjbqodbecuudhhxnuctxzejsykyivsifrphjgkmhdizbkyeutfqcsdaejxucyeqwcdnquhjhaabdkzzmywolhfgnwjmgzhitaqapwghqstaxjxbsqtoumfoacocfqiuxpghhxjbstlrfogdvekzqyuivjshirfnesnmpwxmorosahtwientfzeoesnrvhctlobflefaojbcwwupkxlcruvddsvskfwfbebeohrvxjymlzzvsouudscaiijqoccvhijhtknzarineumcpgyvsruzdmuuyaftpmmdkvgkgbwjwryjknhtwwkntrvhycyfwucwgsopiaotkvpkjzceovkjbspzivszfmspgnrvybepvylrxzsixiknxeoimejwgtzlrzlkqkfjjezzkbobsixonygrjgadduercgkjddvbfxoscjuprvyzanaaaqixfpgqssnzehdqfveweyjywrhnrisfdzyfuayhpozyypfjzhafrmfwpfjlfllmwjmhjlungmouytmbgscohuadwrpgclurdxzexnqldniemkwjraqxlctvcvyoaenlqcghxjhsnphfcnffahuntaaiyctixrloexcmzirbyeudnwjiiinzlmcittfgddsejbfwjyypyoeblweidrlouewxptupgvvxsbabyzuyvtqhlngaettzmuabasggggcorpkaqsoglmdpubxsezlhhumjzxgvsaqifvchnqrxlqoevantzsrhwhwemsfnzxdfdrvrqpwzuzyenahzriauycywowdkkendgdlwsxxcldmmqattsmkoknigkhusdcgrhjkbikfuwmzjzsjsmfeeronfbzfyqdirqzorkhoaxkirjhsiwhnfeqlznqeyevxfjiljubfayyefndpqrlumaljbehwltmleabykehfgtnnwsynqfkdddpdotgtgtexrbztvouotmyedjiqkgpmlvybixqxcnccspjstwidkwapibqpddgtkzgbacahxeasefduldyvkogtoihzmecyemrqnvsiewsmvjdzaytyjdunemvpcrcxhlznvgatyzjgmsfxnustocigabluyxozuamymdxqwbdtiouojcioskpypdsqssiqnduzswggucfjbmkedwrdjszvajwkzsubduqnycmscramhmqtutfbfyfutubrznxmtnixilguxwxtzbhfudxwlmbqyhazfhfqcdtonozxgozuixuxftsioipgfzwcnbntlpsogirjwsxftsoruxkuboagzptrskfdyfmryttddpiyqoyawhetedlnwgkyhhenockyiqnifqmkxomgeoblrgyeovqwpupsidbvrbrqydrddiluaxtpmoijqfvvfznyoatqxgibioprxwqygjiufxvljasovbjnyqwhpgfbgwouewbgawzjpdxnsrpdcthurtqqjokwdurglayptittccruwuhuypdysxutpjskxlvhnbfnldmnojxdkkdstltmwhmycfpiswbzqstasiinehpoftcgnkispayrsaewvrbiyhowqwfkebufzrddentckhhipqtpawmkdujdktqdmrnofwkkmjzgpnpepqetadjannpfsaxmihbdspokihibrgdvtmerphplvoxmuwjcnynnxixhviptjqgrttzkqlfxgfopxypfihirpkertallgdtmqnxmnqgtvjnnsytqxvvnyrsatdelyacuidmznmoucyojvhklatknnwrxtltynduyieolzcsxlcvlgyzficqdodwwihlyexkmfpsjqdrsvdgsgixwvzwrzsgriexsynjpljtnqatetyodbivclenxfoapcrmhtoxketfpfhrkdwnwvmhlvrnwpqprwqdnharfmzpkbimahexzkkrxciqgrvozpxeppoexpjjhqjzqutqjzkactfijvmefklpkpoohhyoixvylfogrqqqmzfaqqyxkfzpyfeceinuawugrrihxnpnelmstbqedpidslwlwnsqqmwsbmlrgidnijwqxkwqsqgysgyehibbxngwvygebghomjoeryipldyebskzkogohmrlnihfdcewpjyjdzidoiewhxrvmdjlvtcdnjvgkqfzyvadwkdxbjklraypvtkpjdfjwldaoaxgdyljahyjpzdlfcmxnwkasgpilbngyapxgterpjwkfncbqbrfonapuubcgygxasphuhxawustibjnpgpabqplmsukrkltdcvjiorotkqjrfycnrwmifuftjmppatyznektnrihpvlmzpscnlfwsbasxeayntxqarlnrxhqlxxliaahdexgvwxupsjskxdemzquzvnajixrcbhdhqdjgnyofvzyzafmmitvllltvlcwagqkgyxwabhsjejfjlgnahiythkjxlchjthpmebvotmctammmnmsrbnvrtfvvhoawulojivjyidtfrmxrebpybsqnvjyjjozmtxubyuuzkxsydkfxzfbcqlvhqybyhhjbpstgfzvvdnwfejjmbvcccqefsiqtxcqamayahnjemxltplkvohbcerufauwrbjivmyrdqxbubdgbdcyjwlunfrivwetfrzsuzwsilcstobttivmjdpywipddachsaitcbjbehapylwskepucmzxneovtdaawievrceqbzgzcqkplhcgfvhmpgthqbwyniajelyuvapkelsbtggpgdnnddpzqvkrelxhjmlijpvhhawquwocdqustcmdelxegnmbvaretgmkghsiwlsnpgtxiwhomiseytgggwayfnuamcdtdjlgaypppdnxklpajnvnnvcqkufcwtihvnmthyrdweqlgcqokptmaeffkkkiyhfdqrllnixogmbdiadtgikvdzddvzwlhvtcbpdgripixgarndbomqncsgtorakxzyzcezwmrglxizngfasucuegrrzkjlrrncrzbwjbnzohtedthhgtgzmawdwwmhcutulzobuiwloocosdoqoqvlecwipvrljwbjjfhlowkqbcmxduflimananobsuzqrzjweyuodofwwznodwfdxrzqwatukwhfpvdrdlrhmrumqgfzumjttxatauenuazeqsootlqhnuvtmunclwdnkvmoesxgcaqkawibgqdscytvotmlouojnjmikebyvdiauwomjszpvumlkajcknzlrejxrwkwygspctwceivbcfcudtewldnnvcgmokznsoyophwerhrezdsqspmdnfplbgfopwriugexkwmcpguudnrefvikiwznuupawzfoqeisvovjayktmyiknywdsqexxxinwjxwntxrylmttsmzwztpchakaopceinqstkzxjfihjbwklwctzbglfmscodvoqarihsifobbijcnusurvcajaiuvghtacdekcblizcthujbsihjzkxflpfirzkfwaunrijatvcoeonplrobhhwncrpwmesedjmalpbdegxxujzrpkwpawoiwxotbjsllivnxwuedddixtfloeumzlkzzndoodthdswqbzncdsodibsrcreyfmmltnudqqopfgdseyywdrqboihycpllwrrkhypvomkvtlmbkamaxsrokidrnwtqmxmojyckcekcbujnoowwogdzfutronjdmfokgzohszqcweqinbeujhoutyvstqrtwukmlfuusmsgysomfpiwpbwfkepwlpzdanqvrszolkgqaiodqkmaxccrelqsiqnhpihgsqhffcaaldbdtqaznodjhwyewymtxnqidgqyaaccshguaiuxhodmtqdguweiragfczbjswasonptbngohsorlovsxuewtwikmdxvoqcvgbnnprydrvtfkfxyapxicbwsiikbplsjwddgujdvzcygffiminzqpqqdvokxhttcoubfjvmqziuepzlszbtcwymekwvpfxoncapvbrjuynswhqqcruydzqgclyggurlimluzwtffffreonryxurejgwhfcwlhjikxgrrheuzczkxglcgadcdvhukkkjbtbdwzwkwuzrepfkkhhgdmrtaxvlvylqppwixxypletibvfhurdeswgtwomdiypyxxpxegdkqtttxpxelufuqunapvabvzybuyveazccmoigyappspgerloqttriuockrhjpmytfpbtuvujjlqfdthlrdhigwzrvespsriaquvlnkuuaqsuitruengctkfmlgpttpmiwacqwjinjrydjewchxkytriecwgygorxxxqmeishjvwuqveelvbycrmghxynalihysmxqzgltcnzuzalkbabmcjforulfhztxjzkproqqtbjoezpxtpvhfufvezpjbcnfodyutexhpbevnlxailzrmblqlpngrtejdccjwysxnrsxpkfvabbklftsjsdkatkyvndljvyonzgtglibuqffvtanvfmzudrvkpqwckwqwbmtbonchmwdxawrethirgtxgryxoxihxyuarcnfykwtjkjrhmxroosdtfboynrbclezkcmdyeovoxrddlyqirlohvpsiwkwhsmreeaotyqkxxsydwzckkenlaejtcjbkkqldtrslvkoookwrlmvfvccbzubfeehfwssdjxapumrccbotlknnzgybjwxbtqsslnvybbiuvunygtlpfpmjfopcdkbsgohndomghdwrdbeevfcknntvjexbzxwzcifahanpnkgxrarwvcqmlswwcqvvhypuvagbogdqditmguwpgvknaouvotkmknuhsnvomzsqirmulmcwgmacmxgtaewfqczedxpzvopzjrhmxrbjfidssfitndrhirwgyjxiqrieqlmcrfkxwlpfjjcwiwhkdiohcljqewbmdgcpwsvremieeecptdgykfryigujituoidyifejwzychfxmxxgbklcatzcxqyqwnlsucwvhxlkhhwtjsdtrsmrxvdzzuqujhstfbziudbxewzfmsunxflquujiocwmxzyafnprgbagpvcphhsukvhybqyvfsjbmutoyopcgbguftlpxnshepmtcekexodvjqvzutvafsftmpnnhthzqzrftupicpkzbvzdknjmoekrultshxxgsygsqfauadbtziulidufksksynbolhhdkqubfbhvpuvxnnmkfqgdfzngwieyqrkctnkztdaxibazlabnbbcjzckjdgvbsdwramvkhtsddtvjdcdvjaitcejbhtqbqdtjlubhvlbgdiznrqxmrsvylobandhcmvqkdixwdkswjglhwpxhwffwgjsbiadkwclivtgyrpoxuotsmzsqubijanjsrygkviplaehwudoiaixxwotasnfnramnlthrzhgvggoocruuitcrwuzmfawhfmqpoxhydaxtfomzxsjlxsriclxtaaakgbbtidqkrpuyelmpfvekukzvbldcnspeihjchsqdaiacciojjxdeuftmstcnqpekqhxutkpwdmytlxrhucnnxubfkazjdttxcssceakcuppgqnbycizytpjrykevqihytphtluiuatnmpbfwcqdlmlwamsefmmtjqvymmjvoaiinkkncsoeqpuhusdqyccjjnuabylfngypqcshgowdcjtgqhhfglwdehtrzilezxqtambjwsjbwbbsdocjmhbfihaosrdulmkzzadeourfenadyuzorjufajozlmlsxxjhyznussivhnnyndybssztaqgaycdmthwemqijepazfjfeebyhwhgzmbxuvrizlxhibspyixtenafnpihkabjmzjxbqcgsqpgzbdlmmirrfexbdmdvdtoqdccriopoprdyhokyciyimxlxkvgezvfjlwjzurmftqhjdxdzkylehyywfgsahmoodypljfptinfkosovogopdeixflwpgmdlpvtzgazreteuqplbfdpzpnvcrvsheizcndblnljbcswcasgwqdhjrgubezkudqvfaufmaxpfrnqrzznvjnutrzmjwaykpvdxzmkodbaadofljpcakszaicpgobldcuvycjzxpktyacwmyqqqfbtddbhmehullqyheivvswpuvyogruvoxyfmixuibovthrsauuqsopnozuhcpyaedviiuevtiyhaofrrnpgetgswtesglmngphtmbeafbrkdpcdqwbqblfobbgwcwgzbsvmxkjzvqpzldowtyxcnnnlydobdxzfftmtrtdgumpxdavnptzfrvcdgtuietbbtbjeatahpuvtwyvwwfcmgvfvojepoqmnnrogrzkpxaryeggglclpysjifbgqwbjyykvovfzciujsnljdrezguzkcixdpsbvgxuknnovwxxyprphxzobmpfpnneginstayfjuqgdlwxuxpexubkbtqeldwktyhojphfukgkjzfdwqbrndpehyrjanbvkykllbwkurhhhnuxmzibdvffyaheyjwgfgcdavojclzwoqsklpysgygcigzanyyvmlgatnllggysjsphgjcepybttkucspbumgantabsoymurhjpagaphikzvobrfvnlczawrsyvelxwxbggsjugwokuflprynywuqvpptkxlkhtzltwbqwujnaqbbjkqfewnfcsrvzjsplxjbljzjrvdkrvspafnicbnnmaxsynkhxwbiscqxlqchcqsftbsgtsxlypyromkttwmnagiqlancbeicyemaosbucgrlxntgmdagfegzfhidcuhdxhhipersqfrwdwgqhsknlthadlhjsxlmtansofeafkntoootfdqbxxjqdyqyikogzozabfueojrcjxmilmmhouhffuzlsjidmtuutnsrligrwxjdgectjiunceghovwsnbfjisuyhthwqiwzpzsxqmlwamdtjtxcnnbjyljnywuaqhzcrvshkcmamjzmewjeumzmpwsfwknvyirrvgpgqlqmtoobvetnrpbdothtbhaqerdpgvulijfsfpcogiskrmjxgqttdemkyodzcercnnqhdfjuiisckvfvblabozgpxggwizhtlieaavtecshbksazoxzfeertratptlblfkiafiajtuyumsxqioajxofeqjqsdqfxgdtwzezzpjzheuxlpoeilzaxyhpfzrdnwmdzupwppnvxvopwbvnpujmaliurrrwmxgnhnuzzxpklxfhpuaivgroqtlinowkyvakftymmuqhrqojarxhrumwkugxcglvmtjygwiyqvvbljvwrfgqigmyxrpiwsumtyxkuuhpoxdafxaxsmybuweeomfkbbmcgmnanovpoxziunrwlbszphuvejzjjvmeuzaheerhulygzirpsxktvuqqvmarkpbbujkklsqhpljanmypfbhlrjzawhurzcsbszsxcoaatlsenubsxaugehacqvgajzfyhmpcsfmjspposvhcqracbpdxvqpvdtgcsylakgcjcwduahbpzwnpbhtqzbufwjomnhemiasilyymncbrejotwxntmelzrjmabdkeijopgdptsqxsufwhgljbgxmkbxfbffwenueofpqgofckjltpcluymuekuscczqmvlwpqgcyflxwveztjymfmobqsylgtutxhnfmttdlvnzuxxoglplbdgykithjpqiqwpjwpvvaisdqalwyawadavykcegtfhgohldyqamsuvjbulzevzavrbykquvogaurelwjnpbepwwobprijsfinkoqgcqnotzxqoqndrkvmyshfadmdadsuctlimgsdhrgqvlvqkofjbijjwwzamwokhgtlpwqihohkmagilpddtaifytmuiduuqlafhvnjgtsmygmyzddacwqtrdmbevuqzjagvzfpnjknirkculozjnaeujqprbkerajxfbpoomyelhvieafkfsvlakojpvayuivgtwqjfygwminiwksyhqgarvwpezsmovvrpysiipjccgnkuwntxwifvzkaukpfytcglpmteficfvgnpapjpjolvhrhrdcwipibsaolkzduueukmbhaxjozhilwygyiuldshtzltxzaznkbbxyavdzppkyhvubahpudkkesnzrlpfvluafoaazgncoilqoudjmuspsjsurzdxffuzgxidfblrjzoejyhnwolazyytuivjutxzphrqsaetgapdivmvponoyvlksmbccalwxrsahwymcnfpcyqpuybctfbvtexjqbrmrdzaajhbwvbdvwzvctovoagstuphhubajiorzibbcwmligzczxqpcketqtghyjcaqpgnoknfaktlqaeuwgbldlnikkzkiqmyjevmdsfiubdzkexuyamjnlgexzuypezjuezqnzwqwjlpmhfkxrgqeszxjedjwyivtbmmrnpjgsfvvtfpnkuqrrlaqyiiztbsssewqsdxzysismipbuawtdvorowzluoudtticwanzjlkaidbpmmginpdkqpiwlnwkykmuvixldkakowerxalovvskydeoyxokgqixzxyoepnnfbikfhncxkgjjlouwfzxtfarnxwcbhmtjebankpzdchzzgbmiipelsvpfobcduxhtvuesbwykmmxxuioaybmzpkgfyyerqliicnkqhthxnavowptvrelidrolhjxdibtikbhdmqrlkaekbvfcrwtnnhxwoxkukbphkdlxnwvhgqvogdrongtpinpwawamztvflwetuytcwtbffwjqqrjigcawmfzhmwyqoelojuqnrwupsnvajlfeipkozmoblfmwfvfwjxseeiebriypcuijauhduxddlniksevxuppjsvfwfrvniavebjbfwozeyhsldzttpxsqxjwxzhwrsemfmzejnyzshbeobdhrgtclsf\",c4=\"wthvqxcsrlps\" 1626006833639000000" + }; + + pRes = taos_query(taos, "use line_23881"); + taos_free_result(pRes); + + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); + + printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); + int code = taos_errno(pRes); + taos_free_result(pRes); + taos_close(taos); + + return code; +} + int sml_ttl_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -1301,6 +1326,8 @@ int main(int argc, char *argv[]) { } int ret = 0; + ret = sml_td23881_Test(); + ASSERT(ret); ret = sml_escape_Test(); ASSERT(!ret); ret = sml_ts3116_Test(); From 627429d490dca7aaf2e1daca7897b6a6fb31e407 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 16:05:57 +0800 Subject: [PATCH 070/145] fix(query): fix the comparion error. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index e797c8df15..ccc8fe0130 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -788,8 +788,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - - if (pReader->suppInfo.colId[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { + if (pReader->suppInfo.colId[0] != PRIMARYKEY_TIMESTAMP_COL_ID) { tsdbError("the first column isn't primary timestamp, %d, %s", pReader->suppInfo.colId[0], pReader->idStr); terrno = TSDB_CODE_INVALID_PARA; goto _end; From af775af69b97ca87e6b71f786394091aac38e53d Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 26 Apr 2023 16:17:23 +0800 Subject: [PATCH 071/145] test: add test case for alter replica --- tests/parallel_test/cases.task | 1 + tests/system-test/1-insert/alter_replica.py | 114 ++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 tests/system-test/1-insert/alter_replica.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 590c6738b3..5b668f460b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -142,6 +142,7 @@ ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py ,,n,system-test,python3 ./test.py -N 3 -f 0-others/walRetention.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py diff --git a/tests/system-test/1-insert/alter_replica.py b/tests/system-test/1-insert/alter_replica.py new file mode 100644 index 0000000000..900b64d943 --- /dev/null +++ b/tests/system-test/1-insert/alter_replica.py @@ -0,0 +1,114 @@ +import taos +import sys +import time +import socket +import os +import threading +import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def checkVgroups(self, dbName, vgNum): + sleepNum = vgNum * 60 + flag = 0 + while (sleepNum > 0): + sql = f'show {dbName}.vgroups' + tdSql.query(sql) + flag = 0 + for vgid in range (vgNum) : + v1_status = tdSql.queryResult[vgid][4] + v2_status = tdSql.queryResult[vgid][6] + v3_status = tdSql.queryResult[vgid][8] + if ((v1_status == 'leader') and (v2_status == 'follower') and (v3_status == 'follower')) \ + or ((v2_status == 'leader') and (v1_status == 'follower') and (v3_status == 'follower')) \ + or ((v3_status == 'leader') and (v2_status == 'follower') and (v1_status == 'follower')): + continue + else: + sleepNum = sleepNum - 1 + time.sleep(1) + flag = 1 + break + if (0 == flag): + return 0 + tdLog.debug("vgroup[%d] status: %s, %s, %s" %(vgid,v1_status,v2_status,v3_status)) + return -1 + + def alter_replica(self): + # create db and alter replica + tdLog.debug("====alter db repica 1====") + vgNum = 3 + dbName = 'db1' + sql = f'create database {dbName} vgroups {vgNum}' + tdSql.execute(sql) + sql = f'alter database {dbName} replica 3' + tdSql.execute(sql) + tdLog.debug("start check time: %s"%(str(datetime.datetime.now()))) + res = self.checkVgroups(dbName, vgNum) + tdLog.debug("end check time: %s"%(str(datetime.datetime.now()))) + if (0 != res): + tdLog.exit(f'fail: alter database {dbName} replica 3') + + # create db, stable, child tables, and insert data, then alter replica + tdLog.debug("====alter db repica 2====") + dbName = 'db2' + sql = f'create database {dbName} vgroups {vgNum}' + tdSql.execute(sql) + sql = f'use {dbName}' + tdSql.execute(sql) + sql = f'create stable stb (ts timestamp, c int) tags (t int)' + tdSql.execute(sql) + sql = f'create table ctb using stb tags (1)' + tdSql.execute(sql) + sql = f'insert into ctb values (now, 1) (now+1s, 2) (now+2s, 3)' + tdSql.execute(sql) + sql = f'alter database {dbName} replica 3' + tdSql.execute(sql) + tdLog.debug("start check time: %s"%(str(datetime.datetime.now()))) + res = self.checkVgroups(dbName, vgNum) + tdLog.debug("end check time: %s"%(str(datetime.datetime.now()))) + if (0 != res): + tdLog.exit(f'fail: alter database {dbName} replica 3') + + # firstly create db, stable, child tables, and insert data, then drop stable, and then alter replica + tdLog.debug("====alter db repica 3====") + dbName = 'db3' + sql = f'create database {dbName} vgroups {vgNum}' + tdSql.execute(sql) + sql = f'use {dbName}' + tdSql.execute(sql) + sql = f'create stable stb (ts timestamp, c int) tags (t int)' + tdSql.execute(sql) + sql = f'create table ctb using stb tags (1)' + tdSql.execute(sql) + sql = f'insert into ctb values (now, 1) (now+1s, 2) (now+2s, 3)' + tdSql.execute(sql) + sql = f'drop table stb' + tdSql.execute(sql) + sql = f'alter database {dbName} replica 3' + tdSql.execute(sql) + tdLog.debug("start check time: %s"%(str(datetime.datetime.now()))) + res = self.checkVgroups(dbName, vgNum) + tdLog.debug("end check time: %s"%(str(datetime.datetime.now()))) + if (0 != res): + tdLog.exit(f'fail: alter database {dbName} replica 3') + + def run(self): + self.alter_replica() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From ca4df761f1273c36eb8ea7245a00d539e64af2e1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 17:14:54 +0800 Subject: [PATCH 072/145] fix(query): fix syntax error. --- source/dnode/vnode/src/tq/tqScan.c | 2 -- source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 27db66f048..1c916d9adb 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -74,7 +74,6 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqError("prepare scan failed, return"); return -1; } @@ -119,7 +118,6 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("tqScanTaosx prepare scan failed, return"); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ccc8fe0130..fad2e883aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2220,7 +2220,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { SRow* pTSRow = NULL; - SRowMerger* pMerger = pMerger; + SRowMerger* pMerger = &pReader->status.merger; int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { From 224d87b1313e1474660918d3791b8146b41b76fd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 26 Apr 2023 17:58:14 +0800 Subject: [PATCH 073/145] fix:cosume null if rebalance --- source/client/src/clientTmq.c | 5 ++--- source/dnode/vnode/src/tq/tq.c | 10 ++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 54e929c9a4..9e60f8b04d 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1363,7 +1363,6 @@ CREATE_MSG_FAIL: typedef struct SVgroupSaveInfo { STqOffsetVal offset; int64_t numOfRows; - int32_t vgStatus; } SVgroupSaveInfo; static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopicEp, SHashObj* pVgOffsetHashMap, @@ -1399,7 +1398,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .currentOffset = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, - .vgStatus = pInfo != NULL ? pInfo->vgStatus : TMQ_VG_STATUS__IDLE, + .vgStatus = TMQ_VG_STATUS__IDLE, .vgSkipCnt = 0, .emptyBlockReceiveTs = 0, .numOfRows = numOfRows, @@ -1458,7 +1457,7 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows, .vgStatus = pVgCur->vgStatus}; + SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows}; taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } } diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0080feadbe..7539761f4e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -368,6 +368,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } taosRUnLockLatch(&pTq->lock); + // 3. update the epoch value + taosWLockLatch(&pTq->lock); + int32_t savedEpoch = pHandle->epoch; + if (savedEpoch < reqEpoch) { + tqDebug("tmq poll: consumer:0x%" PRIx64 " epoch update from %d to %d by poll req", consumerId, savedEpoch, + reqEpoch); + pHandle->epoch = reqEpoch; + } + taosWUnLockLatch(&pTq->lock); + char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, From 213e8b89208f60efc2a321bd7f1c57c4b06882c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 17:58:23 +0800 Subject: [PATCH 074/145] log: disable some logs. --- source/client/test/clientTests.cpp | 45 +++++++++++---------- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 12 +++++- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 055ac450dc..65f115618f 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -165,7 +165,7 @@ void* queryThread(void* arg) { int32_t numOfThreads = 1; void tmq_commit_cb_print(tmq_t* pTmq, int32_t code, void* param) { - printf("auto commit success, code:%d\n\n\n\n", code); +// printf("auto commit success, code:%d\n", code); } void* doConsumeData(void* param) { @@ -1053,19 +1053,24 @@ TEST(clientCase, sub_db_test) { } TEST(clientCase, sub_tb_test) { - taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + taos_options(TSDB_OPTION_CONFIGDIR, "/home/tests/dir/cfg/"); - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + TAOS* pConn = taos_connect("vm116", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); tmq_conf_t* conf = tmq_conf_new(); + + int64_t ts = taosGetTimestampMs(); + char consumerGroupid[128] = {0}; + sprintf(consumerGroupid, "group_id_%"PRId64, ts); + tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "cgrpName45"); + tmq_conf_set(conf, "auto.commit.interval.ms", "2000"); + tmq_conf_set(conf, "group.id", consumerGroupid); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + tmq_conf_set(conf, "experimental.snapshot.enable", "false"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -1074,7 +1079,7 @@ TEST(clientCase, sub_tb_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); - tmq_list_append(topicList, "topic_t2"); + tmq_list_append(topicList, "topic_t1"); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -1093,15 +1098,15 @@ TEST(clientCase, sub_tb_test) { while (1) { TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); if (pRes) { - char buf[1024]; + char buf[128]; - const char* topicName = tmq_get_topic_name(pRes); - const char* dbName = tmq_get_db_name(pRes); - int32_t vgroupId = tmq_get_vgroup_id(pRes); - - printf("topic: %s\n", topicName); - printf("db: %s\n", dbName); - printf("vgroup id: %d\n", vgroupId); +// const char* topicName = tmq_get_topic_name(pRes); +// const char* dbName = tmq_get_db_name(pRes); +// int32_t vgroupId = tmq_get_vgroup_id(pRes); +// +// printf("topic: %s\n", topicName); +// printf("db: %s\n", dbName); +// printf("vgroup id: %d\n", vgroupId); while (1) { TAOS_ROW row = taos_fetch_row(pRes); @@ -1111,16 +1116,14 @@ TEST(clientCase, sub_tb_test) { fields = taos_fetch_fields(pRes); numOfFields = taos_field_count(pRes); - precision = taos_result_precision(pRes); - taos_print_row(buf, row, fields, numOfFields); totalRows += 1; - printf("precision: %d, row content: %s\n", precision, buf); + if (totalRows % 100000 == 0) { + taos_print_row(buf, row, fields, numOfFields); + printf("row content: %s\n", buf); + } } taos_free_result(pRes); - // if ((++count) > 1) { - // break; - // } } else { break; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index da08bd01ac..a318b9886e 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -113,8 +113,16 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { - if (terrno != 0) code = terrno; - dGError("vnodeProcessFetchMsg vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); + if (terrno != 0) { + code = terrno; + } + + if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { + dGDebug("vnodeProcessFetchMsg vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); + } else { + dGError("vnodeProcessFetchMsg vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); + } + vmSendRsp(pMsg, code); } From a751f750224edb7cdcbbe55e07b33207fe532da6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 26 Apr 2023 21:23:39 +0800 Subject: [PATCH 075/145] fix: fix syntax error on windows. --- source/client/test/clientTests.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 65f115618f..b9062fc8ff 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1060,9 +1060,9 @@ TEST(clientCase, sub_tb_test) { tmq_conf_t* conf = tmq_conf_new(); - int64_t ts = taosGetTimestampMs(); + int32_t ts = taosGetTimestampMs()%INT32_MAX; char consumerGroupid[128] = {0}; - sprintf(consumerGroupid, "group_id_%"PRId64, ts); + sprintf(consumerGroupid, "group_id_%d", ts); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.commit.interval.ms", "2000"); @@ -1100,7 +1100,7 @@ TEST(clientCase, sub_tb_test) { if (pRes) { char buf[128]; -// const char* topicName = tmq_get_topic_name(pRes); + const char* topicName = tmq_get_topic_name(pRes); // const char* dbName = tmq_get_db_name(pRes); // int32_t vgroupId = tmq_get_vgroup_id(pRes); // From 6436351aa4ec3725f1e9feded7aa65b1ac6a05c8 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Thu, 27 Apr 2023 09:13:57 +0800 Subject: [PATCH 076/145] fix(taosAdapter): tmq lift blocking time limit (#21100) --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index ba937b40c1..4a8f4864b3 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG e02ddb2 + GIT_TAG ae8d51c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From db1d75bb48bfee6eb1a3e0f2ecce1c3629e73cf7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 09:28:20 +0800 Subject: [PATCH 077/145] refactor: do some internal refactor. --- include/common/tmsg.h | 2 +- source/common/src/tdatablock.c | 2 +- source/common/src/tmsg.c | 2 +- source/dnode/vnode/inc/vnode.h | 11 +++----- source/dnode/vnode/src/sma/smaRollup.c | 4 +-- source/dnode/vnode/src/sma/smaTimeRange.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 34 +++++++++++------------ source/dnode/vnode/src/tq/tqScan.c | 4 +-- source/dnode/vnode/src/tq/tqSink.c | 6 ++-- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/executor/src/dataInserter.c | 6 ++-- source/libs/executor/src/scanoperator.c | 27 +++++++++--------- source/libs/parser/src/parInsertUtil.c | 2 +- 13 files changed, 49 insertions(+), 55 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 02c097b8d0..8a5f3a72af 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3423,7 +3423,7 @@ typedef struct { int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq); int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq); void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag); -void tDestroySSubmitReq2(SSubmitReq2* pReq, int32_t flag); +void tDestroySSubmitReq(SSubmitReq2* pReq, int32_t flag); typedef struct { int32_t affectedRows; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f4d2ed01b0..3558feaa66 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2388,7 +2388,7 @@ _end: if (terrno != 0) { *ppReq = NULL; if (pReq) { - tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFreeClear(pReq); } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d9802244b7..5f8236b7a9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7436,7 +7436,7 @@ void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { } } -void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { +void tDestroySSubmitReq(SSubmitReq2 *pReq, int32_t flag) { if (pReq->aSubmitTbData == NULL) return; int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index aecfb9c3e5..3bcb1c9d2e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -259,17 +259,14 @@ int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); -void tqNextBlock(STqReader *pReader, SFetchRet *ret); +int32_t tqNextBlock(STqReader *pReader, SSDataBlock* pBlock); int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); -// int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReader *pReader); -bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); +bool tqNextBlockImpl(STqReader *pReader); +bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); -// int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); -// int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ce987ca88e..69b3f9c3e0 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -684,7 +684,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 " failed since %s", @@ -696,7 +696,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); if (pReq) { - tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 5058a7fc76..2a26f65bf9 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -332,7 +332,7 @@ _end: taosArrayDestroy(tagArray); taosArrayDestroy(pVals); if (pReq) { - tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2cda12c0e1..0e9caf24ae 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -288,7 +288,7 @@ void tqCloseReader(STqReader* pReader) { } // free hash taosHashCleanup(pReader->tbIdHash); - tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); taosMemoryFree(pReader); } @@ -322,12 +322,11 @@ int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) { return 0; } -void tqNextBlock(STqReader* pReader, SFetchRet* ret) { +int32_t tqNextBlock(STqReader* pReader, SSDataBlock* pBlock) { while (1) { if (pReader->msg2.msgStr == NULL) { if (walNextValidMsg(pReader->pWalReader) < 0) { - ret->fetchType = FETCH_TYPE__NONE; - return; + return FETCH_TYPE__NONE; } void* pBody = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); @@ -337,15 +336,14 @@ void tqNextBlock(STqReader* pReader, SFetchRet* ret) { tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); } - while (tqNextDataBlock(pReader)) { - memset(&ret->data, 0, sizeof(SSDataBlock)); - int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL); - if (code != 0 || ret->data.info.rows == 0) { + while (tqNextBlockImpl(pReader)) { + memset(pBlock, 0, sizeof(SSDataBlock)); + int32_t code = tqRetrieveDataBlock(pBlock, pReader, NULL); + if (code != TSDB_CODE_SUCCESS || pBlock->info.rows == 0) { continue; } - ret->fetchType = FETCH_TYPE__DATA; - return; + return FETCH_TYPE__DATA; } } } @@ -367,7 +365,7 @@ int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, i return 0; } -bool tqNextDataBlock(STqReader* pReader) { +bool tqNextBlockImpl(STqReader* pReader) { if (pReader->msg2.msgStr == NULL) { return false; } @@ -387,20 +385,20 @@ bool tqNextDataBlock(STqReader* pReader) { tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg2.ver, pSubmitTbData->uid); return true; } else { - tqDebug("tq reader discard block, uid:%"PRId64", continue", pSubmitTbData->uid); + tqDebug("tq reader discard submit block, uid:%"PRId64", continue", pSubmitTbData->uid); } pReader->nextBlk++; } - tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; return false; } -bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { +bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { if (pReader->msg2.msgStr == NULL) return false; int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); @@ -415,7 +413,7 @@ bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { pReader->nextBlk++; } - tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); + tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; @@ -451,7 +449,7 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } -int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg2.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); pReader->nextBlk++; @@ -560,7 +558,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD int32_t sourceIdx = 0; while (targetIdx < colActual) { if(sourceIdx >= numOfCols){ - tqError("tqRetrieveDataBlock2 sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols); + tqError("tqRetrieveDataBlock sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols); goto FAIL; } SColData* pCol = taosArrayGet(pCols, sourceIdx); @@ -568,7 +566,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD SColVal colVal; if(pCol->nVal != numOfRows){ - tqError("tqRetrieveDataBlock2 pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + tqError("tqRetrieveDataBlock pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); goto FAIL; } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 1c916d9adb..e049e2d390 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -203,7 +203,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { STqReader* pReader = pExec->pTqReader; tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); - while (tqNextDataBlock(pReader)) { + while (tqNextBlockImpl(pReader)) { taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; @@ -262,7 +262,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReader* pReader = pExec->pTqReader; tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); - while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { + while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 62b81305b7..c2e6946b04 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -695,7 +695,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* len += sizeof(SSubmitReq2Msg); pBuf = rpcMallocCont(len); if (NULL == pBuf) { - tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); goto _end; } ((SSubmitReq2Msg*)pBuf)->header.vgId = TD_VID(pVnode); @@ -707,11 +707,11 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* tqError("failed to encode submit req since %s", terrstr()); tEncoderClear(&encoder); rpcFreeCont(pBuf); - tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); continue; } tEncoderClear(&encoder); - tDestroySSubmitReq2(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 0dfde8f579..2e6d452e95 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1388,7 +1388,7 @@ _exit: // clear taosArrayDestroy(newTbUids); - tDestroySSubmitReq2(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE); + tDestroySSubmitReq(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE); tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); if (code) terrno = code; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 90d740bebd..22f388d406 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -301,7 +301,7 @@ _end: if (terrno != 0) { *ppReq = NULL; if (pReq) { - tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } return terrno; @@ -326,7 +326,7 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32 code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); if (code) { if (pReq) { - tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } @@ -335,7 +335,7 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32 } code = submitReqToMsg(vgId, pReq, pMsg, msgLen); - tDestroySSubmitReq2(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); return code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2389c7252e..8542481419 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1646,10 +1646,10 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextBlockImpl(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); + int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader, NULL); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; } @@ -1687,23 +1687,23 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { - SFetchRet ret = {0}; - tqNextBlock(pInfo->tqReader, &ret); - tqOffsetResetToLog( - &pTaskInfo->streamInfo.currentOffset, - pInfo->tqReader->pWalReader->curVersion - 1); // curVersion move to next, so currentOffset = curVersion - 1 + SSDataBlock block = {0}; + int32_t type = tqNextBlock(pInfo->tqReader, &block); - if (ret.fetchType == FETCH_TYPE__DATA) { - qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, ret.data.info.rows, + // curVersion move to next, so currentOffset = curVersion - 1 + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); + + if (type == FETCH_TYPE__DATA) { + qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, block.info.rows, pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); - setBlockIntoRes(pInfo, &ret.data, true); + setBlockIntoRes(pInfo, &block, true); if (pInfo->pRes->info.rows > 0) { qDebug("doQueueScan get data from log %" PRId64 " rows, return, version:%" PRId64, pInfo->pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); return pInfo->pRes; } - } else if (ret.fetchType == FETCH_TYPE__NONE) { + } else if (type == FETCH_TYPE__NONE) { qDebug("doQueueScan get none from log, return, version:%" PRId64, pTaskInfo->streamInfo.currentOffset.version); return NULL; } @@ -2072,11 +2072,10 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock(pInfo->tqReader)) { + while (tqNextBlockImpl(pInfo->tqReader)) { SSDataBlock block = {0}; - int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); - + int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader, NULL); if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { continue; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index ac504b9809..a3b067b94d 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -324,7 +324,7 @@ void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { return; } - tDestroySSubmitReq2(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); + tDestroySSubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pVgCxt->pData); taosMemoryFree(pVgCxt); } From 206c62703a2f278b946548ed1e3696c850159d18 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 27 Apr 2023 09:33:56 +0800 Subject: [PATCH 078/145] fix:modify the compile error in windows for long string --- utils/test/c/sml_test.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 76dc66169b..f1f4bbc1fd 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -1139,15 +1139,17 @@ int sml_td23881_Test() { taos_query(taos, "CREATE DATABASE IF NOT EXISTS line_23881 PRECISION 'ns'"); taos_free_result(pRes); - const char *sql[] = { - "lujixfvqor,t0=t c0=f,c1=\"tybxdylsdctesxshtzygycwgicbegupwlocvpijzdybrklsjwtuzclrsqwfoglqzvwywspekqvybdcuzcvsehisgjycmknffusjnptxmwlfbqsusxvgawcrhivmkscabawtjjyszyriojydlxxostnyisxnrtbtwrnficxxmcvgjgxoxccoqtiwowielknelifchfgecighcczyoqszpwvvqyvuzjuyuddcqtuzaifihymwttlzzlcxaobwgfyqtocekdcuolqwylsxgznorxzieznrxrrvmvtvkcfksahsnklyavsytmmmgjzarqkkzbdiocrupuycahrwdhaqzsgbfqxdwyisispodrzkhwgdodimxjihyrhprinfppknqezmgkphxmfunqshcudxrmyyrntkkkmwkdomcgvavdjekcwjkqcoimcdlzonoxwnfyeduqbombbzwexgbgkurbcbmeyxvrmmedsxyrhnqvbgimhamcrxsqkelnvrogarhvjfisinzkiurxycsaxgouaelgwjpuunftbhblrtnzkwmqfkbfnbawiqpjpacdadatotqshxpbvaoioxasbivpevgphgnurxsstxivpgcdxbkaipbnsejqpahsoeodxxazjozchjygksssebnwsjacdbkdpvcblagewuviwfnyblcrililpwnartqcqsdhwtmmgrbxswlsaskpaafosiptvhlgcgmvwsqtkdphqscwwxilvdsiwptzhnzgyfoejsahswmndszkrlatitxskfywgpxafsspoqfzzwogarceudyokprfcoaphqfkfslwmlgpyjuivietnhkofoibjwhvdbwbcckutqbejoovwlsyuflcsjznwnovjjchhgczwjesczcpfwimhnabyqlzihocraosqiaenutvvdynartocvxhjszdtvyqdnejbufpbzyyrskhdupjojkoheccpfbaoldyvgeayrjvkudbiqkieazsmpkfqunlzxasjnnqczkithgcolctdxoefuwgwdbusiqunwcuttzbnwzypwaahawexjpysznwnjyrkyigirmqeszzlgkwzxkrjkgdrprxzzwtzylwsxycgkqwsuxdvllkvxdpobccuxnnmdzqjmvlcyxoljgsztgmytrvgqeewmcpchmpexnobhakpykkftrzxqllgvhpdfyyucpshmosbocudxbttciqzzechzvtjigxjlzbfblfeztamfxmdbxvrhunmpfiavoryoitzsfeemzxpadgodsoydhalzhnghicpniorhcptwzfryucpefokezviorspzkfenyrbdbfkdnixkfijytqrxtvaivfqyxqikladjhxnjndammqvizckinppmwslyyhwkfyyjrjjnpefhesbdkpgjehochlvhskgacmbunndjijonfndvpcyxpuuvwjyfmaejnmakvrfpvmjusomnfzooplswmlbhrjoardxxbaemwgtyufyqpgzsfrxkstoohkshspnbgqgsvmlwbsivfpfwzwijthmvothigzrvhkqlmkwzwvpgedveqiedgoshddorrndjykqowfrohdulxvktatylnuqfqdidodrfxfslemggzmwfqapmgnndqvagtxffetpgpxijzhdpletiwurdqowjmntuvikvkzphvaqdaqznlhtmvunfmlpetysqcpzyjpmjjgnbxollkmwiuhpyxuyejctqdelupexjqpvnowudzpftjpggdtdrxkqgxpzxugqzbgsfkywfvbqzzpcokkdzslsmiwvwbfaijrfldicyhmxdxsuycnbqzejzkvlztysdohdcxiekckmkmfwlnpxlyiamcqrkqrdomoryqekzvvymppjtwubqdatfplwzhvlfhyqiinzqxhyhnymccbogggxyrezckjfqjkhnlopdkmqilfpjshuszdyudyhlgqlngaageodtpyzhdgmasrvgzogiuyhgsjrfzfvkwfpdbadqmugigiepzrqgzibmoumfmrxuuvbaaihunaypgdeaswzdbaqoubehpmfqbklyodcvbgpppputmldxnjgcbpcjjknvtmxfstrgxbylgkucbuqjtfswlgjgtdgrzcgjjonziolmceutclufnesxchwoecperlpwdwpicmspwavxsjunoipxxdfbyipsiojmpdvqygmqsysrsqvhjdunsimkvkvvconhtnnnqhhbpwndjrtapxqbchkrinehqtorytxlhbjtjqkitbfkdjhnthevuxnlokkqnnaztaarvfmsfqcepxgacwomwvckhsbejbioxwmojnooyukswstlwqkasiohatkdmfozncjdawzgaroybvvlcsjiiojxuedjzgwouookoxwfnksvoislpzllcitwwpiafeqcayktzurnoztdgjuwhjljszreoljjowttsteitrwmujqdaidhlcxfzdefkvbkmsmuyhozphcyuxwmpyipxfxdpdfojaksqhefqwhpqcplgmomuhezvcwcrdaksurrbqirlvgjjezmmooflmdepohrijqfwbiqxpepnexbpbdwvvzqgwzquvocsalethkqetdgnexughqqnbnytkmiwrainnqlyuwuljkffsjaommyftidtyamffyqedxfvacnvrrckqaxlwyvjvvaztjqxmtuiiltrrlntefluclbrkytjstoeznorpomhwtluojbnzqrzksaedosuslwdfftvcilagbrrxweldblmeclqaskclradfldhyufwqbiitwlmtccaxmswhjpzmfccqcxzyoneiudmxndivuteqkapjpnqsnxksgkorxwkjwkrrbkplrmgqfcazkjiprhfmxuxshdkldkfmefctapwpynozskxpxrnsluwuzdvhprgfpjfddnzncdtawkjbnktjxogaxhpcymxugrdvlkrfilgsvcglklihoiziarzuawuggrqpuhhnpjuijllckkruslupluzhggccebqqxktgfxjrnwamwopzqqsuvjlzyqnmudfqfxvacuoiwwhmawooaognltuowfmbslbtiyxoddglgszlpdlkwabfbiwdvxkmyvnktpiscceiniuvohafyudpxpqfowbwcuglyzbrceljxdszzozpyrmuoewmanlbfzibnmoyrtowsgqdglikzvvqrzjjoulopqnfvhvhdqynamxlcnbscjvkogjwqigpsxkkpwzejwnwkxtqjpibfrropketlkxolxhpdsjkvcduxcfregcogjimqshjhqamtcwefdzgkelhojnfmpcnafvhdpkgsmzmatuzivrsaowkaohdeeybbkzljnxcazvbobginldgseyquzwqyxjccvagnvofatcfhpfwmrdvjfqdovsxhryrpksxcnbypntvucrhovaupbibvqozvjfmhyrayvqizncbusqfhuhxaffshymjfubrohpetnretdvshlxmurqvdtymawuqyxbhqtfkebypnutldzelwyhbexsvznlyzeswxdymisetpybygvsegmxiribznlypjrnrlheofgavvlfxdaeyepwelpkhascutjaskrgkzddwhyhexegguqwpyxtuusystfsthwhaabheqxrvqxoshuairbfxhxyzoxslwydstlebiansxnjlkcdondfnillshxtagbvufpnlzhlfpnlxbrtuuesiouktwfjpnkzwypnperyakqrmbuvjdzfmayjsnekpiozozkhqwusqucqeqduhnnezagclocdfcisfdobjafrtdusmcdwtzhrdylbdvaudyolxjhnnxbqzyzlvesifftcczwyvxvpxjgnrddwozhazbhgnabssztudplvhhjxshnhpryiekxurnegvklesafkvuifzubljkhsrfufabtrykfdwexjiqpwkcvxbfhcvozvxzjubvqzgaxjfrtgnbkectdbbozqqkfymwcrbxfelargrnqggseceslnmeoyskruxivvpslxyzccvbxejayvtrhpykjyleoevwpzdetyjncwntfjyqddyqgbrvdtgoyngpsdvtiqwcyugvvjvltuhuwrxthcnrmjemvfluccerjylureacftsfsmbexudajyyehohdqteffywxstkwtkodyvbpxddjbmrwiccaxisidmvmgiligukllqokyhateqcqkwsvvzyeejaqwsmyxcsaavcvhshdoidcpuioqeqwjxzpceqjypmlnosupcbglvddxwsvmnzjnwvktpdujotjftekcnxiwyvmucqskubkgcmcpfsixynqdgxgjrjdxdhrrgluqjqunbbepizkchmqmrlioonlhxgluldiijonwryoffyfxbwrdiretxeedlaeznyfcplfxucxwaygjgjzpvtbdpljsuqdqlcoohgqfqtxrbkqkdjezuczuvuwjdhabuvpzlctjwzlqkbrmglffkdxxhhfpruhsbnxtfwnlulhenjbzddnipxkvewdakueehpkllugcmkoyzvgryvbddlbmudmhaznghbpzthkzvowqtubhkvihdsqayytztpqytdcjarkgnflapjqcpeoffakggnupyebaqxvpwmqwwincpmnjtyprjhyjhwxcyulgsmcxuihpjfltultigzifvsqdtgagwkyniqycftqzebioccfragqulcqlpzwdcmygehpqzbxvlbunohlfnjfpbwxwrqglyldhzsxmndaqqctxrzugomxkyivwlrrhkkfhmkijndrqdqppdtnvsudkfnbetzdelkvezaedxelrqekktarjrwxdxvpjwksavwbkflmrwdsanigmgdillfrxhkfmbivlvetiqcfmkqqqrdhsyvtvsiijkjbadizacjwsifcbutsikrqhhyctanumwdxflvcpwrdccufmufmlzomxtsvhvawoypxijkdgefwdutiipyoudnevhqiwvtmmjchuvrjtmbdbscbulfijmhpeilyfmardjiprckybweyccrenqkvvrniimwtsdleorfrcrgfbczqsxckciprbobdclcnpeibzqcluvyutijtwzqlrwjxspzqleeatzmqtsdqnuqjtqhacbbvkwtbteudvtmqgjagiqkjmdhuvfxwkucgpwkqxpsewgpvfqgubkiqgfcpvwlhpmwtjahqdhjhmevgdoikopiqtrfbpdmbeyzxdzubnwdgwqdtxcvdqijmpuidwplifmemhzfnegwkjcoszyewvwwqzjldgsdlkecasrthnmpfmrzcvgqgiyvgbnjqrvcwpdfrtkozwncxbnfgnjjuzgbqpugsefpbdxltadeuyrwcejklyjggsifvnetzwvdqbhlakbtemunafezlkgzzeziuxziwyjxciicokcsxxzefiosdfxmvcyvvsjtnqedncbpngkqwyhygdwcmjdnrrtpazdydudewobwyzfvejbutmktcnrlirjjmzegnraradtinrrbusbzisrfrostexwoeusqrxbuhhdycneorvrwxdixhjktnhssqicujibyfpdnluulpqmxranemqtmhclmcaznagekocqttjalztonnvqdxxajzblpizdlphcafbubavzqlhuryumvxsjjnubsffwduxddypvgvtdmqozgrhwvrwmdpjvdbexqugrrpxpvayqbccxgbvhqejmzuuobtkwopledyuqxlmhhawvppkjbyrrtjxitoytspwqomweunqplvagndtgvahwxkfxbeusydbbfurgtwtgjnryljupomgksprjqravlvprcnhoaagpcwapsfoucokrlpmcjtazxqspgyzbcaearbkptcyhkbecpmmjkzyxwdixbebqkgurdqgybvsmzsenxgljpxcfjmvrrrxbnvlsgdwvjmsjvhttdnrgowrcgggzldxirxluxrjlbueqlqrnmvajosdwyjiwwbmqteiwmifrsrlfpvxuoxykpukqfsgxdxunnbbiuoivyczqaylyrltkbsucrwxoosghizxztiqlbkywasxkyqhoesuwdvwzwjfnznfrkbjmlntuirfboqcpkhqfcfjdkoroydcnpwtlymvbwvdwlqqazprpjpwruvtmaoazyeddxqobxvqimqapjswmviuotvkvalmgjdtrnupiloxihdgcbujrqqvcljsksbxanffwhcxtvzxpwvhoctthkidvolnvfrpmolngrsxbbqoqizmjcvzveuarlbrcgxwwqrdefefjhjqmmgwdkipvsriiiubfhiffcpkdlbondnzmelvlhwlqlpbhrmugkeaonhfrvcimninyzauigftcesiihtumuxwuktbbulqqjnkxgkdwbslfdafwdentobwafxyhyhbasmiibaxjoyavplbnqciuvwpzgckfrwuvhbgwfcwzxzqcmtytrfelbvgfdnxjsuzeeiuqfkmrfmudxktyvupucnlfvnmkxscslxhzbpecbaiqjnbydrfwrgptegptfzkayhltrisahwqtudwhwjotnhxsvkgviuciagqvlllumknjtjcarlfccaslbrqpsoihlupkphgpgwgzodtksgrgexcaybrltrlqkzadpoinhkmzoapdxflyexwnsgknmxuupnhijxltmwdzjwpsspotmkfothqebgmfiafxdytvwuwptyrjepyjihxxnfdnuyxrcdzppoxvjjiujlbjtjuusdtejkpuiompzkhrqvskcjnhehskjybmqaryryzzddjgvwbwvozxaxurfoxnpsfcmyemeisgtfrhrbvxdqdmkklluemluklxhfmsxspzznlurzcuswdfsdmgssadjlemornqupbwbukizzvlufwxogmgluwdksnevsvpqzutevsyidudtphvphqtdaqsemnpcpicopkxmgazmdqfnxjxbuunohrvmknumchplkvvlbqgzpquwlmlcxiuevgnepdfvklwplahzgsscbghfjagcrqjlpjobiqdzhqswtiojsrhbdjmwljowfzebncvdxnsvqgkzrygetamrwbupghasymtiwkmlndvhkckazssyjlymkjotwtbdfbwecfrsilltrpkaeiszklwqcbkxepmenazonihsqdtvxzeojlbtlprmsfzxnxzdguhygtyiyqnhfkussqxbkrvstdkxnovtmdvcjmaugeizqknamjnpuwhalxqclygvdnecomtlzzymlbctwfufdvkqzedoeeuabdqnumkmfqgvdqtexmpeyiikqycqxtanlnqvbeunaowyqgjmdafxsopblxmimafeyvzmxmeqhhngoxygmhakwisvlnhcqzurxlitwplasuqzllrplahahvwwkqfxktigjanzhmfdtwsdydwbvbwwimmxszzwlroucukuzannqtrrgqztkgepipxilxxjcrwfdrixjdmadhqyhukeexiwjmqswhgegjvawsnntgtjncspitspgvipxvxsqvofjfgkzkbhxvnqthikgdsqyzswprfyxsxiozwjfgnziurlclpoucwpxoqgaqipygykyeltpkfvotvnustvzlalcgbqsoojzjfyomfwmepdwnfdtpyhyjfzqgjokyujudzkfrzsnzpuerxlffzdzadgcslpivtaunevdzbynyixhhzanumhopvlsvutnuytqbsyaqwfaovpqluvcqowmjlrmollasvmorzqptldwpavcdmvkmpnmkgljoiukaaqndhalataljocaowlwreswkcjsiujeoewojbigpsozmfenvzxmissyerxiglumbkgnnzwzoxkbzwgsxvhzlkqsfkeqfwnajkkymyunsoyukwuxfwicmzoadtvhesvqmzxwkuceshbkrljhqipjlsejdjycejkbobkkwggbfvmatdkqllkzkzbeeqpxctdvqjfejpnflyyqgbkveistvtpsyosexusayaawcblcwlfcuqjhjxvtmhzbccdjtorivyxvbzlrpzpoljnzowtvyevkbwvdwljynbjxahqxzhxhmaklckqoemeudjidvkclsaqxhavadwdkitdbuqjneaqvyzcyqilixeglczhobrlknjyyquxxnlxxiohmtrqkcgenwxxgglumjzyygrozfvvxsuautmunkraggxhihaafxghyyexwfwbqmqzynzvmygeakzmqnegvwcwrkjplpqfawkazkykmapleonxqukjqjkvpopfrujizaoawdxlcqhyzkyjwucbgsflvvnvfroybnabigpipyghsiyotdfeqrusgghzswuwvntlzqlgzsmqllevdcysuelxbyiyfofxyclmafrupqszsshakoskgtcsoecvaoduezmqczqxwjyaildgxuomhrzfbqqxfnkztjrvaajkcgxmwqvmrfrzojazdkjtsmulrzgrspwdtgvqkhxddkkxhkcasxlbejnsbdesakmrvhyaziigrfunuwjcrtwllvxqosgubtheowoukcejnkeywktloeilohtposjbocsvntqlkwislefrszsxnzjvtvlabigcifkpgsinbzofkapdzjgobytxlxpxnqtkzlqywetksrjfzsnxhshudmwxztaksskeqxqzakqebpdxhcjfrqjwmwlptgvvbhubkkrpvblzalfhvjaitreulgolcfbtiftwraluhhdceietxdhcicvedqcsbnpnsrpskfrrembdbhfxwumwrwiuvqelqfkypdihofvkgivahaaeyqljivvsykhcqzwyzpdtrwiimjiudnmqgbcgdvcqnyzmfyrktjeosjojhykwwqgvbawnobchrglwujfsddhztyhrxfxpaudksdhiagxebgsvzhkazcprmrdoxeryeudnufozhnlskvmbdwxqxlpmpzwujkinxhuyabvvrpdqafwpliohxydkqykejmqkihjmtykgptjwwmraiczztxtvatzarvlgpczgrbuahywkcmlcusrqyhcpssdkmsecemqfejiwoftvtchqllrdrztfjnmfitbbdoptpbjkrquqsusdfkhnywrymtlymlshqqwnnmyvmquoskahqopzrkioakiqsyhejutaqcrmqjcxgbqciajlqoeffuwvxmvhaxfuwoumxhgnccvzxbhrxembmohjsqlxzzhrynjnkobimzktxvxlqacglzoborokeazhvmkcgzdyqmaajdzhlfgobussodslwqfcpsmyhblkjszhyyhtdkxlsiloldpclulpbhiftyuxxhwbahusttnescewxsbtwjoclaudpunyenljxdguzygdwpnbfjfqoulfduyrtreinewgcsxfwrnohbfkqrmqloqyddprwedzqopudlzyurpzryaltuybttczggepmplazssuwshxasmleqivtgqhtioifmwqihnpcuxhdxaxwibybphxdfgbsawzwwawowtbeljudcyhrsdttgynrbcqvaojovpexxnevtdtsudukjbofusthzxjsnmjstenvnjxqektukpndxjfuojzmachfodeykybxbmdkuprqkkzuyvhhbwlvlwhbtunslevzaccdqxcibtwiuujwfvwowgiubudppeenmcdnhthnkmymzdbyianzgsilcidlefernnpdaottybwysyclmyrgfyjjbglifbfbzsrrqeyebtbdiwrqjjmmrcuzoxuafigderwgcinmivdfmgvrzwettesjpkaptfyogfhbntzwywycgvelrixpgaclbcwabghcldjflycvtaijslaharpcosojqyrkmzkrxtofigaolnhbsrwosntcngeuxbmrdodtnaouwnmcozrpjfbltqguikjzbzmjdwmginbmbpkbiglcxurpcdkmbhvwicxyonpznbvaxhfzqqkosdzdbkftywxxfwfcgzqibsqwrpzqdntrobcfsjxplgzceplxjugfvgfvyrasurwjyqzhzapydjtquusohlpsmwektnxyvmgzcnymwiuqzorxvlvjkuxwqwxqbhcujlzggzypymjqpsfxtruijklkaaviufnygwomnthsvgzjheqoloftemnoqhwxrhrheqdmgejbyszlieitydasqrsbknrtqhnnwpabflhvfluitruebsdgxipjimeqqegixngbiyepzgjokkaadamrpdtleoakijpccpsvpyhkwyubcpbjxratjstbwnzztbyenvudidekaefzftaefraotafbnujgcgihojrfwkbqkppzusxpqhzmbsjuyrmjxvcydwjiedddgeypqytrevbvwayymnqbjzyjvxxlojwkdmpgfwgrlqgkwbltuejvjpaqcowfumrqaxrrzlbharznnrbxdcyrtufmkoxkjkyqisdyxpxqcsaembumiaeofuxurtfqwhiomofwxlqbedlbmmcloqivmjthdnenggqipizewoqmjqyjcnellxdnlnwlaxsruifwpajjcqkwapktjyzlqxhymngnthawemhkrnjcnghbdaoynvdtxmjdvrbyhteqqjlxtupjbolrlfedyfkrzguompuuthhkmkjdfupmnuyrehaeylxhlgjwprexkybaflumitudfilawpmksysktnhxbdrvpdsdxklstgejkeibrjlzjfmaguoibjmrzwjkfbmpbzzpmgwiafrizxkyvqwtfodmkwcbryrubkwduiaavxzfibnjigllqudqvhoxjliiwveuwxfzbsqflxuncxxzezxxqkxbnzaudjckdixzwckkfdfwxzkwqjujwurodouzomdysynsbdwgzvuaqhpusqyqsbiriibtuoyakddpcswwbrascztkqwkxtvnpafmxdwzjqodebtxxayokgoshoxdyqxlxyrscseasvvtdhxfeztiriqapzhcvvvuzpqncippbwhlbxokpyxaixcuwbahemlpdnayzjafupsqzhdbekhqungusybissgzkhpjeaiocubqemhkngdukobnozozoaagbwdqrluhwsxbfkbxbxmyrqsgeqswhdaxiummendqttvvwpoflpuubgazqmxehbmmchwqnbdlskqzbbnfllqteukizmskrbfapgbdwfrktwhqxlrzmuklepritlyrfmoyuzcrsqoegbryexhhpqxlufehzafotnxhcemqyfqekuixozczikejsyeevgplfoopqlprnfpricfwdvaeojlhagvctecqirsiyppjqsaosmrwholuxpxezirvhddlrlgmlukyjvziwskxcypxuatmmaywjreemktplsijkzblagncodfwcymmkfohtteypcorifflnwqckcutohaljtdrthvzkfuwpoxjtlpsoxkmfwdiyzkkdbjuanbtdxpfefbjkqonpgrtfwhulkssbotqgvndxpjtvexwafnaiczgtwlrtxrlsntxjzcesplwafenddrouaezrbanevpdkewojsyxpwsnmtyfrnqxrnekeselocmrdyknvvsfilayqjryqkfuwgcvpjjuveunfmwqfnwdrbyyyrzrtovwjeimcvotwrlxpjnzcwsppkysormxdlrohypvgszexmvcslcmqkzoodlxuezndemosvxrktpvxzyuymbvqdekzzellkonvicekxeoffsqpwwggoktypqmrwdkrnxheoxpsfdhvczrhmihlekjdvkvdnikawfhxuconhfqyahpddryuimdyshipxemekvnozyrvieezytfecydfixiydrveaamcdqsofbicyrnfzrgqsjdcpgkejrvdqgttjqvkqmqiesemvuafyowkeqbgxtdqfhrirofaefbogdhnwacxxogaobeaoglrofiryzhxmndgywzpdrrgimhptsrpskznohtiiijdnitaczajewlrichkvkzijkjhzrurzaqbdedpdcftarwsqzodkizjzltojvsiqnuuzmwnircgpodazxnumryyeqzxrwobzpzeauxnpdmabkhkoxvqjspnqublgunntcpoqsvrncgkojnmzjrifcmncqxeuvkyxmevbgmmakxscrmhwrbxmsbksadqnzvasrikfvhgryofnxzqzzknenhwdrlmdykumnsufqreywpcsfoeuxdifyivguqrrzyjknqzlwzwcmfehfcfkkoqfyfholtnpglpwodmwfhtwyehkdguhygzkbqcnjaltsgqdsvlzfkytttyuuvduxbssglbhgxzqdgturhaxoeaeigbrmzcpdtrvoeivmvtkiqrjiskdkdpqlualqjgpjxwwgtuglikhhrxcoqwtkzrmwcarodawtszzjnlwykzulclkoqaexvyhlfhsxreqdzjvbvznhwdcncfjscamlqseieufjvfiwixivgshyrvjyqlslusujnjosbkqcyrkhwqgmgfvkzgeugrsvtiklkrvjqjwcyrjemomtgewdfbdmnundgsatqdbftjhvvcvndmtmnswajowwcxiuhmtojthqkrzuvfmvbxdbvrxjbwijfuxkepzhjpdajilscqjgfxoldotaronmpkhcczehntcushirzzbhygnhzaryvehviqczxkckjbwqxubimjfjkwmsuobpwhcgywqxbwlgsasvonjpxwmyyfgasvsomgjgiavenjcbwwhqhbhzplxzqpyguecjyicsjkpcwwfjylqnvtdqoikkjbppaljzsnoxxgfxzvdygutlxnavblxalobhsufoyusyeiggxplniqrzcbbmunyqbfudjmhsjuqfpucseihoixjrhokgnscoeuhrrecfwjrdojflmqnsosailywobgqxiatgdgehlqooxrospcmxscsnfoidijdzjfudprkgkdolzmyccpseewqznnmzuxxtuafuvgykzielxhfhoxmyoewfhqlwwhczwioankvdugnapxtxnohqpclcftahzbpmzfwmtpdzqwasyjjfmalihvbkptxthuqnivalmqnurkxsjepppepcprbeczbrasanjlohbyxbkmrjvwexqxwhpbmnkjathbaygihnzhgxuixdledopbsgzwjmeocwgunzrkcgavowuevsutbajoiiytozuxxzphnqkkbgmqjqdwrqaqxkiamujoiypzfhqgmflwjrwjlbnkvzuwzwbhltzcxhxkedcfallyiywiowvpwdvpgdauwiaqfefbgxyxcfhszdoohwzdoqfgdzhdjfwtcsejtmoqaxkhdeagyvccplrjqtjxahlixxwtgdpbkwovccrhpmtqejliklpldljipzzxuwjlokvzhbsaarsqlyazsxxrixugbnijxifccfmteizbfhdzqsabjebwpkxyfooockqthtodjjjxepgyicpdnrndopdzkoucytywunnjsweavxoxonyovnqneqqqpwyilsobiiuegvnmwhpgsmsjymcrgrnucsqsdgeuwjbaironzfbfjnboiqcuhoqyqkdtorctjdjfiqbsjbjdrhjwaynbhgoigoprxhcwdrrspcbjjykqeydvrtctdqcdgbklfmbjyhxtajpvmefrefmlcdzbnllscvoqjjhpfwgagbjyjxoznbajpcuzbuxaolbvtaqkefujjxbedgbyrgxnsybsgwkuyukjsdfyxdgvusklfitokuetjlsshvkpvvfzxtmstjiftlpglhdjinjfoluokejsmmegioundlngqhlehmxjvobmxpcqhgljmpnwmluutgaryhjyjzkcgodxympmftevxdusledjaczzsviwttygxaswfgqzkkdhdgsphteqcycvjvvukchshbgsumvfgzagjkqfckqmpzvykuhcbbxqhmkkznjmimmopjgitgsddmjaogwsoublwaqpyhxlanprmnufltgaaklurhbelomdpusskwahiaonzqnlkpftiiycbnsxhpajqxsbcesedzpcdeowotgjrxwzlqllhgnudbudxlpupvzajnqngwsbkkgnqjcfgdwnypawuyuqoatdaxveziizabyktndzkgojxqaunonhupgndhxpwayzrkvqwzxriaskzmidnukxeunpulkouhwulnsnauwmubddzyqhbqsbnlhfuvdihmacylzenwuwcrlqtummcjrkqbfgbezgejcvpgurdmdtdkzzvviffxzndmxmmvjnljbnzjrtybdlacgsyzcrhqovlovladilnntzawsyxhmwyzldklxwzninbxxjlkbhifwgzpjxlkfzqblhrgwmgronvbttikwicibcteexbwggsipkubtqbhxemtifqrnfdaewbrgywfcpqfdacwhcsshgyrvrkddvghbvwtuwacqwswyaqcrqzyvlnjnzkrilimqzcwmhllzlindulxmckdhtneslpnlodzqzfxlragjtbngazbgrkskclgjljxfbrlhtsrukiqtkxigmrkuikydmqlbcibnkqtgbenlivlflmbyectljucdmuvbozumpopmkflsplzkbwtwcsarhqixsbgsqkhangirikdilewqdumsxekthzjgalyrzqystkzzebvviqcxdfzfecupgrhcrpulqdvtwqzjinecsdwambidunlxisrhnhmqtlaojxuhrgspolfhggbspywcbyflaagtgiliowspwkcgycfpchukewdulmwsdwrwzetilsomqtckejnutxssjyporppnaexvarmnhgsvwiodtrnbgivzzbzfppjebldtxthwsutydfqirqouxmxocpgircdztvhauayykwbauwozxknpfbddyvxsjfutbkdsaovcjkdpkfeknfsspeuydaxdkkfbxohsnmeudggxcxolojxgcpjqmowkhxgwgnxwrhmqfguapakhlerfyoqhwwawawcpgbuilcmdobzmwgwbeqqquejetwvnsziarxdvouugmetjjmhfcvnaetyvjbzmwwpkddjsxeqbdczecrlnzazjjddlduhbrmfhpkelmkhyvmadrkgxcvwyveyyagfmvtrpwdfrmefxotdrduliitysextojhsirlsiagqjbbvcctkcdmkcbthcsnfuxyreeffixdlowerrgrxtovtrhugafbgruayddylerfxnqnjpijvcyxlrqkmhufhwqsqaagrziaoldkqcwdallejbkurbtwzwvzhfbtggfezjubvcjkfipsnoowfwvsnezakllfwwhwrgagpstsddaxkivbmzndgshouzgrzbpqwuaquqjyobdaqliyuufjwivdtdqzutofvwinukwcqhmrjlytvchgdahvdqvnauiykevdyilclhanlrlnfgbvqjifzqmftnxcitirzyvgjjsmyjbxtqwliqoprikkfexvuiaqmonzjcmpdhcnutfunsftbicgifdynnpmaaexxxixfrtszbneujarczqaglzhfaqfbjikhfcogfcduigyolvcrqusassrlsnebmyqlspqvkxbfkqftbwtrmosvafmahhxvdummjsbhcweotqgmdzkxsvczhiummtzyadnnmvpbzubazitewofuievjzpwkyqwxrhoevbyvuosbclquuxatofrwfqmadptmgjuovcrlhrhmyafwibmrermfqkscompeexmtlannjqazpsruhwbezgfxkbpxqghhhtoudcesesfujdcmjecjadzmxrnpbwywhbjrsvzkymybqrnxrokzyorkpsssemgjvalzdavgupfybiyqbftvwscwwqsynmvxmnoyinqihrtjqkihafybaszfztohtombfgaarblacoaelaopvhqyomgotovmbyixzpjdjocleizxkocdyivgfmerqahdjzsiousuxhnuojgqkbjdykcnpeosgpzvwwqkdtukpattwbwepxruqtcxrxuoojmtzwozldfabgbshkztljrrrrdkscgvqbtqpeewxrbplbsrowzlyezeuhgjqmcfozjyinygjifdfyjvorhltqrzvoemboybyimtwpddtloajtroimjpggoxhfszrkwihedutonowdgudhnzulrdsafdorgxcymdzztzwvvtyvfxsqyivnzhcamystgvmussyfkzwincofzfkpijmsekofeizotnqchvgllmvmsptqqrryodudxtns\",c2=\"gbktdxiyhtcyabdwusjotcpfcyjkhejmztqkqchngrygrhxhdqhqicpfgtjmforxsfnqhhiqicphpjyuqyeximildnqsledxeulvlgkizpmvyixjueiqkaeldfiovsycazzafrukyabxrcpdxwgglfazjwztrjwinwjnbcwzyxwidkxhjqudnbvemeoawptqwmljfraonjagujeindnxmluxspoudowoxmhiignoewukyokbjbraszjjensivupwzlcizqwstksjxaxadmxeowgyomgacadhvmageboakqvrgwkerzgeukpbnfzxsqezybseggdsxrnfnewfmfoxarffcjmqywcorhuroaqjydgdgrvfaewgmuxosbrufejhllqccmpxqwpmdkkcibttkbxjmkwjfblawheaeprhzcirqvhguxxqkhvqylapygexkgpwxhmteuzpkfnaughjirionsiwijfhpoucpoxhnkopqslwfgzlvirzhiuqkruwubckmnxumkbnrdnnukssafbcebbhcjdmbcmmlkeaitvjrvfksyapfoipytswfolzpmabsbeyuzoruthachwmsegydxhnmnkdidejmbkxbsvteleqjepaxqdyxuiqnkfrknuqgybmmvmkyrcakpxubgwyvrtzvsevidmabieqfymoynzjjyywfqghkcliyzasleomfjqrrlgcgqegawhphsafasdjwrkqneugafrefqhyadpfnqnojvfqpmcoxavjzmnffpcewqyfjlsfuvflemmrxoddfbqprpwvujdxjoebzgsymzvsbclynhasoyuaypsurlqtacoulhjfzbbarulvvrutuhrbmdjztqubooccojgunlwwdyprkluiepsfnhrsfvfozyljimkjcewubwhjomnurvolstadcllfnascxzocwaxnmknqifysapbvnxquahdywzzowpfxzrpgccawmctajzwhtaqrwkwijyiussyinntujacpaswozwqjdrskcelljbwcdrmnqxbxrpvnbikwjhyesounmzsxnhwnrgnvplcdijxoojfzstospkdisijkueaozwrgxyvbbaakycrnjzytmtalinmhnzgpcjxwauicyyxbiorkaarduvmoomvosnhekswdkzeqdxaiogucqgnumbceuedzcjruemmfbtprsuwovvrtukslmrvuzqmuzwwvyshegnjonabvjqgxccjyvxpuzzivdstpxbrjelfomtbfxlshfchhkaauwdivewrbpcebjfnnjattfzirfwkjhotjjmyretffruhevpdpvptbhinbemqucewwrpdgcecwyphkyclteijeejuetoqkevzotovydvjdlzazouwwsdbmbfemvwmczepdybryaaegxctwroegrslmazoyqwirgrtrukpdfvaccnntktwnbifuvyvqmkzxvpizgupihuxhuuzticuettmtwbyxplrrikzwhdrhrdsxqoithfjcsvattazdxbsfzalshaugxdblpphgihebsnjgxsgwjwzokvebrffcqaxujvodtrhvjgfchpdjliyilxzoxyykvcnzteopkgbaqfjblicmpnooinkkctnvyvtvhjdrvyzdlczfrhbkxkprvlvmysltaehphbddcdrsiirkbsaokuvuvupbkrbzmtowsirkkiqbtpftrnggkzztfcbvuufwktplnyisdhorxtbhfzoonansxktktejqulorjqvtuizehhnnptcrnqbrzrtwiqkfvsarsnjwsbaofxggmstebenchnhyhwysgbvljpsfmofrmfqyxzluodrxapxtzyrwjjvxgxwqbkgontxnvpnatlraevbyzegdoybnxaxowweehyszoetpalfztwfpngyqiyfybulllicmzdtekpgyekhqfgontemyfhlkkqugjgeznydroqwfdxkimsakuicxoxyzuksqaqmajivttvrpgarsxqmjntvkozuyfcftjeysrjvbpzhtyazeflayjbgogpajzmptodqlkxubwdlpjsnkrwazvcdtifahvnglztfewnkgzrruhyjgbbqtdelpaxdpjjfmywegkzramunnngcdbyrzgruwffuxmtdwffcdymawvmdisttmdkhknnsfceqpzfyuaxltnggajcklephspnhxakwcsyzzzvwftuwkcqrijbbqvzcyefettcrlqfqacicoprvuvxeqecbgprfuqyincldclygvjfzbwsqdrqncyfzzfnunevnksiqeplmqbsibhmmgbrkpzfdimuofytoscprgokemdczkascacsyygxuxgqnpiwvrzzqreidzlvxvhotzdywfrrldxlumvzgvvnjbmadzflvdnbzsgmrkppyduzvlkvnefybimiadjdumfokwvvnmqyjvdejuhnpngsaggoqdwsrlmwpppvemuxxpdcfrriuwjxmfzryemksacnnbmuyacmnwxyjswqlnyixqldrbshavvrripyfxlrdjkqcnbbojrtkbgvesgujlooksacqhfdbamgetdplovnhljjqmjahegsagdfkpmjxravsmoyhchwejjeztnkixyrgxxswuthwsnqgumuvslqfhlxtuzawchvvshwxziyepyqtwvwevgljorevgczohpaeyaeckepxyvuwgaxppcyaqnrjlwqzlngpydoqvbyrgrwnpnwwnnyutsfioumwdnmhyajcndbhxmieioqnebweycevjvaqbnenxhxbvforbjokucjlyhecrhgosfkpwhowjuqnacirzyjsnjtghtcddupwdhlyadgmtoxlcxxsvlbfljmbwgjiallhqnmajwqueasywvhdtwhcrqeqmzsdnxpzjzjsrbnmlssmxeaaalbghrfamooumrlhjgiabpcanzlsxcxkzdgvujpttteynfumwnmpyeesoqtfyicvmrjvzkuusdlkevmepbivpxhjwnmsumnatvsdfjgiktkjyhwmsexhrxxspedhblfffkhrwbqwxxrzkwpkvdqevpqxlnobztmhlecywidjthkutjxnjvgdxfdaqyvqzkcxbrfyapkvmirijqejqeplqutogohvhuezkqnykusrkjmrfcqgvbnjhiifygztnxqwytdccgivbwqmcpcznxamdvycgjvujkndtqyvtmtluvohbaesurssnbwkfzbhbvlbpbvvpdscouskdzscofsdcjzrxiyibiedwjkulvfxvbllsksbuhcpypsuiufvfqglwlhbptewsmjaufmomdzqfyhgrdlaeugtziyfddvasrcgtkmfixeeeinnfpwdxocmxmyrhywiraisgalvgbvcoqxmzrdgcrhjkofeoggtvnjjgcuufgnzzvcqbnquhdybsbpeinisiunegdwrowcbtjhgwsouozqxzzwgllzbouwggocywzctxfwpcmdvzwrkznjaptgqfgxbsbohbxetazinumeohfyundmkpwigovlhodunwczozpvfqcqvimiagfyqoqsjzwtjuxpclttlipgwguhjbtqkudwvpbyspmxbakrumxfhjgpbkgchnddkphqdjtjaevjppiunemhrbkdvhnqiosgtkzmgcbxsbbsfqkquttvbvlbzuwsxyynpqicqmbzgbdighddklpaoxfheffsjfdcecdtjseyxtuzslkazqcbaexwvvpnggshwirgnmkomhreobzrphlumzeedsrhkrlbmzmooxkxumhissxywzhvymtfonhjkumsuswbewfllwbtsqhnyyhwsftsveqxttufnnfpjatlgmplhnyazzawcpbicdcbcgcljhivcjhueodayzhucsxkolwtnkxrlwqecdrgdxxlevacapfajtpmkznatwsdnhqheppevoxllwtqrnuqsnwxwpggjwnqwymgptmnaytczgnodjnpywxzzbcdembzwghrusvzsljnrucxmenylqaigquexutijtyxonmubkumyufdgsnrehofhuctdhqsivymlndqmwvgnesclizqeeefakpkzoimqdkzsbgweixhxxkjskupobzjbgucynyslqrklnyiqckymtcpfxahaudxhhtqrwnnqmwytgcxcomavrlypydodumwshrnrqinttrtnjzkqklifgpqdypddnbtjcmddgubvxhiuvcmcvavzomgprqlaqylyldosabcxijleklcrlrfhlikegkjtrocvjwqdgfkazqyjyjxxytwzsmhdvllfwfateyxqcodmxtmwnsqovilydnschxvrlifvzkacjrsilbjpbijvloqugatmrfkagcekyhesdmkqhhhfvaebpqdsmambainjyikirvbdxhsvazkvgnhomtfuzyucvqwkokbvmrrmutkcafeyfaqtnrvoumixsukutnvpwtqbyqpuhfbrtzkncfiiazcuhnoqzfiolfztacpvfmpbzpxtamirdhhlujmtilanjnekgjgqeikxrksksakapsjqtmjgvnhkjlepqjykdrgykumnaffvygsfrczhyqflbrprlzhowbomkqbaspqwjqcddryedcdlylguzehqazzuwidmudyrxcwnwnmxwelpjwfpwgjjfnqjunmaxjvkkizcbogxfcykrvuflwcoxfyjhiibuvxjuwwlxpmexdtsuixnrkfiwokxyqpgxllhlqiixsubitvzabebevgjhxuimagjbmptueteinygkjuxtphuoelhratsvjlrrxkrntnzvjtnmllljvrqkfnwprhorjmbddctcldwizqishugevemzthppahnrjhmjdzouqkehpeaiowatxndeaxiwoglhmndlbtlueinzfxajxxrreofsdpmcmlkpxkdcghsqquqqbnknwiownrstliohvjfqebogbjxzwdlgqyeqigrunelgyjbheuorbqkqtjcmikowuhtroljbzuohuohkyedpuvzjfhvzabeyxvnzkcfznjwdhyufofayoaeixfpcsvdzrpjolmcuyvjyjiwijxdgosvbmdjevdsoomjnubaoezcwcdimbbswxocyhigoxxciutvakbqmjrikgalxoggvetffzlxrpgahiaxujjibtexexurnoykmlorvtmupdqsbuzmhtosiyvxzzklgdvdodytpokeruozszybpjtkkjmcrzqfxanscdxjfyocviywmllgyctdylvittmunxidmyligvsalttzosvqoqwnemxurywwabtrmbbdkpazayqjtgvgfinfocepqidlwivfsufffknabzjekeefowowwakghguphpdetpwkwhnrspciybwrmektwwyyrdqfffzncusdfvaueoynvywvuwyubtaqzwprtcsklngyeqwpskyhahgpjrkygzevcsmjoodokineycadbeqpkqynjaevvxbsvnvpsspwgbotahlwvthaaisbchydtkiqqtpvkzwafdgjnxhcvsgekcabgejasoczoljyhfegyurirjggckoxuvlyjapmgsxltfhtaybhitcuknzdntzilpnyoowscrkivqzrdxhijpcwvulgwxolobendvtkrwdeywgmigbyvjgfmtwvabvrlqygyceawwgbktnrkodkuwiyermlxjlmowbjgjssodtwxiftoqfglfeyruvxqwwpmvfdyhqmluqlryfsvsdjomosaersveqtsbwnnljrlqpgbulkghhfoksxvemneujjdiszejmzzvwuuvbsoflwfiquonkhtwtifybhllhsgencyocoaoqdgeyffuwkmfixreakanytytqctdulbptidsjlqakpbhfpiypkzwolpvlyjxiwgzpaavbdasadubwuvinxnpvdsjzssotrmzvxmnnifausqjqvimsrykasgbkxvlixwqicvboxfrwkhxyzeyzcqwngiojucwrifyegtzcdjpphxjgkfjjvspphydffwaszaxntvuhubtzorsqmukttbqdiowvuwrlttlhnnodqablvjxevcegmqozxnktqqpvapmfxjswzgojfkbswtxqyjaaozmdyuthlhdxxtodzgfmkzhplinppaacumbpmwhujyqozmomnzcvjznuqexphqbyspxkmkyflwbpzanupzwmtwmewvrjzigkdoseeimtchqlzuslymvkqyzkvukjoyybqrvugblteyejozaiierunlwwwydtmeeggdxzugifvpwqchljcennboypxfxxcsllleszolngabpqcbwitrzwwskhguheeaojynxojxmvcwpcbxhpdxgaqpotilndchklvwjorelviuvzcmtomfpexbhbdzdoaxrennltjijfvcsbatuuudfbpxhbossijadggzurudirohwaekbudsbwqdqgmehltmmsdiiisjtnszpxtjzqfhlxoigqxrrtnjkbdvohqncmxritaiujvqpwomlzwllrrslowfnnbgmlccvtctsavavdgiuukmtkdpsmengisfkkiudxlrgpgeyknokeokgfwoinwxsywhphiwoafncgfudulmntwmgrkbcajfhftqkbxoskmeieadearjxlcvcysgclntgdxfzunbeanmtoqzofibzldefzzcgptwlsrfaborgiogbkqhdkujdzxglthwxvbpgcqcqjhdzhwpzepcsxscfqnucuzsfvnxmmbftzheclwotsbdltfaqbgwlgsvgwuoxbkmnpkusvgsjspqvtcoatlulygejttbqobcxavznncxgvqedqsytdxoptekfxzvxznghhxzkwlfyeuueoaykhqqocoryawouwcsyjpgonetqzkkuuosngpkrmcbovgifzzunyndjgugudxsgwutmzzygpbnrtssndrjurstewmkbcrrqznrgvanrogjksfqjksseqtuvjiejfwhviwthforlrvnbzznptdqvbqwraqtfhvbwdrdhoriheyjwicnarnojglmibvgzgxjmfpvwnpsisevmqhbatydjcbroknfdnnaxwfyugylhkeaieahvztldzvkaxgsoilegvsymdyaedryubdcryjbdshbqqchmpzfkmnjqtkzxuhnsvjszjosacmpswarfeolplhagcjippynbitkshlfhbospcysbeapflfsboqebxgucyrdqorkgifvbjtoivlyskzghhesmnxgyqnjpnivacszmrunydymoahxlqksxxqxokbywvxztetcbfsenpdwhnwcipdcaqrjllnofhnsppltqdkhbbqkyqtctcyogqqhnwvbpczlyzwlzkoqabfhhyewhehngsmmuwnjpkqghxbchagpmklenlpsdsobozanbjbmkimtzcwfmvigvscgpqvolyeanqsllectfsezmdrdcfwfgewnqsgjrvxpwlieojmwcwpudxvrnzelenzzqzdvljtouwspqxwrfwhojmvihuqazugwjoxmkprysrjjxoeihgvdohuppnweujlbuloleidasqjcumyylkrmvopxsayhkabbapawseytzaasbeathakuynlnppjikdmsrsxsmsjlsnkzjrwxmipssryivnlysrbydvxarryfqtmwwocctdkhbstcmzvnyvbdddpxmkaaozohciqhudtlqmrtladugjnjnqqrwqpbvieamavmrofyfpqgjiapzbibhdlaltijoyzfssyvcvuuwxabsszfrnrjrkyujtullavdlrwfgusajwordjrfprzzypmubvyzqwdtkcovyoyvikbyofcufxtauzaczjoovcuanqjnkhuvkhxepwebqvpwpixotkxvscmluplqrscyvodoqqizfoxmuvvtyrazsokdlliryyseqksmdowblnhrendcghvstojiysaithzshwnwqhwyczycyclkuqlzlgtxlroigdtdtjjcdqgthnobwkdgrctypxloodprvucnjccfaxrmhrizlnhmvgpjqphphlmokocagxtkzoyujaosgegnaqmosmlzigggwnplxfinccydwrlubnfcbuktuvzqzigzrikaggwybrceobeipldcqoypkieakmrtfgoxwaductesiumadblvtmxvwbyadsundkldbzinegbbumqsglqxzyvlzpustistittgpmmdplroqyhlrgyqiuttcjhuvllryhddujgmegmllykicujijrpkhjswhdpfnuqeimuchoenbhrkjbudmgmoecdtpjuzmgufrifrepetvhgkpskyoqdxzldghanzzeyvdoiwtxjwyiiitwjlnqnemwawtvryrbsiofnjnhvxqeimygfqsgzdyfzkfygqmjogjdkzlygrerzrzdhlzjvpkjehzzhpknmmkchwaxivdtlnuqhcyewisxnbfxwieliaopxerlispaduvdwsbipaehwbhhmlelcjhtvjljzsubusmuaqltcgbzztyuvkhfchueietozbdsgfblcxqbcssivkmbpvhpblrhbwwrmgspfxzivsyhubfrggasfvllymqiziscoohfdykcackgufyyyvqmiifmqudgzpxwghdeqtzzpecjgqdvuazegszacameezlrwazcfiafiqebsqiieolazctwplazyqsadrrqdmygxeoizzrpzamkuqjcoonwaxqpyqbqvoxyweptoruqootamklssmornmzeywgkppmppxujtnaddvswlojhgnxcbejrveltbkztgjffdeeeimmnyompgoowwtpefdgxvgahuhxoyydoheafxxisyozamuplzjovgbfmmtpbzukjinrtgqfqmbolvcqamgntdqhnsruppqywqacmytzotbkralagxzmewrklgjvvsaymhgaqhocdpilhqfdydyyalenricsweystosryxehtpkxyvflhdqtkyoxpaigbkrlpfdhqowfopjwnhepwknkmrrvfgnfjzfqdrzduulteuzajexildqdfntpqmghdrqanaduukvunfwyylxhfryrvyevdrkjeemuhiwcibkrozmwztfhnconwxmuvmeddjubbnsbvfzdvvjwpuablrddlrluhuavupbvgcpdrmdmyxhfomdnvlejzfvkscagxksahkqoxdtnuowlwbjkshhvpltppcmxuojcydnbmtncnfqeghgigldzmgzupufwbxlehdazyidwbfdsqkyduazvlkwwpiceswgndilkkteggbwqltistitqcfoaaxjmwgqhurysxydcrehdcwjrogqjkflelakffwjyxfrzektjepbgoravydgbgrpilkreppmdkwhteolfiwzpubwqfjmswnoyfgzktzdcfvugglxrpactfpjhwokdcqadfjnfcrkhzkkcyusgxuhaakzvponmccjjdjpegzsezakjscbhzfloxrxlznzogzawlrermoqnnmyxmqhwgcigqdavizcsiwmvdmrowiltbrlcaunwwypftpsctprgxeswfuppwmzuwypsanajafvilpyhgpvfnptxcgowyvbiycbusgxwurgxzxpjtytflxrfncwroybadfvpqjexxuhlsldaxcrwmkhljrdlduifzhilkdvdfguyygsoptfvyjoczbcnexuqacqbfxpuhixicoyfomfzcykifksrhqkcsxqguftsciikixsgkhjkfgyppjfafdcyzcbguhaqtunqadtqnptczmdqsfpccngyxoxjldqekslchvbqunjvqsqmzuyqllcjitedfjilatiwrabvowjqfpwdhuoeuprwzojzdqnlogooauexucthykvxcijqfgrbyjgoefnfscvvffomkvzellewgvyzbooodgbctfwdliylikxxzkxsxeehdvvdnvwjdykbiktdsusbfxbwglmjoumifnccarbmfinhtponkjxdvnejuzpnkkkiwidbbqyctobgccxvoecluamqhcdcwnsfcflzgozzhbigzzqoakyoczwrfnfrteayiumxuqugojlfvlenglofhqepfnzaxvvhcwkmuyhphevaphskyzrrujjnpvukozybauovsqnikhpidacpqpkrwjtsxiridtgebgmudphncdaufagzcbzvqnhqiueamhdhhfcyxlbesnaqbjzjgiufxpycfpgmnhqepbbyxnnyldtyictqgqdggrazhaanuosgcqqilnyqpmlhwwywabeoradrggsmhsgmfgcakrxsysoatxoagfedftrdwowvhzekvakflhyuwovomiithqjaythpfyacdpujlaroqstboqkmxqjxcjoreltkcmebqrhkvgltiwzqgzqejzyhdansagxtpxoqmrjgdqhqqogvcahqdvltifwrqnfpgnapqgmjlmgyoonghjcsaevtrowlypvgmkmfkhpocufpuvqkufxgleeyhzvpvfhvdwpdajtgplagmxsocjsxdpwygtigvuywavzihujwednsfzqafictknjivdhhhqiaktrjaeyyhipepcgcwmnsdgvukbctrdwwrbnhcrsnavksrkshefhzxkgzwoqbpvkwduebjxvpwfnufdvtogqsjpfwzutwlmwfxxgxdyiuwutwcpiomszwyodfttmedmojhwbfqrglcsznsigmwwxldrtlaertjcxdkmgabtjhqkniygughprqsgpkvkihstencpxfwtbzoncicycykfupewqluqhpwsdkkwlkkdbjfdhhctzdggxoxzeoetxulhqldjhgnvgmkaiwfvyhpzblzwrotlukarptvemurteqccygzqwanxldploevkvcvguebyoaqzsnfzkcuncwmsjwnringmkfdvkwiomfhorooxtainftcarojfgtsxydljtdvodndjvultedottiymcjkmckewmyefralfybkduxelvhfitqogcmkphostnlirofutkxxoyfszslfjwsdwhbtgeigyxwoowudwoqddmjgrmoaqcsrfjeevjqvmpzhfyhgmeqeqvzbuktmvvejcnhulwijpomtmzesenebjwghajcrpbalfcdamjaymjbhbolemcdatvdfwjhvowfcnnchchnsnzxrakzmbirmedutuecbhtawnzohdtreikwwcjbnrpijcteprpnvemyyfejkuntgtbmjglubdciggtponyfsmkoltpbhwppmushyjoqbiqqtxaitydioxaabnysiskegrvodubrtqsziufxirlfkzcqglwnsedrajknrdqthyqhrreichephybmhxjsmlickwumijnxqmakdzdihxeyrywoamwammpjigcfllkjqiawradzeidrgtbcegntiuqzfnhyegweefodjrffqsptpvevrkoahbdbufudnryplzxchtxlzturwozzselulupdbawirsfqemspwbhxlfzvvpeausfmgvfszsemonsoiwtvjcmiabcemymwxonqikpvftvphjrbojyabaujyggycgrvbqgvtcfnmvkcllenkmnnvsijitjvpgkhsuxonahpohyhxagalhqrfktefhweothwfigrymrfycvurblbxcovnulmoyaumbzzcrxodnatwloohhhqdhmbejqtdlsheymcnhbhihsxywijtptzqkphxnxijvoxqmqjcrednrpsovejsfppbseyvepmqmzuayjitjxnpbeixqtxkbyojgbkcoyyhatvylwnmkoulzrkprnfeaanplfikygbzaqfvynqiwbnktushixejwnqoafjzsvsxvbtfhbqnulpgibsmbttavotetdvoitrygmgatkmcrlprmrbmuttwjymngfzeiqyurjaqerlgfebbyxvhfklpvcfaapiduybsfrmqtzhcbhorpicoskvcenejqhqbvemaoqpisiqsqnzrwjhnwecrmbnkjvhymkzlsfjgzpbdtxcunkzfikluelrdmgsnudvsdrzotqxfqedfxnupvzugyqdndqsxzigkrzygflvcmjkgcxpaqmjcheirjdhkdqgjbfxxoeunnzmfolmdglfxmtamxllsjiqcwdsvpmrqxcwkigaxyikwqovqzjrndikrnbqkrsacpropabacutdzdgiiulskkdncyejuhkimwzknjcrwsekvlllfvzwwfmcufiuwlqckwvvxsbgwljtfmmcdhsuoruclebftsednfeywqvscyenwsnhngfdiblncskochybucvlubhwrblwrxeztlvqypzcijlpdowpdeajmqacbnndsbozkbjxymycbqsqcvvalusqqfuckkshaivpwjammevufcipcqqapazczhrnanlxehfwapdqtvvpumpxffoinnrardkjwslkblcwqmkqbbvpdgukqkqxabjubaotuzihmubotrycgscdkxqavdqaniwcreznkkstemywdryvfyzdrepxujcqpuazsbtpylqwtbxlrdrzprovhefcutrlwakwdncbbprmmmihchjpqqckcwudgfbnuoysfpkjxsmekfgcmjsrpilhdjgpnjfbhvpcmuyrgnufpiilpaehfmhiarakvhmtgjgvhxlhzzwzbldastgxepywscoawrnktkcridestqxlvmcgjkhjepstfctdzavfrhthmqatnjhkgwphymojuwxvrtzytengjnzkqcbjyklgzceaybwigpabfdjnomddwrwpqkxqrqtdzgcperndtfucshdrjccbeqrszbiqkqxpyjlarvyuozvsalzazlmkealqdakcbdkxjrvsrulykhcsurbjznufdnvxgafuvnljqyvzwgosbienmpsydbakqleautgapvpzrxfrkzigboxndrbusaeacxlhrgpjcxompwobmrfjwacpbadyfdudvxyfobvcdykaghxtofcvfptvutmdyorynyrsvakkkzyzdmdpikjsqviuskpttuezzvumrwaiyitfqevjxyovewlvqbtyyylbumazzfexbexljasfadeuwftznkjldjbildslxauisimcuarhoiyufdcrhgnjtuiaxhezehqvizgvtvzkyxmddxmeazxuadrschkfebbsnohnzhgdxuhnhytqlmeymmukpmrxqubmfdouhlanemtqxihqgxgwmywlrdlpasmvlecruinomrgjicnjcwhmdrneyddxqocnpoaqpdehoatssxuutojzwvgtmgdqewuuoeqdxefznohmumllnszffopxrelmadgkfixpvzhdlmvmehwtxezdqagoqmkjpzpmnlmosmbhgcakchmfqawcinwdtimxlikxcswbsddeyzyeuwspnwnjwsktvgkrfzwcbeuhkzhigjegchyyvtynltamfdowzzrsnlzkciidemkqhuwmfielywgvqftwyhwptgajusjwzbnwcdxsfjnbvryhkbwjnucnazaziwhybrsyvfzgbstqsldjjnjsncyhefpwfsviozwfovsffrwydjdmbyyshkspmgqjohsziorvjhavziblqnakebmcewqtsmapimbrxxkrjzrdommdjepmatibehxqfefkkvsflsxmmjrpzxefwnivmdxxckpwcznnqpihfalttvvxetyjhpglvatjrcvrcppnzcflasedekycezwhuobcqolwucksewxrlzrzlnwmbgyrihthtoqmvvmyemvrvhzsmtjsaxuqblopbljdqcyvagmdxlvfhaqjznrrmunyenanuoaonulihoqlcdksuufelnpppwnnhglxojejltxqzegicxehtcwlyoyeyclsikswclkftxymwcmremslaiqvfivzvojbxiwwlvxulhmuovblllybtggsdzppogoggcedszbtganflldfmujjpagtzfpnzafwrohwlgemfitgqptdzebugkcaicvujngtanekcjpgdfpyyeoputzmpaawrmkqbdqcfkaidxkdacvggutkmpgzosxqflkuzekimbxkcwhiucnwhlxmoulwotvbedutlnffvkdcodqptqlxpbwfzubgkcidrrfwmlhkhphipuvahhzeofdycjcniuttcyvfqruvsskjvwwstofoztwgrsbhmbwvuwufxcvbtkcnhjirbfnkvwfadxaorvrbwkhyohjszzeepbeooqrggujrnzbaufwekttvhlpgiteazpolhlhfxvxbaqzymzluulaxwbypbrornvvtojlobdnchebpnzeruixchhmsdjghhtjuklbxefddpjoxaxurxtvsprflewwdctybnoehsptmnophswidlxrkedtyxexsymbwnaystyzvrfxngplhijdkrudbrsivrcblxxzvselbsxzgpcudwpvqbjgbwkhzoembhswweocsfybvtuapmbhahlboyurnduztbusvwohusuqgqfjboqcxynnrwhqjakvuaoqkilkrrrgcaiatcuixhhahseelfyynfqjysbtxmltlpnawqelgzhtpoilbpypbkdbitkpluroyebqdrlzttsvadwqfkzmmlufkkcsidiyumicadnkqqwbpmftzamgmnhlrcrakjdhwafmasphbvuauqtoafhnskqdxguverbqhmhxsmegjgktqxudowzzyfmbqpxkzhrfvtmyqkssohwuymhcxkvilpkwoizmbsxtjywieqdssyjhhqwyfyxcnreyslelviylnvnlqytiwbqjbzatjvzfcjtwbvjzqvkkgdnjgegzjzdxxrwpqoxnymbslggzgbynecfsdfxbblksuootiygyykrgeesnmtaqgntigxmniyqjrivtgdfeqslesuaouewxjvaipowsapfefhpwyaerlfffnaasjuzvtixozeusbdcnpdzuesihoscbxhxxdewtmbdoqfwwmopsfrtzoijesrbxkyilzosakjtawazbytonefsjsfqimzvccwifgbdbsdvruyovoorepilrpsyilmzumaxazmzluqojtlqwgovaasiwuzoqkaxjyloipraxmjczhyajqicgganlzfepxrvmtnvuwzvoilgeuhliovpkntmmqplrqmesswsuflgffvsbviqinyrqbpfppskafusdxjjppzrrmzqmjvqkzbjxmewbvgrdlnzeoahumrhgekhcvrqosmtpwnfitlncplcolgxntugqofqddvtidzrlilcnyqxztepwwrttgprogocskiqkcwrhxbmyvghegndvkmcxxsrulumzemjfnlwljnzedhmheqlzsnrkhshztvypzkdbrqcsangkpmowlcfkkpzifalmmtexcgvhgiszstssshgbwjxgiepbmoohydhgvcxmeiyodiykfwqkkaubxmwlnclqpzpdouxwcfwcwrlunxhfpvrjvrugfjkmbvsyxfdqksvcsuhzzvnsjakwmutqlqcmzbzydxrvqwdtvohfdisdcsmiwutcsdvureydnplozpexwzfwhwlttdrgerbbirwpfrzpfsyytvminrllmyuphkoocbdgzxsnykantqjjcwywvnarhxtvdvharsgwyovcmttolujvxpwaidlwqtmvrhsmsmnvfutxohhralolautjgbluawhosbfkqrcbtclzomdrstzttetmbbiqvzetpyayjcvwjodkjcjdciubkvbbsycwxhebvxqigvktizeczjunvahzflturkmxnjnpllwhbyrtyjdvimwsehggpjnzzfhgiaswnhdrnhexibsraczwlluvaskhblqcfbuuyvwkrmcsjbcbwsbvkadcyccdzkqismnhsimirljyfsxeujsnuucgyelxnthwgrmicpulkuuyjiyeluclpizhkmzwnxkrvlijdcuoottlejvfvyhakmakkfwnsyuuheumqwcwqqcmszzskwclpirxhzaiwlpkxxfbmzalbntywytrgjfrfwjcngldhswclqsjklikordnsnupkzjdqfmjnyqpayrvzdnwfhqzumulaiwepjsquhmjnsmfpaiwmjsblntoiyqsvfessdausjqpyrjmbvqncowmfhqgmgkihfoeegammurbvlgutbeetykxpdlliekpvpzkrulyfrrxzwvwdaekzsibiupqxlbaunyzvqgtifephsqgkhfigfehrphxurydzokdvegitu\",c3=\"vxkcjddsuawtepxvxchtwskwqqrzennujhnrxpmleuoxfubjnzabiyveoihsgcbhiqbmjpokzfgmkmjeybeyozwukyynalxvgapnamoyqxrxpdlhiznbdzffeaulesrlmiekrawnpoyshayrrrgewfduxvanlrswkbnkbszccgxenvexylholxyvzaxgqhagdlnrothoyjmtjskgfbarbuvxsfvdinfndslzixkxkbutsoosptobakowrddzdhpqmbvqbloqunxpxvouiqaovcvrsnicluesufmzfnxceanmqaqvssakzkicxpzjwkzqmxkrqikirudmburkapifwfanwaiokpywaqtwmwkqdzmpbxrhutcitbpnukiklzkpzvgxwggvefosqcaakfwiganvlmdksshduidasgpocjeiobivplbjgmmyvxiilzrwhrqaiqvqbctlbnfkzysgtcqbzlbaufpohpwnsutpzlqlwveszxzrhtsqjueimkxwdtuqkjnguztdjxqnphxhodhskmmpbltwmpsjkgckllxwualstldwowvsadfszirrgdpkqckwgtqllznxiikohmtbhyqjmkkevpujfdjwdbhqqdppbbgesdnydtgrljdzxjvkurdjadeqsbpwkgbwmdtbpxbxmufjlesditdqvrmsujwzgjrvlthxpcnrmntkcrxgkjdistlatctmotlccvvsnkdtrzmvhgaiemefmsfjmvnugwwgcgttschxrvfldrjgccqdwadlwqeyjmpqqkmtxoxpbuxkshybgkbbhbuivpkgbzxyxyndqnxvlmbzxcotvstpjasvbcpxxtbownvgubdahqvljzubbbzwbxnavzhdocqxexrataojmgneaijuduroomfspimmflgwebprtjafcduirtvwobmzchrsavexxcqjnjzcmffsghyiqvqxtjcupawpnxvugrutjwihnedwvhyinyplvdakcirfgshorpukohazfaamacxpweiozjqvmbnuqznizwoxgpbqdexngztmhtsqrkwshutycisvaxzdklyjwbchhgnoppcrhkzxbtfvloxqhbzmyybxnrhdihtuxttajsyaaawlihinggcjvnvqscpcxzwynnbtcdqcuckdpndgzzaszwfnfritafwdjezhduewbtasjyvpjvltyqplxgzaqhedhyjkhsyofnzuaiqfpdahtgvtdopsegehvcmnhjytnbenuzbdpjlvvjrncfijkyanjmvgehmkdatyrmeszhkifcciwiftuqiosxkvijulcljkqoxkknqfgvzdmjtwkvcvhdzrzkbbygyvmkhhwyfvfztcminkdvetbikiumnslkdtuxrmljwbuadareqofjfkwiusxvsugfuqvwegjqjdswcipjkmwdrqhphuxahhfowmbyxdmldfehbpbbdgyuxqfrrbleevczrgqrnvxdzthuhcbcqokenixaldwyzmphvztflzfcjryvkojymlpvhyjwbpzjdvdkyrayyegidmlpvzahldjzdcfsdjkuigpetdyfujzbhkdrjauujpteotkpmjiunhripbwjkkuvncsxwjfzitmlklrhpmdguknivyxkgaxwylzbpqwsfnwdeyzhqvyovtaffwlamstodjybnbxzjpxmmdwxdmdveviqiymgrxwowsarcblphbjqxeunubzugxlsbcnipleumluhnkbtipmmojthzuwmttehasfrwmwwgxhlodyezvzhivflnjxfxijrsjvpjgmflqluwrtteeprceakeenhvwswoxpxdtveenwxdesymdofphsitykfdvahmgwevwjemfczvxsbjgpbymvikxndigoabcysnaoyvmspmtqeqsuevmvdmbronpfatcvdgmqfewzyjldebvhokgftvwgkplofgokofvdhvjkrmcjrgpaxexovatfngmxyhwxaeykjwloywdbjfyacawchfrmlzpqwtkayrkefqesionvnvdjloherriulnewgxsnbxijscoagslpsqqjymmakutogzsqnadcfbxlflaexcevswygpuvlhviahkkfxqaigsusmjbjsxailxavobljjaggacntcvxqltybhvdlxjtywfdszkltxpyklkcpjgpzqoxibjgcriastqktgzypvmuhmhjcgpmyimuajpdtpjnadnicmqmptcgancejqwpqusiswzlgpljiaytrhlygysyysmbkgzqvzugenreqdlwsjigzgzfqbfsrwhmidjuskcleyrampuxxlvpmmvjttxsesguyuyeptaqkopndlhifeawacdkdnkcytvqjsfllztmetciiwiwvwvcfaauvhobphjgykwxirbfvcnzybjaosekztvvsfoizcdacffruglgehlpbdakagvinbnqjjkkqnallyllcdfnchdtibkhfhdzpxrpyinxzvxukjqzszchdadjodqcsqbpwrrzofggrjxoukruiyticstltwyrepqnggocnwvronxzsmacikdzhmfwoejvpqcallhtbkclzihsfptttmnyozwziniblwaypbmfwwvhodeqabfaqksutyhhyvghwrdgmvenrfxebswqkdxzlipxrrjlrvkzmzcbwbwbpmppuygtuddnvtpwvxzebuhcfqnytbdllnvbbsjsvavlnqregweqnpsmxrzluepfkhqmrnslbtvnetmezclcjcbcjjowyccwdqrnsczmaqyyikhcxcrpdxzfeojplatokdbebyqfapvrqafirtlujdyweuxerrnqhzsxlkfditeqfvsmgsayudxxrpgncsaaboslauuwwiaudlirjzzzqgxdefuuhoauyqxiyoeqltafenkilqinmvnobaswvumelcnqebwdzcsamahxpnjejiehqhkahqiijqhidizhlbcvesmmqcolscsgjxkugytveczeqkuyzqeieyfqkxrxsrgetollnjohzazbpetkbpptkemvszcgbhofiprsgeenxuplbkwcvrwebbcmvnsoflbrjhuebbotbwxqlbfaccqjeglvysuwlmifnrqhydcyotwbxyhobgvcmtxcbbddahdxjqyzvnxrkjemcrpmdpbbsiyptxwhfuqvgnnazfkwawfxlwmowzvbrkuexzaqciflhmitxdolplxuwirertdqpomnnuyihvdshrgagnitzjubzeztnlivavdcxutmhoorhkvgburehikmjnjanogepcmpydipkbuioluwkbimktrfdqlnqdaaffuivujwdbpnabujflbzktlnygvsybmjedtcpnhcsjaegryryhumoagqbjwjaswccngzecdprtdaqnrwfrgfhbskqwrackdbiwowuojibenlnygzacobehavuvyswmginattxunmyabciagzblprwmzmsoqfekfjeniacqkfgdgbmvifpzjudlwufomuoqghukxitmxedarciefmzusymbxhxlfwkkhvkdmymtldkjyrbkytmqackabcojvztpoaspqhhnobqybwbxauywldgbugzycpikulerotyfdmcsxdmdmuiujsxpnppendgoyqpzyfoxjrpuhrphjlbussdavuzibdrjzxpieqxvpzlzmwosxehdhsvmbgejlxhsfspnvplstzkepsuocdccvgohxutkagtiumagzvtycsmxtezhpcvfgjwgxdiznntcykukzabwypejptvmbqkwxueyrmkxsbccjglfniejuomvbxyrajkmvphrixvovnhowpsfeaqgetvxulqdnbkaeodzaktdcvzxsedpokfttkiizuvcsrovoefbuivrudyxeltbfcgpvtllgqezqtxxwbnehlrarkfzylempzrffggibtfjoesxdpvdnuygszsgdtspclhtcmxptcsxcvcczvgkmcbsxxxjvphyliyavjobhgfkrvdapbtccjwhisqcszbtkxhidgkutseylkelkjbohdnttjjrvbwlnvydlztonqfffnvnltlkhzammbmigojosqebauzjshzlmvrjlounkuieepaccbgkvsgmhwkkxhrjrpxvaicybwawqepppmtksbbvmftbxzygrjxckllfczxhwvffbyrfofcyqccmwemgmgtotelgfceanpbqgtvdfdxllwaopnywreezllawwtyndscvlnazpdtpsjbedsdxxctoknhyovyukjjbciqfyhmqrpwndsotzndgqljulgvmsvpjeveuuugbfkuzftmrrplatnqedxfugiqiygpkkwsezgkkysazggzxjvaqevvmrfhexkwnszfdiqcqtfyhyuhngbxikejqdumrixgrhvzpojekglendurvsypgvdplczqhgvzxzalaqvppxioxxzztqhiqztoeghqwbtxajeabainpbjusjwoclubwsawbzkomsluvzzgrqwwrancoijcbfzsqlshhfzgvemnqsayluybkhixbvpaffheanfkykswlupntvqindckgkvbzcasqmhgfshlohrieqcwfjrwdrdnxmwczotqetaddnliazgkykfldloknouqcsrzuerkaubfisparhqnsxqdsmafhrvqsuwqlvbnmrymdgcafzmrslzmtihnnjkmnfkhqmiksjehwrvnmgepjndcqxybqzelmecerbkjnwfhmigxrrbqtyqqhgqjicffnlfmwtwzsxdlhxihxndslxffzakjezffvsdzgfkgszbuuvapttgctncmisgszmhcvihbkaduggmyedenkfgxrqotdgbqxxlyhlzfntcdqmtbxxfohupdgeaggmavwrrarkltzevgbugbxckvopibdiyldafvultvkcbxvpiiqzemscdobfjflpftkvnzbbvmmhqktbyeuvuqgxghqrulqbqvaisgesmymqlotumpadyltchbfrrmvlzrooxdpybebeekjpkbqrzvurysderyqdhgpmvhxpuwcukrnnnpgnyhnhgomqtrxwwtnlzojyelrhsebbhxsnwuoikpndiyplzvkzbuuutnjdkohhouslkashpiuwzjggojdnwcgcceeacrazajorlqjhbadoplqnmksxcjbyfvquvqsrocnrjrjtfxrxdlskuizpukmcekgrofrliiofepmtfjypneantxykrzaepambrhnfqryxfzqxquvcerdbabtawpopxbluwmborofprhrsmevvrsxrrvtrfxvadqxucbzsidocydxzbwtjxvplxuvyvwqadeskvtvamtehvwnmrsadpgehrcfikjfykpumshvmhdtwmoxcwziqdheydcizganzrjebjaztybbrrakkalkkmxzrjzwreogplkvevtwmktraylkhdtxrflczlqevctcmvzlarvohmbrsavdrklgynbzdyigejsjnxcgvhjlkshmlethafjnoxljtcvpahqwzinozuympfjyznaaqsqmivpoxuvvjmwrctmuwywjtbaveycywbhxmqtuqiswvwexmgvcoqumdmwufxufbairrxdnhdihkbphnspyiktvlffybnqycfgygcjafkabvmdfgknqthvgzkjhialvfrcfobnsfdohajaryoijwhvurqcwqzzxzdyymylpymjjedlrysazyjccqzdhsbwkoqkbxxqpkuwtfjvdgsfetbknckqfqrdlhlqnpbrmodhmuckuvxevbglgcpnxwsjmumlulddlgxrawmpdxpdrjttorhvwuectmugccxbcrmhgrrzjvipcjhzhyzgbebtnkatzfezkbojngoyeskwyztdwwdxqznaehzyybrwelqujxjkyyouturrkfimgasncpjmacyohgyuxnwlkiknwsqucklpyjeugsmufsbwvghklvijkssyeidpbgrbmelczojkusujxltqsymlijbczmqyovupdnxoqpsdllwabjpyrajpanpvzepkrlkhqxomddbrchdytvufjibjltixzxwjapscvxnnmgesepxpovfymvjudyzxysxeotamqbmrwhppdmqgqikswijioojhnrsobriwbchirovqrdvlehftmeeprdsqckigvxfzrnobviaepvkxwudhfpetwyhhvncvkaliiwzeoqxeqgbcrsxnfoocmhnxyhqhcpqievjoleuknldklzvxoynasifhcktxurtkujvgdhrzgxutvlccqqqnbuoatkkgqacpdmunowfwjalxqofhykycngqwdsdlijzmgscixqesunjedntwphjzgcbobgntrwqhjylsdjaascopthxoctpeftbgrgpezwnkwgrhfnkysuglbhypkprkqjczztulaepiesffwsclwexunewctbglkroevghlhmwgjqctsxdvuzkkjpnlzvvqkbuswkxmzqsosmefkpwleoczaacmulrbaqzsliwucuviwabxnmqsbzcwajakkoqbgekqngkakaziqqsvjgwyhcdzregrygipegvpwkwxgpivytpjfqcwhuopxjfxcjeynipnyeltaqljuehuzgyppdtnkwnmvjhsadjtiwgdchmlwamxwwlqyetgsvxtcwnpmznadeuwnvwkfizzpnrajekkfjsiksvuqnazoozgycvlkuavgdhzhetebgamnezbxqxgjkqfegcdpkhnrxruscxaklsrgejozxsvrzjennpzkykbelympqxpfvjcnnjxsvlpcpjbubnfjpzagwhhriqwujollcvvzkrauonugebcktrvyehqefzvcixeuiiibwrdwzpmpcxcxaxaywgywrhcfdztsnpyirdqfgobttxxilqqqqrlkcbwarnkntjpmwhbpyoljrptkfabhocadjezwqgpsfnmowssuzvpfkvbtyfbhkmpqduhukoegkwhfsqsrvijuvchsysrxwhrlshzmjnjzcoeywnrqwthroxzpuyzgxjntlaxbvjvknfbkkuvrzibevzqjlpxkvfjhyawphhwqzuaimueuezndrwcdnkjajvruofmynyybuuvapsopqrthiizcyrmmgsvryxjkcvhetsubdcklnrsoicuwtffohmlyvcsyzfjeuadlnimyhphumveetazwawozhnthquaaofdtmczubamxtskandzqlwjvhkpyacsxakymhjdvsjfosvbyolfxhfsarqiwxykuasndnlhcamifbyfzavniwizqqpimpsxpjefvnvbzdbsnsmxdvjujhnhuhjbqodbecuudhhxnuctxzejsykyivsifrphjgkmhdizbkyeutfqcsdaejxucyeqwcdnquhjhaabdkzzmywolhfgnwjmgzhitaqapwghqstaxjxbsqtoumfoacocfqiuxpghhxjbstlrfogdvekzqyuivjshirfnesnmpwxmorosahtwientfzeoesnrvhctlobflefaojbcwwupkxlcruvddsvskfwfbebeohrvxjymlzzvsouudscaiijqoccvhijhtknzarineumcpgyvsruzdmuuyaftpmmdkvgkgbwjwryjknhtwwkntrvhycyfwucwgsopiaotkvpkjzceovkjbspzivszfmspgnrvybepvylrxzsixiknxeoimejwgtzlrzlkqkfjjezzkbobsixonygrjgadduercgkjddvbfxoscjuprvyzanaaaqixfpgqssnzehdqfveweyjywrhnrisfdzyfuayhpozyypfjzhafrmfwpfjlfllmwjmhjlungmouytmbgscohuadwrpgclurdxzexnqldniemkwjraqxlctvcvyoaenlqcghxjhsnphfcnffahuntaaiyctixrloexcmzirbyeudnwjiiinzlmcittfgddsejbfwjyypyoeblweidrlouewxptupgvvxsbabyzuyvtqhlngaettzmuabasggggcorpkaqsoglmdpubxsezlhhumjzxgvsaqifvchnqrxlqoevantzsrhwhwemsfnzxdfdrvrqpwzuzyenahzriauycywowdkkendgdlwsxxcldmmqattsmkoknigkhusdcgrhjkbikfuwmzjzsjsmfeeronfbzfyqdirqzorkhoaxkirjhsiwhnfeqlznqeyevxfjiljubfayyefndpqrlumaljbehwltmleabykehfgtnnwsynqfkdddpdotgtgtexrbztvouotmyedjiqkgpmlvybixqxcnccspjstwidkwapibqpddgtkzgbacahxeasefduldyvkogtoihzmecyemrqnvsiewsmvjdzaytyjdunemvpcrcxhlznvgatyzjgmsfxnustocigabluyxozuamymdxqwbdtiouojcioskpypdsqssiqnduzswggucfjbmkedwrdjszvajwkzsubduqnycmscramhmqtutfbfyfutubrznxmtnixilguxwxtzbhfudxwlmbqyhazfhfqcdtonozxgozuixuxftsioipgfzwcnbntlpsogirjwsxftsoruxkuboagzptrskfdyfmryttddpiyqoyawhetedlnwgkyhhenockyiqnifqmkxomgeoblrgyeovqwpupsidbvrbrqydrddiluaxtpmoijqfvvfznyoatqxgibioprxwqygjiufxvljasovbjnyqwhpgfbgwouewbgawzjpdxnsrpdcthurtqqjokwdurglayptittccruwuhuypdysxutpjskxlvhnbfnldmnojxdkkdstltmwhmycfpiswbzqstasiinehpoftcgnkispayrsaewvrbiyhowqwfkebufzrddentckhhipqtpawmkdujdktqdmrnofwkkmjzgpnpepqetadjannpfsaxmihbdspokihibrgdvtmerphplvoxmuwjcnynnxixhviptjqgrttzkqlfxgfopxypfihirpkertallgdtmqnxmnqgtvjnnsytqxvvnyrsatdelyacuidmznmoucyojvhklatknnwrxtltynduyieolzcsxlcvlgyzficqdodwwihlyexkmfpsjqdrsvdgsgixwvzwrzsgriexsynjpljtnqatetyodbivclenxfoapcrmhtoxketfpfhrkdwnwvmhlvrnwpqprwqdnharfmzpkbimahexzkkrxciqgrvozpxeppoexpjjhqjzqutqjzkactfijvmefklpkpoohhyoixvylfogrqqqmzfaqqyxkfzpyfeceinuawugrrihxnpnelmstbqedpidslwlwnsqqmwsbmlrgidnijwqxkwqsqgysgyehibbxngwvygebghomjoeryipldyebskzkogohmrlnihfdcewpjyjdzidoiewhxrvmdjlvtcdnjvgkqfzyvadwkdxbjklraypvtkpjdfjwldaoaxgdyljahyjpzdlfcmxnwkasgpilbngyapxgterpjwkfncbqbrfonapuubcgygxasphuhxawustibjnpgpabqplmsukrkltdcvjiorotkqjrfycnrwmifuftjmppatyznektnrihpvlmzpscnlfwsbasxeayntxqarlnrxhqlxxliaahdexgvwxupsjskxdemzquzvnajixrcbhdhqdjgnyofvzyzafmmitvllltvlcwagqkgyxwabhsjejfjlgnahiythkjxlchjthpmebvotmctammmnmsrbnvrtfvvhoawulojivjyidtfrmxrebpybsqnvjyjjozmtxubyuuzkxsydkfxzfbcqlvhqybyhhjbpstgfzvvdnwfejjmbvcccqefsiqtxcqamayahnjemxltplkvohbcerufauwrbjivmyrdqxbubdgbdcyjwlunfrivwetfrzsuzwsilcstobttivmjdpywipddachsaitcbjbehapylwskepucmzxneovtdaawievrceqbzgzcqkplhcgfvhmpgthqbwyniajelyuvapkelsbtggpgdnnddpzqvkrelxhjmlijpvhhawquwocdqustcmdelxegnmbvaretgmkghsiwlsnpgtxiwhomiseytgggwayfnuamcdtdjlgaypppdnxklpajnvnnvcqkufcwtihvnmthyrdweqlgcqokptmaeffkkkiyhfdqrllnixogmbdiadtgikvdzddvzwlhvtcbpdgripixgarndbomqncsgtorakxzyzcezwmrglxizngfasucuegrrzkjlrrncrzbwjbnzohtedthhgtgzmawdwwmhcutulzobuiwloocosdoqoqvlecwipvrljwbjjfhlowkqbcmxduflimananobsuzqrzjweyuodofwwznodwfdxrzqwatukwhfpvdrdlrhmrumqgfzumjttxatauenuazeqsootlqhnuvtmunclwdnkvmoesxgcaqkawibgqdscytvotmlouojnjmikebyvdiauwomjszpvumlkajcknzlrejxrwkwygspctwceivbcfcudtewldnnvcgmokznsoyophwerhrezdsqspmdnfplbgfopwriugexkwmcpguudnrefvikiwznuupawzfoqeisvovjayktmyiknywdsqexxxinwjxwntxrylmttsmzwztpchakaopceinqstkzxjfihjbwklwctzbglfmscodvoqarihsifobbijcnusurvcajaiuvghtacdekcblizcthujbsihjzkxflpfirzkfwaunrijatvcoeonplrobhhwncrpwmesedjmalpbdegxxujzrpkwpawoiwxotbjsllivnxwuedddixtfloeumzlkzzndoodthdswqbzncdsodibsrcreyfmmltnudqqopfgdseyywdrqboihycpllwrrkhypvomkvtlmbkamaxsrokidrnwtqmxmojyckcekcbujnoowwogdzfutronjdmfokgzohszqcweqinbeujhoutyvstqrtwukmlfuusmsgysomfpiwpbwfkepwlpzdanqvrszolkgqaiodqkmaxccrelqsiqnhpihgsqhffcaaldbdtqaznodjhwyewymtxnqidgqyaaccshguaiuxhodmtqdguweiragfczbjswasonptbngohsorlovsxuewtwikmdxvoqcvgbnnprydrvtfkfxyapxicbwsiikbplsjwddgujdvzcygffiminzqpqqdvokxhttcoubfjvmqziuepzlszbtcwymekwvpfxoncapvbrjuynswhqqcruydzqgclyggurlimluzwtffffreonryxurejgwhfcwlhjikxgrrheuzczkxglcgadcdvhukkkjbtbdwzwkwuzrepfkkhhgdmrtaxvlvylqppwixxypletibvfhurdeswgtwomdiypyxxpxegdkqtttxpxelufuqunapvabvzybuyveazccmoigyappspgerloqttriuockrhjpmytfpbtuvujjlqfdthlrdhigwzrvespsriaquvlnkuuaqsuitruengctkfmlgpttpmiwacqwjinjrydjewchxkytriecwgygorxxxqmeishjvwuqveelvbycrmghxynalihysmxqzgltcnzuzalkbabmcjforulfhztxjzkproqqtbjoezpxtpvhfufvezpjbcnfodyutexhpbevnlxailzrmblqlpngrtejdccjwysxnrsxpkfvabbklftsjsdkatkyvndljvyonzgtglibuqffvtanvfmzudrvkpqwckwqwbmtbonchmwdxawrethirgtxgryxoxihxyuarcnfykwtjkjrhmxroosdtfboynrbclezkcmdyeovoxrddlyqirlohvpsiwkwhsmreeaotyqkxxsydwzckkenlaejtcjbkkqldtrslvkoookwrlmvfvccbzubfeehfwssdjxapumrccbotlknnzgybjwxbtqsslnvybbiuvunygtlpfpmjfopcdkbsgohndomghdwrdbeevfcknntvjexbzxwzcifahanpnkgxrarwvcqmlswwcqvvhypuvagbogdqditmguwpgvknaouvotkmknuhsnvomzsqirmulmcwgmacmxgtaewfqczedxpzvopzjrhmxrbjfidssfitndrhirwgyjxiqrieqlmcrfkxwlpfjjcwiwhkdiohcljqewbmdgcpwsvremieeecptdgykfryigujituoidyifejwzychfxmxxgbklcatzcxqyqwnlsucwvhxlkhhwtjsdtrsmrxvdzzuqujhstfbziudbxewzfmsunxflquujiocwmxzyafnprgbagpvcphhsukvhybqyvfsjbmutoyopcgbguftlpxnshepmtcekexodvjqvzutvafsftmpnnhthzqzrftupicpkzbvzdknjmoekrultshxxgsygsqfauadbtziulidufksksynbolhhdkqubfbhvpuvxnnmkfqgdfzngwieyqrkctnkztdaxibazlabnbbcjzckjdgvbsdwramvkhtsddtvjdcdvjaitcejbhtqbqdtjlubhvlbgdiznrqxmrsvylobandhcmvqkdixwdkswjglhwpxhwffwgjsbiadkwclivtgyrpoxuotsmzsqubijanjsrygkviplaehwudoiaixxwotasnfnramnlthrzhgvggoocruuitcrwuzmfawhfmqpoxhydaxtfomzxsjlxsriclxtaaakgbbtidqkrpuyelmpfvekukzvbldcnspeihjchsqdaiacciojjxdeuftmstcnqpekqhxutkpwdmytlxrhucnnxubfkazjdttxcssceakcuppgqnbycizytpjrykevqihytphtluiuatnmpbfwcqdlmlwamsefmmtjqvymmjvoaiinkkncsoeqpuhusdqyccjjnuabylfngypqcshgowdcjtgqhhfglwdehtrzilezxqtambjwsjbwbbsdocjmhbfihaosrdulmkzzadeourfenadyuzorjufajozlmlsxxjhyznussivhnnyndybssztaqgaycdmthwemqijepazfjfeebyhwhgzmbxuvrizlxhibspyixtenafnpihkabjmzjxbqcgsqpgzbdlmmirrfexbdmdvdtoqdccriopoprdyhokyciyimxlxkvgezvfjlwjzurmftqhjdxdzkylehyywfgsahmoodypljfptinfkosovogopdeixflwpgmdlpvtzgazreteuqplbfdpzpnvcrvsheizcndblnljbcswcasgwqdhjrgubezkudqvfaufmaxpfrnqrzznvjnutrzmjwaykpvdxzmkodbaadofljpcakszaicpgobldcuvycjzxpktyacwmyqqqfbtddbhmehullqyheivvswpuvyogruvoxyfmixuibovthrsauuqsopnozuhcpyaedviiuevtiyhaofrrnpgetgswtesglmngphtmbeafbrkdpcdqwbqblfobbgwcwgzbsvmxkjzvqpzldowtyxcnnnlydobdxzfftmtrtdgumpxdavnptzfrvcdgtuietbbtbjeatahpuvtwyvwwfcmgvfvojepoqmnnrogrzkpxaryeggglclpysjifbgqwbjyykvovfzciujsnljdrezguzkcixdpsbvgxuknnovwxxyprphxzobmpfpnneginstayfjuqgdlwxuxpexubkbtqeldwktyhojphfukgkjzfdwqbrndpehyrjanbvkykllbwkurhhhnuxmzibdvffyaheyjwgfgcdavojclzwoqsklpysgygcigzanyyvmlgatnllggysjsphgjcepybttkucspbumgantabsoymurhjpagaphikzvobrfvnlczawrsyvelxwxbggsjugwokuflprynywuqvpptkxlkhtzltwbqwujnaqbbjkqfewnfcsrvzjsplxjbljzjrvdkrvspafnicbnnmaxsynkhxwbiscqxlqchcqsftbsgtsxlypyromkttwmnagiqlancbeicyemaosbucgrlxntgmdagfegzfhidcuhdxhhipersqfrwdwgqhsknlthadlhjsxlmtansofeafkntoootfdqbxxjqdyqyikogzozabfueojrcjxmilmmhouhffuzlsjidmtuutnsrligrwxjdgectjiunceghovwsnbfjisuyhthwqiwzpzsxqmlwamdtjtxcnnbjyljnywuaqhzcrvshkcmamjzmewjeumzmpwsfwknvyirrvgpgqlqmtoobvetnrpbdothtbhaqerdpgvulijfsfpcogiskrmjxgqttdemkyodzcercnnqhdfjuiisckvfvblabozgpxggwizhtlieaavtecshbksazoxzfeertratptlblfkiafiajtuyumsxqioajxofeqjqsdqfxgdtwzezzpjzheuxlpoeilzaxyhpfzrdnwmdzupwppnvxvopwbvnpujmaliurrrwmxgnhnuzzxpklxfhpuaivgroqtlinowkyvakftymmuqhrqojarxhrumwkugxcglvmtjygwiyqvvbljvwrfgqigmyxrpiwsumtyxkuuhpoxdafxaxsmybuweeomfkbbmcgmnanovpoxziunrwlbszphuvejzjjvmeuzaheerhulygzirpsxktvuqqvmarkpbbujkklsqhpljanmypfbhlrjzawhurzcsbszsxcoaatlsenubsxaugehacqvgajzfyhmpcsfmjspposvhcqracbpdxvqpvdtgcsylakgcjcwduahbpzwnpbhtqzbufwjomnhemiasilyymncbrejotwxntmelzrjmabdkeijopgdptsqxsufwhgljbgxmkbxfbffwenueofpqgofckjltpcluymuekuscczqmvlwpqgcyflxwveztjymfmobqsylgtutxhnfmttdlvnzuxxoglplbdgykithjpqiqwpjwpvvaisdqalwyawadavykcegtfhgohldyqamsuvjbulzevzavrbykquvogaurelwjnpbepwwobprijsfinkoqgcqnotzxqoqndrkvmyshfadmdadsuctlimgsdhrgqvlvqkofjbijjwwzamwokhgtlpwqihohkmagilpddtaifytmuiduuqlafhvnjgtsmygmyzddacwqtrdmbevuqzjagvzfpnjknirkculozjnaeujqprbkerajxfbpoomyelhvieafkfsvlakojpvayuivgtwqjfygwminiwksyhqgarvwpezsmovvrpysiipjccgnkuwntxwifvzkaukpfytcglpmteficfvgnpapjpjolvhrhrdcwipibsaolkzduueukmbhaxjozhilwygyiuldshtzltxzaznkbbxyavdzppkyhvubahpudkkesnzrlpfvluafoaazgncoilqoudjmuspsjsurzdxffuzgxidfblrjzoejyhnwolazyytuivjutxzphrqsaetgapdivmvponoyvlksmbccalwxrsahwymcnfpcyqpuybctfbvtexjqbrmrdzaajhbwvbdvwzvctovoagstuphhubajiorzibbcwmligzczxqpcketqtghyjcaqpgnoknfaktlqaeuwgbldlnikkzkiqmyjevmdsfiubdzkexuyamjnlgexzuypezjuezqnzwqwjlpmhfkxrgqeszxjedjwyivtbmmrnpjgsfvvtfpnkuqrrlaqyiiztbsssewqsdxzysismipbuawtdvorowzluoudtticwanzjlkaidbpmmginpdkqpiwlnwkykmuvixldkakowerxalovvskydeoyxokgqixzxyoepnnfbikfhncxkgjjlouwfzxtfarnxwcbhmtjebankpzdchzzgbmiipelsvpfobcduxhtvuesbwykmmxxuioaybmzpkgfyyerqliicnkqhthxnavowptvrelidrolhjxdibtikbhdmqrlkaekbvfcrwtnnhxwoxkukbphkdlxnwvhgqvogdrongtpinpwawamztvflwetuytcwtbffwjqqrjigcawmfzhmwyqoelojuqnrwupsnvajlfeipkozmoblfmwfvfwjxseeiebriypcuijauhduxddlniksevxuppjsvfwfrvniavebjbfwozeyhsldzttpxsqxjwxzhwrsemfmzejnyzshbeobdhrgtclsf\",c4=\"wthvqxcsrlps\" 1626006833639000000" - }; + char tmp[16375] = {0}; + memset(tmp, 'a', 16374); + char sql[102400] = {0}; + sprintf(sql,"lujixfvqor,t0=t c0=f,c1=\"%s\",c2=\"%s\",c3=\"%s\",c4=\"wthvqxcsrlps\" 1626006833639000000", tmp, tmp, tmp); pRes = taos_query(taos, "use line_23881"); taos_free_result(pRes); - pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, - TSDB_SML_TIMESTAMP_NANO_SECONDS); + int totalRows = 0; + pRes = taos_schemaless_insert_raw(taos, sql, strlen(sql), &totalRows, TSDB_SML_LINE_PROTOCOL, + TSDB_SML_TIMESTAMP_NANO_SECONDS); printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); From 4e07f5460b731e1fdcf7f55bd35224f163118f4f Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Thu, 27 Apr 2023 09:51:16 +0800 Subject: [PATCH 079/145] enhance: relase enterpise pull jdbc from tag 3.1.0 (#21102) --- packaging/tools/makepkg.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index a590835257..3da005c405 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -341,7 +341,7 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then tmp_pwd=`pwd` cd ${install_dir}/connector if [ ! -d taos-connector-jdbc ];then - git clone -b main --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + git clone -b 3.1.0 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: fi cd taos-connector-jdbc mvn clean package -Dmaven.test.skip=true From fe3f3f478a4a1658bddd87b489ec47d572b39894 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 10:15:52 +0800 Subject: [PATCH 080/145] fix(query): check the init status for pReader->status.merger --- source/dnode/vnode/src/tsdb/tsdbRead.c | 115 ++++++++++++++++--------- 1 file changed, 76 insertions(+), 39 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index fad2e883aa..ab368cd4aa 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -246,8 +246,6 @@ static int32_t getInitialDelIndex(const SArray* pDelSkyline, int32_t order static STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, const char* id); -static STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid); - static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static int32_t setColumnIdSlotList(SBlockLoadSuppInfo* pSupInfo, SColumnInfo* pCols, const int32_t* pSlotIdList, @@ -1354,16 +1352,40 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { return TSDB_CODE_SUCCESS; } +static FORCE_INLINE STSchema* getTableSchemaImpl(STsdbReader* pReader, uint64_t uid) { + ASSERT(pReader->pSchema == NULL); + + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); + if (code != TSDB_CODE_SUCCESS || pReader->pSchema == NULL) { + terrno = code; + tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); + return NULL; + } + + code = tsdbRowMergerInit(&pReader->status.merger, pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + tsdbError("failed to init merger, code:%s, %s", tstrerror(code), pReader->idStr); + return NULL; + } + + return pReader->pSchema; +} + static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockIter, SBlockData* pBlockData, uint64_t uid) { - int32_t code = 0; - int64_t st = taosGetTimestampUs(); + int32_t code = 0; + STSchema* pSchema = pReader->pSchema; + int64_t st = taosGetTimestampUs(); tBlockDataReset(pBlockData); - STSchema* pSchema = getLatestTableSchema(pReader, uid); - if (pSchema == NULL) { - tsdbDebug("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); - return code; + + if (pReader->pSchema == NULL) { + pSchema = getTableSchemaImpl(pReader, uid); + if (pSchema == NULL) { + tsdbDebug("%p table uid:%" PRIu64 " has been dropped, no data existed, %s", pReader, uid, pReader->idStr); + return code; + } } SBlockLoadSuppInfo* pSup = &pReader->suppInfo; @@ -1912,33 +1934,11 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas return code; } -STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_t uid) { - if (pReader->pSchema != NULL) { - return pReader->pSchema; - } - - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); - if (code != TSDB_CODE_SUCCESS || pReader->pSchema == NULL) { - tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); - } - - return pReader->pSchema; -} - static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { - int32_t code = 0; - // always set the newest schema version in pReader->pSchema if (pReader->pSchema == NULL) { - code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - return NULL; - } - - code = tsdbRowMergerInit(&pReader->status.merger, pReader->pSchema); - if (code != 0) { - terrno = code; + STSchema* ps = getTableSchemaImpl(pReader, uid); + if (ps == NULL) { return NULL; } } @@ -1953,7 +1953,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* } STSchema* ptr = NULL; - code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -1982,6 +1982,15 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* TSDBKEY k = TSDBROW_KEY(pRow); TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + // merge is not initialized yet, due to the fact that the pReader->pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + int64_t minKey = 0; if (pReader->order == TSDB_ORDER_ASC) { minKey = INT64_MAX; // chosen the minimum value @@ -2011,13 +2020,14 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } + // todo remove init bool init = false; // ASC: file block ---> last block -----> imem -----> mem // DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { - init = true; // todo check if pReader->pSchema is null or not + init = true; int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2203,6 +2213,16 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader* pLastBlockReader, int64_t key, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + SRowMerger* pMerger = &pReader->status.merger; + + // merge is not initialized yet, due to the fact that the pReader->pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } if (hasDataInFileBlock(pBlockData, pDumpInfo)) { // no last block available, only data block exists @@ -2220,8 +2240,6 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { SRow* pTSRow = NULL; - SRowMerger* pMerger = &pReader->status.merger; - int32_t code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2285,6 +2303,15 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } + // merge is not initialized yet, due to the fact that the pReader->pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + int64_t minKey = 0; if (ASCENDING_TRAVERSE(pReader->order)) { minKey = INT64_MAX; // let's find the minimum @@ -2596,6 +2623,7 @@ bool hasDataInFileBlock(const SBlockData* pBlockData, const SFileBlockDumpInfo* int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { + SRowMerger* pMerger = &pReader->status.merger; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; bool copied = false; int32_t code = tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo, &copied); @@ -2603,6 +2631,15 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } + // merge is not initialized yet, due to the fact that the pReader->pSchema is not initialized + if (pMerger->pArray == NULL) { + ASSERT(pReader->pSchema == NULL); + STSchema* ps = getTableSchemaImpl(pReader, pBlockScanInfo->uid); + if (ps == NULL) { + return terrno; + } + } + if (copied) { pBlockScanInfo->lastKey = key; return TSDB_CODE_SUCCESS; @@ -2610,13 +2647,13 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); SRow* pTSRow = NULL; - code = tsdbRowMergerAdd(&pReader->status.merger, &fRow, pReader->pSchema); + code = tsdbRowMergerAdd(pMerger, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader); - code = tsdbRowMergerGetRow(&pReader->status.merger, &pTSRow); + code = tsdbRowMergerGetRow(pMerger, &pTSRow); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2624,7 +2661,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); - tsdbRowMergerClear(&pReader->status.merger); + tsdbRowMergerClear(pMerger); return code; } } From 3093726c5a86eab77778d9a6c8471b1840ca6bb7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 11:49:00 +0800 Subject: [PATCH 081/145] fix(query): init row merger for tsdbread. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ab368cd4aa..a46f7a7924 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4449,6 +4449,10 @@ static void setSharedPtr(STsdbReader* pDst, const STsdbReader* pSrc) { pDst->pSchema = pSrc->pSchema; pDst->pSchemaMap = pSrc->pSchemaMap; pDst->pReadSnap = pSrc->pReadSnap; + + if (pDst->pSchema) { + tsdbRowMergerInit(&pDst->status.merger, pDst->pSchema); + } } void tsdbReaderClose(STsdbReader* pReader) { From 12e0f3fda83114f2545849e5dd783735911eac05 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 13:36:09 +0800 Subject: [PATCH 082/145] refactor: do some internal refactor. --- source/util/src/tarray.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index b15bb519e7..896108eb95 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -483,7 +483,8 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t // todo remove it // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { - taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); + taosqsort(pArray->pData, pArray->size, pArray->elemSize, param, fn); +// taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); } void taosArraySwap(SArray* a, SArray* b) { From d83f29dc230ce354a8cfa5f8f9afd62d38c8f0ca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 15:33:04 +0800 Subject: [PATCH 083/145] refactor: remove assert. --- source/client/src/clientMsgHandler.c | 1 - source/common/src/ttszip.c | 1 - source/libs/executor/src/timewindowoperator.c | 14 ++---- source/util/src/tarray.c | 13 ++--- source/util/src/tcache.c | 29 ++++------- source/util/src/thash.c | 33 +----------- source/util/src/tlrucache.c | 50 +++++++++---------- 7 files changed, 49 insertions(+), 92 deletions(-) diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index a0146cfa39..032517cafc 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -92,7 +92,6 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { goto End; } - /*assert(connectRsp.epSet.numOfEps > 0);*/ if (connectRsp.epSet.numOfEps == 0) { setErrno(pRequest, TSDB_CODE_APP_ERROR); tsem_post(&pRequest->body.rspSem); diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 0faa6eb4c8..f415bd20cd 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -982,7 +982,6 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur) { return; } - // assert(pCur->vgroupIndex != -1 && pCur->tsIndex >= 0 && pCur->blockIndex >= 0); if (pCur->vgroupIndex != -1) { tsBufGetBlock(pTSBuf, pCur->vgroupIndex, pCur->blockIndex); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index be2bd0e6e2..ce2d4b27dd 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -153,7 +153,7 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn // } } - assert(forwardRows >= 0); + ASSERT(forwardRows >= 0); return forwardRows; } @@ -165,8 +165,6 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { return -1; } - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - TSKEY* keyList = (TSKEY*)pValue; int32_t firstPos = 0; int32_t lastPos = num - 1; @@ -230,7 +228,7 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order) { - assert(startPos >= 0 && startPos < pDataBlockInfo->rows); + ASSERT(startPos >= 0 && startPos < pDataBlockInfo->rows); int32_t num = -1; int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); @@ -261,7 +259,6 @@ int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimary } } - assert(num >= 0); return num; } @@ -433,7 +430,7 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SEx } int32_t nextRowIndex = endRowIndex + 1; - assert(nextRowIndex >= 0); + ASSERT(nextRowIndex >= 0); TSKEY nextKey = tsCols[nextRowIndex]; doTimeWindowInterpolation(pInfo->pPrevValues, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, @@ -494,9 +491,9 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, */ if (primaryKeys == NULL) { if (ascQuery) { - assert(pDataBlockInfo->window.skey <= pNext->ekey); + ASSERT(pDataBlockInfo->window.skey <= pNext->ekey); } else { - assert(pDataBlockInfo->window.ekey >= pNext->skey); + ASSERT(pDataBlockInfo->window.ekey >= pNext->skey); } } else { if (ascQuery && primaryKeys[startPos] > pNext->ekey) { @@ -533,7 +530,6 @@ static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType typ } static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { pResult->startInterp = true; } else { diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 896108eb95..8e68e78acc 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -69,8 +69,6 @@ SArray* taosArrayInit_s(size_t elemSize, size_t initialSize) { } static int32_t taosArrayResize(SArray* pArray) { - assert(pArray->size >= pArray->capacity); - size_t size = pArray->capacity; size = (size << 1u); @@ -252,12 +250,12 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) { } void taosArraySet(SArray* pArray, size_t index, void* pData) { - assert(index < pArray->size); + ASSERT(index < pArray->size); memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize); } void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { - assert(cnt <= pArray->size); + ASSERT(cnt <= pArray->size); pArray->size = pArray->size - cnt; if (pArray->size == 0 || cnt == 0) { return; @@ -266,12 +264,15 @@ void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { } void taosArrayPopTailBatch(SArray* pArray, size_t cnt) { - assert(cnt <= pArray->size); + if (cnt >= pArray->size) { + cnt = pArray->size; + } + pArray->size = pArray->size - cnt; } void taosArrayRemove(SArray* pArray, size_t index) { - assert(index < pArray->size); + ASSERT(index < pArray->size); if (index == pArray->size - 1) { taosArrayPop(pArray); diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 1d480e7beb..f9606e5d49 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -244,9 +244,9 @@ static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STr next->prev = pElem->prev; } - if (pCacheObj->numOfElemsInTrash == 0) { - assert(pCacheObj->pTrash == NULL); - } +// if (pCacheObj->numOfElemsInTrash == 0) { +// assert(pCacheObj->pTrash == NULL); +// } return next; } @@ -261,8 +261,6 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem } static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { - assert(pNode != NULL && pEntry != NULL); - pNode->pNext = pEntry->next; pEntry->next = pNode; pEntry->num += 1; @@ -503,7 +501,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) { uDebug("cache:%s, data: %p acquired by data in cache, refcnt:%d", pCacheObj->name, ptNode->data, ref); // the data if referenced by at least one object, so the reference count must be greater than the value of 2. - assert(ref >= 2); + ASSERT(ref >= 2); return data; } @@ -516,7 +514,6 @@ void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) { return NULL; } - assert(T_REF_VAL_GET(ptNode) >= 1); char *d = *data; // clear its reference to old area @@ -575,19 +572,19 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { if (ref == 1) { // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may be // destroyed by refresh worker if decrease ref count before removing it from linked-list. - assert(pNode->pTNodeHeader->pData == pNode); + ASSERT(pNode->pTNodeHeader->pData == pNode); __trashcan_wr_lock(pCacheObj); doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader); __trashcan_unlock(pCacheObj); ref = T_REF_DEC(pNode); - assert(ref == 0); + ASSERT(ref == 0); doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader); } else { ref = T_REF_DEC(pNode); - assert(ref >= 0); + ASSERT(ref >= 0); } } else { // NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread @@ -609,13 +606,13 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { "others already, prev must in trashcan", pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data, T_REF_VAL_GET(pNode)); - assert(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); + ASSERT(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL); } else { removeNodeInEntryList(pe, prev, p); uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); if (ref > 0) { - assert(pNode->pTNodeHeader == NULL); + ASSERT(pNode->pTNodeHeader == NULL); taosAddToTrashcan(pCacheObj, pNode); } else { // ref == 0 atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); @@ -736,7 +733,7 @@ SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pDat void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) { if (pNode->inTrashcan) { /* node is already in trash */ - assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); + ASSERT(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode); return; } @@ -782,7 +779,7 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) { STrashElem *pElem = pCacheObj->pTrash; while (pElem) { T_REF_VAL_CHECK(pElem->pData); - assert(pElem->next != pElem && pElem->prev != pElem); + ASSERT(pElem->next != pElem && pElem->prev != pElem); if (force || (T_REF_VAL_GET(pElem->pData) == 0)) { uDebug("cache:%s, key:%p, %p removed from trashcan. numOfElem in trashcan:%d", pCacheObj->name, pElem->pData->key, @@ -814,8 +811,6 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { } static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) { - assert(pCacheObj != NULL); - SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1}; doTraverseElems(pCacheObj, doRemoveExpiredFn, &sup); } @@ -827,9 +822,7 @@ void taosCacheRefreshWorkerUnexpectedStopped(void) { } void *taosCacheTimedRefresh(void *handle) { - assert(pCacheArrayList != NULL); uDebug("cache refresh thread starts"); - setThreadName("cacheRefresh"); const int32_t SLEEP_DURATION = 500; // 500 ms diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 926dc304a4..a3cfb815d5 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -150,7 +150,6 @@ static FORCE_INLINE SHashNode *doSearchInEntryList(SHashObj *pHashObj, SHashEntr //atomic_add_fetch_64(&pHashObj->compTimes, 1); if ((pNode->keyLen == keyLen) && ((*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { - assert(pNode->hashVal == hashVal); break; } @@ -189,8 +188,6 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p */ static FORCE_INLINE void doUpdateHashNode(SHashObj *pHashObj, SHashEntry *pe, SHashNode *prev, SHashNode *pNode, SHashNode *pNewNode) { - assert(pNode->keyLen == pNewNode->keyLen); - atomic_sub_fetch_16(&pNode->refCount, 1); if (prev != NULL) { prev->next = pNewNode; @@ -236,7 +233,7 @@ static FORCE_INLINE bool taosHashTableEmpty(const SHashObj *pHashObj) { return t SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (fn == NULL) { - assert(0); + terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -342,19 +339,11 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo taosHashEntryWLock(pHashObj, pe); SHashNode *pNode = pe->next; -#if 0 - if (pe->num > 0) { - assert(pNode != NULL); - } else { - assert(pNode == NULL); - } -#endif - SHashNode *prev = NULL; while (pNode) { if ((pNode->keyLen == keyLen) && (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && pNode->removed == 0) { - assert(pNode->hashVal == hashVal); + ASSERT(pNode->hashVal == hashVal); break; } @@ -370,8 +359,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo } pushfrontNodeInEntryList(pe, pNewNode); - assert(pe->next != NULL); - taosHashEntryWUnlock(pHashObj, pe); // enable resize @@ -446,14 +433,6 @@ void *taosHashGetImpl(SHashObj *pHashObj, const void *key, size_t keyLen, void * char *data = NULL; taosHashEntryRLock(pHashObj, pe); -#if 0 - if (pe->num > 0) { - assert(pe->next != NULL); - } else { - assert(pe->next == NULL); - } -#endif - SHashNode *pNode = doSearchInEntryList(pHashObj, pe, key, keyLen, hashVal); if (pNode != NULL) { if (pHashObj->callbackFp != NULL) { @@ -514,8 +493,6 @@ int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { // double check after locked if (pe->num == 0) { - assert(pe->next == NULL); - taosHashEntryWUnlock(pHashObj, pe); taosHashRUnlock(pHashObj); return -1; @@ -573,8 +550,6 @@ void taosHashClear(SHashObj *pHashObj) { } pNode = pEntry->next; - assert(pNode != NULL); - while (pNode) { pNext = pNode->next; FREE_HASH_NODE(pHashObj->freeFp, pNode); @@ -677,8 +652,6 @@ void taosHashTableResize(SHashObj *pHashObj) { pNode = pe->next; - assert(pNode != NULL); - while (pNode != NULL) { int32_t newIdx = HASH_INDEX(pNode->hashVal, pHashObj->capacity); pNext = pNode->next; @@ -728,8 +701,6 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s } void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { - assert(pNode != NULL && pEntry != NULL); - pNode->next = pEntry->next; pEntry->next = pNode; diff --git a/source/util/src/tlrucache.c b/source/util/src/tlrucache.c index f4172fbb44..e182800d9c 100644 --- a/source/util/src/tlrucache.c +++ b/source/util/src/tlrucache.c @@ -85,13 +85,13 @@ struct SLRUEntry { #define TAOS_LRU_ENTRY_REF(h) (++(h)->refs) static bool taosLRUEntryUnref(SLRUEntry *entry) { - assert(entry->refs > 0); + ASSERT(entry->refs > 0); --entry->refs; return entry->refs == 0; } static void taosLRUEntryFree(SLRUEntry *entry) { - assert(entry->refs == 0); + ASSERT(entry->refs == 0); if (entry->deleter) { (*entry->deleter)(entry->keyData, entry->keyLength, entry->value); @@ -127,7 +127,7 @@ static void taosLRUEntryTableApply(SLRUEntryTable *table, _taos_lru_table_func_t SLRUEntry *h = table->list[i]; while (h) { SLRUEntry *n = h->nextHash; - assert(TAOS_LRU_ENTRY_IN_CACHE(h)); + ASSERT(TAOS_LRU_ENTRY_IN_CACHE(h)); func(h); h = n; } @@ -184,7 +184,7 @@ static void taosLRUEntryTableResize(SLRUEntryTable *table) { ++count; } } - assert(table->elems == count); + ASSERT(table->elems == count); taosMemoryFree(table->list); table->list = newList; @@ -240,17 +240,16 @@ struct SLRUCacheShard { static void taosLRUCacheShardMaintainPoolSize(SLRUCacheShard *shard) { while (shard->highPriPoolUsage > shard->highPriPoolCapacity) { shard->lruLowPri = shard->lruLowPri->next; - assert(shard->lruLowPri != &shard->lru); + ASSERT(shard->lruLowPri != &shard->lru); TAOS_LRU_ENTRY_SET_IN_HIGH_POOL(shard->lruLowPri, false); - assert(shard->highPriPoolUsage >= shard->lruLowPri->totalCharge); + ASSERT(shard->highPriPoolUsage >= shard->lruLowPri->totalCharge); shard->highPriPoolUsage -= shard->lruLowPri->totalCharge; } } static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { - assert(e->next == NULL); - assert(e->prev == NULL); + ASSERT(e->next == NULL && e->prev == NULL); if (shard->highPriPoolRatio > 0 && (TAOS_LRU_ENTRY_IS_HIGH_PRI(e) || TAOS_LRU_ENTRY_HAS_HIT(e))) { e->next = &shard->lru; @@ -277,8 +276,7 @@ static void taosLRUCacheShardLRUInsert(SLRUCacheShard *shard, SLRUEntry *e) { } static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { - assert(e->next); - assert(e->prev); + ASSERT(e->next && e->prev); if (shard->lruLowPri == e) { shard->lruLowPri = e->prev; @@ -287,10 +285,10 @@ static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { e->prev->next = e->next; e->prev = e->next = NULL; - assert(shard->lruUsage >= e->totalCharge); + ASSERT(shard->lruUsage >= e->totalCharge); shard->lruUsage -= e->totalCharge; if (TAOS_LRU_ENTRY_IN_HIGH_POOL(e)) { - assert(shard->highPriPoolUsage >= e->totalCharge); + ASSERT(shard->highPriPoolUsage >= e->totalCharge); shard->highPriPoolUsage -= e->totalCharge; } } @@ -298,13 +296,13 @@ static void taosLRUCacheShardLRURemove(SLRUCacheShard *shard, SLRUEntry *e) { static void taosLRUCacheShardEvictLRU(SLRUCacheShard *shard, size_t charge, SArray *deleted) { while (shard->usage + charge > shard->capacity && shard->lru.next != &shard->lru) { SLRUEntry *old = shard->lru.next; - assert(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); + ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); taosLRUCacheShardLRURemove(shard, old); taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); - assert(shard->usage >= old->totalCharge); + ASSERT(shard->usage >= old->totalCharge); shard->usage -= old->totalCharge; taosArrayPush(deleted, &old); @@ -391,11 +389,11 @@ static LRUStatus taosLRUCacheShardInsertEntry(SLRUCacheShard *shard, SLRUEntry * if (old != NULL) { status = TAOS_LRU_STATUS_OK_OVERWRITTEN; - assert(TAOS_LRU_ENTRY_IN_CACHE(old)); + ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old)); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); if (!TAOS_LRU_ENTRY_HAS_REFS(old)) { taosLRUCacheShardLRURemove(shard, old); - assert(shard->usage >= old->totalCharge); + ASSERT(shard->usage >= old->totalCharge); shard->usage -= old->totalCharge; taosArrayPush(lastReferenceList, &old); @@ -455,7 +453,7 @@ static LRUHandle *taosLRUCacheShardLookup(SLRUCacheShard *shard, const void *key taosThreadMutexLock(&shard->mutex); e = taosLRUEntryTableLookup(&shard->table, key, keyLen, hash); if (e != NULL) { - assert(TAOS_LRU_ENTRY_IN_CACHE(e)); + ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { taosLRUCacheShardLRURemove(shard, e); } @@ -474,12 +472,12 @@ static void taosLRUCacheShardErase(SLRUCacheShard *shard, const void *key, size_ SLRUEntry *e = taosLRUEntryTableRemove(&shard->table, key, keyLen, hash); if (e != NULL) { - assert(TAOS_LRU_ENTRY_IN_CACHE(e)); + ASSERT(TAOS_LRU_ENTRY_IN_CACHE(e)); TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); if (!TAOS_LRU_ENTRY_HAS_REFS(e)) { taosLRUCacheShardLRURemove(shard, e); - assert(shard->usage >= e->totalCharge); + ASSERT(shard->usage >= e->totalCharge); shard->usage -= e->totalCharge; lastReference = true; } @@ -499,11 +497,11 @@ static void taosLRUCacheShardEraseUnrefEntries(SLRUCacheShard *shard) { while (shard->lru.next != &shard->lru) { SLRUEntry *old = shard->lru.next; - assert(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); + ASSERT(TAOS_LRU_ENTRY_IN_CACHE(old) && !TAOS_LRU_ENTRY_HAS_REFS(old)); taosLRUCacheShardLRURemove(shard, old); taosLRUEntryTableRemove(&shard->table, old->keyData, old->keyLength, old->hash); TAOS_LRU_ENTRY_SET_IN_CACHE(old, false); - assert(shard->usage >= old->totalCharge); + ASSERT(shard->usage >= old->totalCharge); shard->usage -= old->totalCharge; taosArrayPush(lastReferenceList, &old); @@ -524,7 +522,7 @@ static bool taosLRUCacheShardRef(SLRUCacheShard *shard, LRUHandle *handle) { SLRUEntry *e = (SLRUEntry *)handle; taosThreadMutexLock(&shard->mutex); - assert(TAOS_LRU_ENTRY_HAS_REFS(e)); + ASSERT(TAOS_LRU_ENTRY_HAS_REFS(e)); TAOS_LRU_ENTRY_REF(e); taosThreadMutexUnlock(&shard->mutex); @@ -545,7 +543,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b lastReference = taosLRUEntryUnref(e); if (lastReference && TAOS_LRU_ENTRY_IN_CACHE(e)) { if (shard->usage > shard->capacity || eraseIfLastRef) { - assert(shard->lru.next == &shard->lru || eraseIfLastRef); + ASSERT(shard->lru.next == &shard->lru || eraseIfLastRef); taosLRUEntryTableRemove(&shard->table, e->keyData, e->keyLength, e->hash); TAOS_LRU_ENTRY_SET_IN_CACHE(e, false); @@ -557,7 +555,7 @@ static bool taosLRUCacheShardRelease(SLRUCacheShard *shard, LRUHandle *handle, b } if (lastReference && e->value) { - assert(shard->usage >= e->totalCharge); + ASSERT(shard->usage >= e->totalCharge); shard->usage -= e->totalCharge; } @@ -595,7 +593,7 @@ static size_t taosLRUCacheShardGetPinnedUsage(SLRUCacheShard *shard) { taosThreadMutexLock(&shard->mutex); - assert(shard->usage >= shard->lruUsage); + ASSERT(shard->usage >= shard->lruUsage); usage = shard->usage - shard->lruUsage; taosThreadMutexUnlock(&shard->mutex); @@ -687,7 +685,7 @@ void taosLRUCacheCleanup(SLRUCache *cache) { if (cache) { if (cache->shards) { int numShards = cache->numShards; - assert(numShards > 0); + ASSERT(numShards > 0); for (int i = 0; i < numShards; ++i) { taosLRUCacheShardCleanup(&cache->shards[i]); } From 17a5691afb9a4c3e6791a55e47be9b2f55fff079 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 15:37:33 +0800 Subject: [PATCH 084/145] refactor: remove assert. --- source/libs/executor/src/dataDeleter.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/executor/src/executil.c | 1 - source/libs/executor/src/groupoperator.c | 1 - source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/tfill.c | 8 +-- source/libs/executor/src/tsort.c | 4 +- source/libs/index/src/indexFstDfa.c | 1 - source/libs/scalar/src/sclfunc.c | 67 ----------------------- source/util/src/tcache.c | 5 -- source/util/src/thash.c | 2 - source/util/src/tlist.c | 1 - 12 files changed, 9 insertions(+), 87 deletions(-) diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index d693faf7f1..96d061fc04 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -179,7 +179,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; if (NULL == pDeleter->nextOutput.pData) { - assert(pDeleter->queryEnd); + ASSERT(pDeleter->queryEnd); pOutput->useconds = pDeleter->useconds; pOutput->precision = pDeleter->pSchema->precision; pOutput->bufStatus = DS_BUF_EMPTY; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index d8efcf50ca..8e32559fac 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -181,7 +181,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (NULL == pDispatcher->nextOutput.pData) { - assert(pDispatcher->queryEnd); + ASSERT(pDispatcher->queryEnd); pOutput->useconds = pDispatcher->useconds; pOutput->precision = pDispatcher->pSchema->precision; pOutput->bufStatus = DS_BUF_EMPTY; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f61fd1ae01..d79ed0176d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -88,7 +88,6 @@ void resetResultRow(SResultRow* pResultRow, size_t entrySize) { // TODO refactor: use macro SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset) { - assert(index >= 0 && offset != NULL); return (SResultRowEntryInfo*)((char*)pRow->pEntryInfo + offset[index]); } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 3d9bacf39f..aaab95e361 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -926,7 +926,6 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup, false); - assert(pResultRow != NULL); setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8542481419..8d9d1bb887 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -156,7 +156,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn break; } - assert(w.skey < pBlockInfo->window.skey); + ASSERT(w.skey < pBlockInfo->window.skey); if (pBlockInfo->window.skey <= TMIN(w.ekey, pBlockInfo->window.ekey)) { return true; } diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index e59ea253cc..8376caace0 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -408,7 +408,7 @@ static int64_t appendFilledResult(SFillInfo* pFillInfo, SSDataBlock* pBlock, int pFillInfo->numOfTotal += pFillInfo->numOfCurrent; - assert(pFillInfo->numOfCurrent == resultCapacity); + ASSERT(pFillInfo->numOfCurrent == resultCapacity); return resultCapacity; } @@ -558,7 +558,7 @@ int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t ma numOfRes = taosTimeCountInterval(lastKey, pFillInfo->currentKey, pFillInfo->interval.sliding, pFillInfo->interval.slidingUnit, pFillInfo->interval.precision); numOfRes += 1; - assert(numOfRes >= numOfRows); + ASSERT(numOfRes >= numOfRows); } else { // reach the end of data if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) || (ekey1 >= pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) { @@ -593,14 +593,14 @@ int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, SSDataBlock* p, int32_t ca int32_t remain = taosNumOfRemainRows(pFillInfo); int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity); - assert(numOfRes <= capacity); + ASSERT(numOfRes <= capacity); // no data existed for fill operation now, append result according to the fill strategy if (remain == 0) { appendFilledResult(pFillInfo, p, numOfRes); } else { fillResultImpl(pFillInfo, p, (int32_t)numOfRes); - assert(numOfRes == pFillInfo->numOfCurrent); + ASSERT(numOfRes == pFillInfo->numOfCurrent); } qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%" PRId64 "-%" PRId64 ", currentKey:%" PRId64 diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6c8e581b3f..5fe2746b12 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -229,7 +229,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { taosArrayPush(pPageIdList, &pageId); int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); - assert(size <= getBufPageSize(pHandle->pBuf)); + ASSERT(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, p); @@ -592,7 +592,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); - assert(size <= getBufPageSize(pHandle->pBuf)); + ASSERT(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, pDataBlock); diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c index 4d348e76f2..a3e26d8518 100644 --- a/source/libs/index/src/indexFstDfa.c +++ b/source/libs/index/src/indexFstDfa.c @@ -188,7 +188,6 @@ void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) { return; } bool succ = sparSetAdd(set, ip, NULL); - // assert(succ == true); Inst *inst = taosArrayGet(dfa->insts, ip); if (inst->ty == MATCH || inst->ty == RANGE) { // do nothing diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 24b25cec80..df32707529 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1662,73 +1662,6 @@ int32_t charLengthFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam return doLengthFunction(pInput, inputNum, pOutput, tcharlength); } -#if 0 -static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { - switch(type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT:{ - int8_t* p = (int8_t*) dest; - int8_t* pSrc = (int8_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT:{ - int16_t* p = (int16_t*) dest; - int16_t* pSrc = (int16_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: { - int32_t* p = (int32_t*) dest; - int32_t* pSrc = (int32_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: { - int64_t* p = (int64_t*) dest; - int64_t* pSrc = (int64_t*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_FLOAT: { - float* p = (float*) dest; - float* pSrc = (float*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - case TSDB_DATA_TYPE_DOUBLE: { - double* p = (double*) dest; - double* pSrc = (double*) src; - - for(int32_t i = 0; i < numOfRows; ++i) { - p[i] = pSrc[numOfRows - i - 1]; - } - return; - } - default: assert(0); - } -} -#endif - bool getTimePseudoFuncEnv(SFunctionNode *UNUSED_PARAM(pFunc), SFuncExecEnv *pEnv) { pEnv->calcMemSize = sizeof(int64_t); return true; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index f9606e5d49..22eee547d0 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -243,11 +243,6 @@ static FORCE_INLINE STrashElem *doRemoveElemInTrashcan(SCacheObj *pCacheObj, STr if (next) { next->prev = pElem->prev; } - -// if (pCacheObj->numOfElemsInTrash == 0) { -// assert(pCacheObj->pTrash == NULL); -// } - return next; } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index a3cfb815d5..b79824e910 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -545,7 +545,6 @@ void taosHashClear(SHashObj *pHashObj) { for (int32_t i = 0; i < pHashObj->capacity; ++i) { SHashEntry *pEntry = pHashObj->hashList[i]; if (pEntry->num == 0) { - assert(pEntry->next == NULL); continue; } @@ -646,7 +645,6 @@ void taosHashTableResize(SHashObj *pHashObj) { SHashNode *pPrev = NULL; if (pe->num == 0) { - assert(pe->next == NULL); continue; } diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 1b12ea0cdd..5d729d27d6 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -104,7 +104,6 @@ SListNode *tdListPopNode(SList *list, SListNode *node) { // Move all node elements from src to dst, the dst is assumed as an empty list void tdListMove(SList *src, SList *dst) { - // assert(dst->eleSize == src->eleSize); SListNode *node = NULL; while ((node = tdListPopHead(src)) != NULL) { tdListAppendNode(dst, node); From 8677b56a4fe606f918d8579243efb8c12b1cc57a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 27 Apr 2023 15:44:44 +0800 Subject: [PATCH 085/145] fix:assert error in tqProcessSubmitReqForSubscribe if put pHandle to array twice --- source/client/src/clientSml.c | 2 +- source/dnode/vnode/src/inc/tq.h | 3 +- source/dnode/vnode/src/tq/tq.c | 36 ++------ source/dnode/vnode/src/tq/tqPush.c | 144 ----------------------------- source/dnode/vnode/src/tq/tqUtil.c | 2 +- 5 files changed, 11 insertions(+), 176 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index cac559b0c1..f727715a54 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -534,7 +534,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); + uError("SML:0x%" PRIx64 " point type and db type mismatch. db type: %d, point type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); return TSDB_CODE_SML_INVALID_DATA; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index e1b1092c28..080e72c504 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -114,8 +114,7 @@ struct STQ { char* path; int64_t walLogLastVer; SRWLatch lock; - SHashObj* pPushMgr; // consumerId -> STqPushEntry - SArray * pPushArray; + SHashObj* pPushMgr; // consumerId -> STqHandle SHashObj* pHandle; // subKey -> STqHandle SHashObj* pCheckInfo; // topic -> SAlterCheckInfo STqOffsetStore* pOffsetStore; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7539761f4e..00684652f0 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -78,18 +78,6 @@ static void destroyTqHandle(void* data) { } } -static void tqPushEntryFree(void* data) { - STqPushEntry* p = *(void**)data; - if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { - tDeleteSMqDataRsp(p->pDataRsp); - } else if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__TAOSX_RSP) { - tDeleteSTaosxRsp((STaosxRsp*)p->pDataRsp); - } - - taosMemoryFree(p->pDataRsp); - taosMemoryFree(p); -} - static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && pLeft->val.version <= pRight->val.version; @@ -109,11 +97,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->pHandle = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pHandle, destroyTqHandle); - pTq->pPushArray = taosArrayInit(8, POINTER_BYTES); - taosInitRWLatch(&pTq->lock); - pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); - taosHashSetFreeFp(pTq->pPushMgr, tqPushEntryFree); + pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); @@ -158,7 +143,6 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq->path); tqMetaClose(pTq); streamMetaClose(pTq->pStreamMeta); - taosArrayDestroy(pTq->pPushArray); taosMemoryFree(pTq); } @@ -569,14 +553,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // remove if it has been register in the push manager, and return one empty block to consumer // tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); - for(size_t i = 0; i < taosArrayGetSize(pTq->pPushArray); i++) { - void* handle = taosArrayGetP(pTq->pPushArray, i); - if(handle == pHandle) { - tqInfo("vgId:%d remove handle when switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); - taosArrayRemove(pTq->pPushArray, i); - break; - } - } + taosHashRemove(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t)); + if(pHandle->msg != NULL) { rpcFreeCont(pHandle->msg->pCont); taosMemoryFree(pHandle->msg); @@ -1091,8 +1069,9 @@ int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); taosWLockLatch(&pTq->lock); - for(size_t i = 0; i < taosArrayGetSize(pTq->pPushArray); i++){ - STqHandle* pHandle = (STqHandle*)taosArrayGetP(pTq->pPushArray, i); + void *pIter = taosHashIterate(pTq->pPushMgr, NULL); + while(pIter){ + STqHandle* pHandle = *(STqHandle**)pIter; tqDebug("vgId:%d start set submit for pHandle:%p", vgId, pHandle); if(ASSERT(pHandle->msg != NULL)){ tqError("pHandle->msg should not be null"); @@ -1103,8 +1082,9 @@ int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { taosMemoryFree(pHandle->msg); pHandle->msg = NULL; } + pIter = taosHashIterate(pTq->pPushMgr, pIter); } - taosArrayClear(pTq->pPushArray); + taosHashClear(pTq->pPushMgr); // unlock taosWUnLockLatch(&pTq->lock); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index d2d17792d3..dca988cbbd 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -206,69 +206,6 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ } #endif -typedef struct { - void* pKey; - int64_t keyLen; -} SItem; - -static void recordPushedEntry(SArray* cachedKey, void* pIter); -static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq); - -static void freeItem(void* param) { - SItem* p = (SItem*)param; - taosMemoryFree(p->pKey); -} - -static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int64_t ver, int32_t vgId, char* pData, - int32_t dataLen, SArray* pCachedKey) { - STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; - - SMqDataRsp* pRsp = pPushEntry->pDataRsp; - if (pRsp->reqOffset.version >= ver) { - tqDebug("vgId:%d, push entry req version %" PRId64 ", while push version %" PRId64 ", skip", vgId, - pRsp->reqOffset.version, ver); - return; - } - - qTaskInfo_t pTaskInfo = pExec->task; - - // prepare scan mem data - SPackedData submit = {.msgStr = pData, .msgLen = dataLen, .ver = ver}; - - if (qStreamSetScanMemData(pTaskInfo, submit) != 0) { - return; - } - qStreamSetOpen(pTaskInfo); - // here start to scan submit block to extract the subscribed data - int32_t totalRows = 0; - - while (1) { - SSDataBlock* pDataBlock = NULL; - uint64_t ts = 0; - if (qExecTask(pTaskInfo, &pDataBlock, &ts) < 0) { - tqDebug("vgId:%d, tq exec error since %s", vgId, terrstr()); - } - - if (pDataBlock == NULL) { - break; - } - - tqAddBlockDataToRsp(pDataBlock, pRsp, pExec->numOfCols, pTq->pVnode->config.tsdbCfg.precision); - pRsp->blockNum++; - totalRows += pDataBlock->info.rows; - } - - tqDebug("vgId:%d, tq handle push, subkey:%s, block num:%d, rows:%d", vgId, pPushEntry->subKey, pRsp->blockNum, - totalRows); - - if (pRsp->blockNum > 0) { - tqOffsetResetToLog(&pRsp->rspOffset, ver); - tqPushDataRsp(pTq, pPushEntry); - recordPushedEntry(pCachedKey, pIter); - } -} - - int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { // void* pReq = POINTER_SHIFT(msg, sizeof(SSubmitReq2Msg)); // int32_t len = msgLen - sizeof(SSubmitReq2Msg); @@ -363,84 +300,3 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v return 0; } - -int32_t tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, - int32_t type) { - uint64_t consumerId = pRequest->consumerId; - int32_t vgId = TD_VID(pTq->pVnode); - STqHandle* pTqHandle = pHandle; - - STqPushEntry* pPushEntry = taosMemoryCalloc(1, sizeof(STqPushEntry)); - if (pPushEntry == NULL) { - tqDebug("tmq poll: consumer:0x%" PRIx64 ", vgId:%d failed to malloc, size:%d", consumerId, vgId, - (int32_t)sizeof(STqPushEntry)); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - pPushEntry->info = pRpcMsg->info; - memcpy(pPushEntry->subKey, pTqHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); - - if (type == TMQ_MSG_TYPE__TAOSX_RSP) { - pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(STaosxRsp)); - memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(STaosxRsp)); - } else if (type == TMQ_MSG_TYPE__POLL_RSP) { - pPushEntry->pDataRsp = taosMemoryCalloc(1, sizeof(SMqDataRsp)); - memcpy(pPushEntry->pDataRsp, pDataRsp, sizeof(SMqDataRsp)); - } - - SMqRspHead* pHead = &pPushEntry->pDataRsp->head; - pHead->consumerId = consumerId; - pHead->epoch = pRequest->epoch; - pHead->mqMsgType = type; - - taosHashPut(pTq->pPushMgr, pTqHandle->subKey, strlen(pTqHandle->subKey), &pPushEntry, sizeof(void*)); - - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s offset:%" PRId64 ", vgId:%d save handle to push mgr, total:%d", - consumerId, pTqHandle->subKey, pDataRsp->reqOffset.version, vgId, taosHashGetSize(pTq->pPushMgr)); - return 0; -} - -int32_t tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { - int32_t vgId = TD_VID(pTq->pVnode); - STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen); - - if (pEntry != NULL) { - uint64_t cId = (*pEntry)->pDataRsp->head.consumerId; - ASSERT(consumerId == cId); - - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s vgId:%d remove from push mgr, remains:%d", consumerId, - (*pEntry)->subKey, vgId, taosHashGetSize(pTq->pPushMgr) - 1); - - if (rspConsumer) { // rsp the old consumer with empty block. - tqPushDataRsp(pTq, *pEntry); - } - - taosHashRemove(pTq->pPushMgr, pKey, keyLen); - } - - return 0; -} - -void recordPushedEntry(SArray* cachedKey, void* pIter) { - size_t kLen = 0; - void* key = taosHashGetKey(pIter, &kLen); - SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen}; - taosArrayPush(cachedKey, &item); -} - -void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); - int32_t numOfKeys = (int32_t)taosArrayGetSize(pCachedKeys); - - for (int32_t i = 0; i < numOfKeys; i++) { - SItem* pItem = taosArrayGet(pCachedKeys, i); - if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) { - tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*)pItem->pKey); - } - } - - if (numOfKeys > 0) { - tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr)); - } -} diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 663dc8bbb9..2398ef41f4 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -191,7 +191,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); pHandle->msg->contLen = pMsg->contLen; tqDebug("data is over, register to handle:%p, pCont:%p, len:%d", pHandle, pHandle->msg->pCont, pHandle->msg->contLen); - taosArrayPush(pTq->pPushArray, &pHandle); + taosHashPut(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t), &pHandle, POINTER_BYTES); taosWUnLockLatch(&pTq->lock); tDeleteSMqDataRsp(&dataRsp); return code; From 91f9b58f9a7f2f9f95397ff04f4ad02aa5376fa4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 16:06:06 +0800 Subject: [PATCH 086/145] refactor: do some internal refactor. --- source/dnode/vnode/inc/vnode.h | 37 +++++++++++-------------- source/dnode/vnode/src/tq/tq.c | 4 +-- source/dnode/vnode/src/tq/tqMeta.c | 4 +-- source/dnode/vnode/src/tq/tqRead.c | 4 +-- source/dnode/vnode/src/tq/tqScan.c | 4 +-- source/libs/executor/src/scanoperator.c | 6 ++-- 6 files changed, 27 insertions(+), 32 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 3bcb1c9d2e..002dcda488 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -175,11 +175,8 @@ typedef struct STsdbReader STsdbReader; #define CACHESCAN_RETRIEVE_LAST_ROW 0x4 #define CACHESCAN_RETRIEVE_LAST 0x8 -int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); -int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly); - -void tsdbReaderSetId(STsdbReader* pReader, const char* idstr); +int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); @@ -190,7 +187,10 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader *pReader, STableBlockDistInfo int64_t tsdbGetNumOfRowsInMemTable(STsdbReader *pHandle); void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); -uint64_t getReaderMaxVersion(STsdbReader *pReader); +uint64_t tsdbGetReaderMaxVersion(STsdbReader *pReader); +int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); +void tsdbReaderSetId(STsdbReader *pReader, const char *idstr); +void tsdbReaderSetCloseFlag(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, void *pTableIdList, int32_t numOfTables, int32_t numOfCols, uint64_t suid, void **pReader, const char *idstr); @@ -231,26 +231,21 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - SPackedData msg2; - - SSubmitReq2 submit; - int32_t nextBlk; - - int64_t lastBlkUid; - - SWalReader *pWalReader; - - SMeta *pVnodeMeta; - SHashObj *tbIdHash; - SArray *pColIdList; // SArray - + SPackedData msg2; + SSubmitReq2 submit; + int32_t nextBlk; + int64_t lastBlkUid; + SWalReader *pWalReader; + SMeta *pVnodeMeta; + SHashObj *tbIdHash; + SArray *pColIdList; // SArray int32_t cachedSchemaVer; int64_t cachedSchemaSuid; SSchemaWrapper *pSchemaWrapper; STSchema *pSchema; } STqReader; -STqReader *tqOpenReader(SVnode *pVnode); +STqReader *tqReaderOpen(SVnode *pVnode); void tqCloseReader(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); @@ -266,7 +261,7 @@ int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, i bool tqNextBlockImpl(STqReader *pReader); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); -int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); +int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); int32_t vnodeEnqueueStreamMsg(SVnode *pVnode, SRpcMsg *pMsg); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ae52db163f..7314e23145 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -504,7 +504,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); - pHandle->execHandle.pTqReader = tqOpenReader(pVnode); + pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); @@ -523,7 +523,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - pHandle->execHandle.pTqReader = tqOpenReader(pVnode); + pHandle->execHandle.pTqReader = tqReaderOpen(pVnode); tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index cd8cefb307..f3ecaa08f6 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -328,7 +328,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { } } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); + handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); @@ -343,7 +343,7 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); + handle.execHandle.pTqReader = tqReaderOpen(pTq->pVnode); tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0e9caf24ae..ead00dcc35 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -249,7 +249,7 @@ END: return code; } -STqReader* tqOpenReader(SVnode* pVnode) { +STqReader* tqReaderOpen(SVnode* pVnode) { STqReader* pReader = taosMemoryCalloc(1, sizeof(STqReader)); if (pReader == NULL) { return NULL; @@ -653,7 +653,7 @@ FAIL: return -1; } -int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { +int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index e049e2d390..0f00a5acb8 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -207,7 +207,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; - if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { + if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { @@ -266,7 +266,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; - if (tqRetrieveTaosxBlock2(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { + if (tqRetrieveTaosxBlock(pReader, pBlocks, pSchemas, &pSubmitTbDataRet) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 8d9d1bb887..0f583c021d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1855,7 +1855,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - uint64_t version = getReaderMaxVersion(pTableScanInfo->base.dataReader); + uint64_t version = tsdbGetReaderMaxVersion(pTableScanInfo->base.dataReader); updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->base.cond.twindows, pInfo->groupId, version); pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; checkUpdateData(pInfo, true, pSDB, false); @@ -2021,7 +2021,7 @@ FETCH_NEXT_BLOCK: SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB) { STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - uint64_t version = getReaderMaxVersion(pTableScanInfo->base.dataReader); + uint64_t version = tsdbGetReaderMaxVersion(pTableScanInfo->base.dataReader); updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->base.cond.twindows, pInfo->groupId, version); pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; checkUpdateData(pInfo, true, pSDB, false); @@ -2426,7 +2426,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTqReader) { ASSERT(pHandle->tqReader == NULL); - pInfo->tqReader = tqOpenReader(pHandle->vnode); + pInfo->tqReader = tqReaderOpen(pHandle->vnode); ASSERT(pInfo->tqReader); } else { ASSERT(pHandle->tqReader); From 7762e0ea4711a44d075dd744de6f5b28f492f5eb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 16:10:06 +0800 Subject: [PATCH 087/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 126 ++++++++++++++----------- 1 file changed, 70 insertions(+), 56 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a46f7a7924..66a6df0e7c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -180,6 +180,12 @@ typedef struct STsdbReaderAttr { SVersionRange verRange; } STsdbReaderAttr; +typedef struct SResultBlockInfo { + SSDataBlock* pResBlock; + bool freeBlock; + int64_t capacity; +} SResultBlockInfo; + struct STsdbReader { STsdb* pTsdb; SVersionRange verRange; @@ -187,12 +193,10 @@ struct STsdbReader { bool suspended; uint64_t suid; int16_t order; - bool freeBlock; EReadMode readMode; uint64_t rowsNum; STimeWindow window; // the primary query time window that applies to all queries - SSDataBlock* pResBlock; - int32_t capacity; + SResultBlockInfo resBlockInfo; SReaderStatus status; char* idStr; // query info handle, for debug purpose int32_t type; // query type: 1. retrieve all data blocks, 2. retrieve direct prev|next rows @@ -205,7 +209,7 @@ struct STsdbReader { SDelFReader* pDelFReader; // the del file reader SArray* pDelIdx; // del file block index; SBlockInfoBuf blockInfoBuf; - int32_t step; + EContentData step; STsdbReader* innerReader[2]; }; @@ -727,6 +731,21 @@ void tsdbReleaseDataBlock(STsdbReader* pReader) { } } +static int32_t initResBlockInfo(SResultBlockInfo* pResBlockInfo, int64_t capacity, SSDataBlock* pResBlock, SQueryTableDataCond* pCond) { + pResBlockInfo->capacity = capacity; + pResBlockInfo->pResBlock = pResBlock; + terrno = 0; + + if (pResBlockInfo->pResBlock == NULL) { + pResBlockInfo->freeBlock = true; + pResBlockInfo->pResBlock = createResBlock(pCond, pResBlockInfo->capacity); + } else { + pResBlockInfo->freeBlock = false; + } + + return terrno; +} + static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsdbReader** ppReader, int32_t capacity, SSDataBlock* pResBlock, const char* idstr) { int32_t code = 0; @@ -746,21 +765,16 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->suid = pCond->suid; pReader->order = pCond->order; - pReader->capacity = capacity; - pReader->pResBlock = pResBlock; + pReader->idStr = (idstr != NULL) ? taosStrdup(idstr) : NULL; pReader->verRange = getQueryVerRange(pVnode, pCond, level); pReader->type = pCond->type; pReader->window = updateQueryTimeWindow(pReader->pTsdb, &pCond->twindows); pReader->blockInfoBuf.numPerBucket = 1000; // 1000 tables per bucket - if (pReader->pResBlock == NULL) { - pReader->freeBlock = true; - pReader->pResBlock = createResBlock(pCond, pReader->capacity); - if (pReader->pResBlock == NULL) { - code = terrno; - goto _end; - } + code = initResBlockInfo(&pReader->resBlockInfo, capacity, pResBlock, pCond); + if (code != TSDB_CODE_SUCCESS) { + goto _end; } if (pCond->numOfCols <= 0) { @@ -792,7 +806,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd goto _end; } - pReader->status.pPrimaryTsCol = taosArrayGet(pReader->pResBlock->pDataBlock, pSup->slotId[0]); + pReader->status.pPrimaryTsCol = taosArrayGet(pReader->resBlockInfo.pResBlock->pDataBlock, pSup->slotId[0]); int32_t type = pReader->status.pPrimaryTsCol->info.type; if (type != TSDB_DATA_TYPE_TIMESTAMP) { tsdbError("the first column isn't primary timestamp in result block, actual: %s, %s", tDataTypes[type].name, @@ -1221,7 +1235,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { SBlockData* pBlockData = &pStatus->fileBlockData; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); SDataBlk* pBlock = getCurrentBlock(pBlockIter); - SSDataBlock* pResBlock = pReader->pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; int32_t numOfOutputCols = pSupInfo->numOfCols; int32_t code = TSDB_CODE_SUCCESS; @@ -1269,8 +1283,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader) { endIndex += step; int32_t dumpedRows = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); - if (dumpedRows > pReader->capacity) { // output buffer check - dumpedRows = pReader->capacity; + if (dumpedRows > pReader->resBlockInfo.capacity) { // output buffer check + dumpedRows = pReader->resBlockInfo.capacity; } int32_t i = 0; @@ -1785,7 +1799,7 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->overlapWithLastBlock = !(pBlock->maxKey.ts < tsLast || pBlock->minKey.ts > tsLast); } - pInfo->moreThanCapcity = pBlock->nRow > pReader->capacity; + pInfo->moreThanCapcity = pBlock->nRow > pReader->resBlockInfo.capacity; pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlock, &pReader->verRange); } @@ -1832,10 +1846,10 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* return TSDB_CODE_SUCCESS; } - SSDataBlock* pBlock = pReader->pResBlock; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; int64_t st = taosGetTimestampUs(); - int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader); + int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->resBlockInfo.capacity, pReader); blockDataUpdateTsWindow(pBlock, pReader->suppInfo.slotId[0]); pBlock->info.id.uid = pBlockScanInfo->uid; @@ -1866,7 +1880,7 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB int64_t nextKey = pBlockData->aTSKEY[pDumpInfo->rowIndex + step]; if (nextKey != key) { // merge is not needed - code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, pBlockData, pDumpInfo->rowIndex); if (code) { return code; } @@ -1913,7 +1927,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas if (hasVal) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 != ts) { - code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); if (code) { return code; } @@ -1922,7 +1936,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas return code; } } else { - code = doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + code = doAppendRowFromFileBlock(pReader->resBlockInfo.pResBlock, pReader, fRow->pBlockData, fRow->iRow); if (code) { return code; } @@ -2120,7 +2134,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); @@ -2170,7 +2184,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); @@ -2197,7 +2211,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, return code; } - code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); @@ -2257,7 +2271,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader return code; } - code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); @@ -2475,7 +2489,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); @@ -2658,7 +2672,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc return code; } - code = doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo); + code = doAppendRowFromTSRow(pReader->resBlockInfo.pResBlock, pReader, pTSRow, pBlockScanInfo); taosMemoryFree(pTSRow); tsdbRowMergerClear(pMerger); @@ -2740,7 +2754,7 @@ static int32_t loadNeighborIfOverlap(SFileDataBlockInfo* pBlockInfo, STableBlock } static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlockScanInfo* pBlockScanInfo) { - SSDataBlock* pResBlock = pReader->pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0; pResBlock->info.dataLoad = 1; @@ -2755,7 +2769,7 @@ static void updateComposedBlockInfo(STsdbReader* pReader, double el, STableBlock static int32_t buildComposedDataBlock(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pResBlock = pReader->pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; @@ -2777,7 +2791,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // it is a clean block, load it directly if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader) && - pBlock->nRow <= pReader->capacity) { + pBlock->nRow <= pReader->resBlockInfo.capacity) { if (asc || ((!asc) && (!hasDataInLastBlock(pLastBlockReader)))) { code = copyBlockDataToSDataBlock(pReader); if (code) { @@ -3064,7 +3078,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { return TSDB_CODE_SUCCESS; } - SSDataBlock* pResBlock = pReader->pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; while (1) { // load the last data block of current table @@ -3098,7 +3112,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { return code; } - if (pResBlock->info.rows >= pReader->capacity) { + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { break; } } @@ -3164,7 +3178,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { SBlockData* pBData = &pReader->status.fileBlockData; tBlockDataReset(pBData); - SSDataBlock* pResBlock = pReader->pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr); int64_t st = taosGetTimestampUs(); @@ -3182,7 +3196,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } - if (pResBlock->info.rows >= pReader->capacity) { + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { break; } } @@ -3197,7 +3211,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pResBlock->info.rows, el, pReader->idStr); } } else { // whole block is required, return it directly - SDataBlockInfo* pInfo = &pReader->pResBlock->info; + SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; pInfo->rows = pBlock->nRow; pInfo->id.uid = pScanInfo->uid; pInfo->dataLoad = 0; @@ -3373,7 +3387,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { return code; } - if (pReader->pResBlock->info.rows > 0) { + if (pReader->resBlockInfo.pResBlock->info.rows > 0) { return TSDB_CODE_SUCCESS; } @@ -3456,7 +3470,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } - if (pReader->pResBlock->info.rows > 0) { + if (pReader->resBlockInfo.pResBlock->info.rows > 0) { return TSDB_CODE_SUCCESS; } @@ -3481,7 +3495,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } - if (pReader->pResBlock->info.rows > 0) { + if (pReader->resBlockInfo.pResBlock->info.rows > 0) { return TSDB_CODE_SUCCESS; } } @@ -3534,7 +3548,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { return code; } - if (pReader->pResBlock->info.rows > 0) { + if (pReader->resBlockInfo.pResBlock->info.rows > 0) { return TSDB_CODE_SUCCESS; } } @@ -4173,7 +4187,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S int32_t buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, int64_t endKey, int32_t capacity, STsdbReader* pReader) { - SSDataBlock* pBlock = pReader->pResBlock; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; int32_t code = TSDB_CODE_SUCCESS; do { @@ -4281,7 +4295,7 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { return metaGetIvtIdx(pMeta); } -uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } +uint64_t tsdbGetReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } static int32_t doOpenReaderImpl(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; @@ -4484,8 +4498,8 @@ void tsdbReaderClose(STsdbReader* pReader) { } } - if (pReader->freeBlock) { - pReader->pResBlock = blockDataDestroy(pReader->pResBlock); + if (pReader->resBlockInfo.freeBlock) { + pReader->resBlockInfo.pResBlock = blockDataDestroy(pReader->resBlockInfo.pResBlock); } taosMemoryFree(pSupInfo->colId); @@ -4622,7 +4636,7 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { pBlockScanInfo = pStatus->pTableIter == NULL ? NULL : *pStatus->pTableIter; if (pBlockScanInfo) { // save lastKey to restore memory iterator - STimeWindow w = pReader->pResBlock->info.window; + STimeWindow w = pReader->resBlockInfo.pResBlock->info.window; pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->order) ? w.ekey : w.skey; // reset current current table's data block scan info, @@ -4707,10 +4721,10 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { STsdbReader* pNextReader = pReader->innerReader[1]; // we need only one row - pPrevReader->capacity = 1; + pPrevReader->resBlockInfo.capacity = 1; setSharedPtr(pPrevReader, pReader); - pNextReader->capacity = 1; + pNextReader->resBlockInfo.capacity = 1; setSharedPtr(pNextReader, pReader); code = doOpenReaderImpl(pPrevReader); @@ -4733,7 +4747,7 @@ _err: static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; - SSDataBlock* pBlock = pReader->pResBlock; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; if (pReader->status.loadFromFile == false) { return false; @@ -4762,7 +4776,7 @@ static int32_t doTsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { int32_t code = TSDB_CODE_SUCCESS; // cleanup the data that belongs to the previous data block - SSDataBlock* pBlock = pReader->pResBlock; + SSDataBlock* pBlock = pReader->resBlockInfo.pResBlock; blockDataCleanup(pBlock); *hasNext = false; @@ -4947,7 +4961,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - if (pReader->pResBlock->info.id.uid != pFBlock->uid) { + if (pReader->resBlockInfo.pResBlock->info.id.uid != pFBlock->uid) { return TSDB_CODE_SUCCESS; } @@ -4973,8 +4987,8 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, pTsAgg->numOfNull = 0; pTsAgg->colId = PRIMARYKEY_TIMESTAMP_COL_ID; - pTsAgg->min = pReader->pResBlock->info.window.skey; - pTsAgg->max = pReader->pResBlock->info.window.ekey; + pTsAgg->min = pReader->resBlockInfo.pResBlock->info.window.skey; + pTsAgg->max = pReader->resBlockInfo.pResBlock->info.window.ekey; // update the number of NULL data rows size_t numOfCols = pSup->numOfCols; @@ -4985,7 +4999,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, taosArrayEnsureCap(pSup->pColAgg, colsNum); } - SSDataBlock* pResBlock = pReader->pResBlock; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; if (pResBlock->pBlockAgg == NULL) { size_t num = taosArrayGetSize(pResBlock->pDataBlock); pResBlock->pBlockAgg = taosMemoryCalloc(num, POINTER_BYTES); @@ -5056,7 +5070,7 @@ static SSDataBlock* doRetrieveDataBlock(STsdbReader* pReader) { return NULL; } - return pReader->pResBlock; + return pReader->resBlockInfo.pResBlock; } SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { @@ -5071,7 +5085,7 @@ SSDataBlock* tsdbRetrieveDataBlock(STsdbReader* pReader, SArray* pIdList) { SReaderStatus* pStatus = &pTReader->status; if (pStatus->composedDataBlock) { - return pTReader->pResBlock; + return pTReader->resBlockInfo.pResBlock; } SSDataBlock* ret = doRetrieveDataBlock(pTReader); From e822dc20250b5eb5d63f3734dc59e9e52a2e6f9d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 16:10:36 +0800 Subject: [PATCH 088/145] enh(query): add reader status. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 29 +++++++++++++++----------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 66a6df0e7c..2d27a6529f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -20,6 +20,12 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) +typedef enum { + READER_STATUS_SUSPEND = 0x1, + READER_STATUS_SHOULD_STOP = 0x2, + READER_STATUS_NORMAL = 0x3, +} EReaderExecStatus; + typedef enum { EXTERNAL_ROWS_PREV = 0x1, EXTERNAL_ROWS_MAIN = 0x2, @@ -190,7 +196,8 @@ struct STsdbReader { STsdb* pTsdb; SVersionRange verRange; TdThreadMutex readerMutex; - bool suspended; + EReaderExecStatus flag; +// bool suspended; uint64_t suid; int16_t order; EReadMode readMode; @@ -2855,7 +2862,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { break; } - if (pResBlock->info.rows >= pReader->capacity) { + if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) { break; } } @@ -4431,7 +4438,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL goto _err; } - pReader->suspended = true; + pReader->flag = READER_STATUS_SUSPEND; if (countOnly) { pReader->readMode = READ_MODE_COUNT_ONLY; @@ -4660,8 +4667,7 @@ int32_t tsdbReaderSuspend(STsdbReader* pReader) { tsdbUntakeReadSnap(pReader, pReader->pReadSnap, false); pReader->pReadSnap = NULL; - - pReader->suspended = true; + pReader->flag = READER_STATUS_SUSPEND; tsdbDebug("reader: %p suspended uid %" PRIu64 " in this query %s", pReader, pBlockScanInfo ? pBlockScanInfo->uid : 0, pReader->idStr); @@ -4678,7 +4684,7 @@ static int32_t tsdbSetQueryReseek(void* pQHandle) { code = tsdbTryAcquireReader(pReader); if (code == 0) { - if (pReader->suspended) { + if (pReader->flag == READER_STATUS_SUSPEND) { tsdbReleaseReader(pReader); return code; } @@ -4734,8 +4740,7 @@ int32_t tsdbReaderResume(STsdbReader* pReader) { } } - pReader->suspended = false; - + pReader->flag = READER_STATUS_NORMAL; tsdbDebug("reader: %p resumed uid %" PRIu64 ", numOfTable:%" PRId32 ", in this query %s", pReader, pBlockScanInfo ? (*pBlockScanInfo)->uid : 0, numOfTables, pReader->idStr); return code; @@ -4823,7 +4828,7 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { code = tsdbAcquireReader(pReader); qTrace("tsdb/read: %p, take read mutex, code: %d", pReader, code); - if (pReader->suspended) { + if (pReader->flag == READER_STATUS_SUSPEND) { tsdbReaderResume(pReader); } @@ -5100,7 +5105,7 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { qTrace("tsdb/reader-reset: %p, take read mutex", pReader); tsdbAcquireReader(pReader); - if (pReader->suspended) { + if (pReader->flag == READER_STATUS_SUSPEND) { tsdbReaderResume(pReader); } @@ -5181,7 +5186,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa // find the start data block in file tsdbAcquireReader(pReader); - if (pReader->suspended) { + if (pReader->flag == READER_STATUS_SUSPEND) { tsdbReaderResume(pReader); } SReaderStatus* pStatus = &pReader->status; @@ -5254,7 +5259,7 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; tsdbAcquireReader(pReader); - if (pReader->suspended) { + if (pReader->flag == READER_STATUS_SUSPEND) { tsdbReaderResume(pReader); } From 802c05150a49577ea6203e7ad5da2ff5156ee8b6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 16:18:28 +0800 Subject: [PATCH 089/145] enh(query): stop tsdb reader ASAP. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 2d27a6529f..eb15400d05 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -197,7 +197,6 @@ struct STsdbReader { SVersionRange verRange; TdThreadMutex readerMutex; EReaderExecStatus flag; -// bool suspended; uint64_t suid; int16_t order; EReadMode readMode; @@ -2995,9 +2994,15 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); while (1) { + // only check here, since the iterate data in memory is very fast. + if (pReader->flag == READER_STATUS_SHOULD_STOP) { + tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr); + return TSDB_CODE_SUCCESS; + } + bool hasNext = false; int32_t code = filesetIteratorNext(&pStatus->fileIter, pReader, &hasNext); - if (code) { + if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); return code; } @@ -3088,6 +3093,11 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; while (1) { + if (pReader->flag == READER_STATUS_SHOULD_STOP) { + tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr); + return TSDB_CODE_SUCCESS; + } + // load the last data block of current table STableBlockScanInfo* pScanInfo = *(STableBlockScanInfo**)pStatus->pTableIter; @@ -5425,3 +5435,5 @@ void tsdbReaderSetId(STsdbReader* pReader, const char* idstr) { taosMemoryFreeClear(pReader->idStr); pReader->idStr = taosStrdup(idstr); } + +void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->flag = READER_STATUS_SHOULD_STOP; } From 6052855ef55b754bd5e0401199edcbc3ca4582c0 Mon Sep 17 00:00:00 2001 From: xiaolei li <85657333+xleili@users.noreply.github.com> Date: Thu, 27 Apr 2023 17:19:32 +0800 Subject: [PATCH 090/145] release: upgrade default version to 3.0.4.1 (#21110) --- cmake/cmake.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 232e86d891..3166a0695c 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.4.0") + SET(TD_VER_NUMBER "3.0.4.1") ENDIF () IF (DEFINED VERCOMPATIBLE) From db5b5c828e602ddc3211e2a10af5f5de72bf74b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 27 Apr 2023 18:35:10 +0800 Subject: [PATCH 091/145] fix:add log for msg push --- source/client/src/clientTmq.c | 6 +++--- source/dnode/vnode/src/tq/tq.c | 34 ++++++++++++++++-------------- source/dnode/vnode/src/tq/tqUtil.c | 5 +++-- utils/test/c/tmqSim.c | 2 +- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 76384fbe6a..33324552dc 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1377,7 +1377,7 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic tstrncpy(pTopic->topicName, pTopicEp->topic, TSDB_TOPIC_FNAME_LEN); tstrncpy(pTopic->db, pTopicEp->db, TSDB_DB_FNAME_LEN); - tscDebug("consumer:0x%" PRIx64 ", update topic:%s, numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); + tscDebug("consumer:0x%" PRIx64 ", update topic:%s, new numOfVgs:%d", tmq->consumerId, pTopic->topicName, vgNumGet); pTopic->vgs = taosArrayInit(vgNumGet, sizeof(SMqClientVg)); for (int32_t j = 0; j < vgNumGet; j++) { @@ -1447,14 +1447,14 @@ static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) SMqClientTopic* pTopicCur = taosArrayGet(tmq->clientTopics, i); if (pTopicCur->vgs) { int32_t vgNumCur = taosArrayGetSize(pTopicCur->vgs); - tscDebug("consumer:0x%" PRIx64 ", new vg num: %d", tmq->consumerId, vgNumCur); + tscDebug("consumer:0x%" PRIx64 ", current vg num: %d", tmq->consumerId, vgNumCur); for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); char buf[80]; tFormatOffset(buf, 80, &pVgCur->currentOffset); - tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, + tscDebug("consumer:0x%" PRIx64 ", doUpdateLocalEp current vg, epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, tmq->epoch, pVgCur->vgId, vgKey, buf); SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows}; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 00684652f0..7f082c748b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -553,8 +553,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // remove if it has been register in the push manager, and return one empty block to consumer // tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); - taosHashRemove(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t)); - + int32_t ret = taosHashRemove(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t)); + tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); if(pHandle->msg != NULL) { rpcFreeCont(pHandle->msg->pCont); taosMemoryFree(pHandle->msg); @@ -1069,22 +1069,24 @@ int32_t tqProcessSubmitReqForSubscribe(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); taosWLockLatch(&pTq->lock); - void *pIter = taosHashIterate(pTq->pPushMgr, NULL); - while(pIter){ - STqHandle* pHandle = *(STqHandle**)pIter; - tqDebug("vgId:%d start set submit for pHandle:%p", vgId, pHandle); - if(ASSERT(pHandle->msg != NULL)){ - tqError("pHandle->msg should not be null"); - break; - }else{ - SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; - tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); - taosMemoryFree(pHandle->msg); - pHandle->msg = NULL; + if(taosHashGetSize(pTq->pPushMgr) > 0){ + void *pIter = taosHashIterate(pTq->pPushMgr, NULL); + while(pIter){ + STqHandle* pHandle = *(STqHandle**)pIter; + tqDebug("vgId:%d start set submit for pHandle:%p, consume id:0x%"PRIx64, vgId, pHandle, pHandle->consumerId); + if(ASSERT(pHandle->msg != NULL)){ + tqError("pHandle->msg should not be null"); + break; + }else{ + SRpcMsg msg = {.msgType = TDMT_VND_TMQ_CONSUME, .pCont = pHandle->msg->pCont, .contLen = pHandle->msg->contLen, .info = pHandle->msg->info}; + tmsgPutToQueue(&pTq->pVnode->msgCb, QUERY_QUEUE, &msg); + taosMemoryFree(pHandle->msg); + pHandle->msg = NULL; + } + pIter = taosHashIterate(pTq->pPushMgr, pIter); } - pIter = taosHashIterate(pTq->pPushMgr, pIter); + taosHashClear(pTq->pPushMgr); } - taosHashClear(pTq->pPushMgr); // unlock taosWUnLockLatch(&pTq->lock); diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 2398ef41f4..da8dc1d379 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -190,8 +190,9 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); pHandle->msg->contLen = pMsg->contLen; - tqDebug("data is over, register to handle:%p, pCont:%p, len:%d", pHandle, pHandle->msg->pCont, pHandle->msg->contLen); - taosHashPut(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t), &pHandle, POINTER_BYTES); + int32_t ret = taosHashPut(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t), &pHandle, POINTER_BYTES); + tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen); + taosWUnLockLatch(&pTq->lock); tDeleteSMqDataRsp(&dataRsp); return code; diff --git a/utils/test/c/tmqSim.c b/utils/test/c/tmqSim.c index f2de219f4e..d98a45f0d3 100644 --- a/utils/test/c/tmqSim.c +++ b/utils/test/c/tmqSim.c @@ -232,7 +232,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]); } taosFprintfFile(g_fp, "\n"); - taosFprintfFile(g_fp, " expect rows: %" PRIx64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt); + taosFprintfFile(g_fp, " expect rows: %" PRId64 "\n", g_stConfInfo.stThreads[i].expectMsgCnt); } char tmpString[128]; From 25010cb11ed04cae25536168e87c4024d12201f2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 27 Apr 2023 22:21:18 +0800 Subject: [PATCH 092/145] fix(query): stop tsdbreade ASAP, and do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 219 +--------------- source/libs/executor/src/operator.c | 323 ++++++++++++++++++++++++ 3 files changed, 328 insertions(+), 215 deletions(-) create mode 100644 source/libs/executor/src/operator.c diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 85424fd7de..6bace8092a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -819,6 +819,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); int32_t checkForQueryBuf(size_t numOfTables); +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); bool isTaskKilled(SExecTaskInfo* pTaskInfo); void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 7594079cfb..b67a475244 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -75,8 +75,6 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); -static void releaseQueryBuf(size_t numOfTables); - static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); @@ -88,43 +86,6 @@ static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo); static SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); -void setOperatorCompleted(SOperatorInfo* pOperator) { - pOperator->status = OP_EXEC_DONE; - pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0; - setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); -} - -void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, - void* pInfo, SExecTaskInfo* pTaskInfo) { - pOperator->name = (char*)name; - pOperator->operatorType = type; - pOperator->blocking = blocking; - pOperator->status = status; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; -} - -int32_t optrDummyOpenFn(SOperatorInfo* pOperator) { - OPTR_SET_OPENED(pOperator); - pOperator->cost.openCost = 0; - return TSDB_CODE_SUCCESS; -} - -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, - __optr_explain_fn_t explain) { - SOperatorFpSet fpSet = { - ._openFn = openFn, - .getNextFn = nextFn, - .cleanupFn = cleanup, - .closeFn = closeFn, - .reqBufFn = reqBufFn, - .getExplainFn = explain, - }; - - return fpSet; -} - SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) { SFilePage* pData = NULL; @@ -484,7 +445,10 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); } -void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { pTaskInfo->code = rspCode; } +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { + pTaskInfo->code = rspCode; + stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str); +} ///////////////////////////////////////////////////////////////////////////////////////////// STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { @@ -949,72 +913,6 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG } } -int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) { - p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); - if (p->pDownstream == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - memcpy(p->pDownstream, pDownstream, num * POINTER_BYTES); - p->numOfDownstream = num; - return TSDB_CODE_SUCCESS; -} - -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder) { - // todo add more information about exchange operation - int32_t type = pOperator->operatorType; - if (type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || - type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || - type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { - *order = TSDB_ORDER_ASC; - *scanFlag = MAIN_SCAN; - return TSDB_CODE_SUCCESS; - } else if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { - if (!inheritUsOrder) { - *order = TSDB_ORDER_ASC; - } - *scanFlag = MAIN_SCAN; - return TSDB_CODE_SUCCESS; - } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { - STableScanInfo* pTableScanInfo = pOperator->info; - *order = pTableScanInfo->base.cond.order; - *scanFlag = pTableScanInfo->base.scanFlag; - return TSDB_CODE_SUCCESS; - } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { - STableMergeScanInfo* pTableScanInfo = pOperator->info; - *order = pTableScanInfo->base.cond.order; - *scanFlag = pTableScanInfo->base.scanFlag; - return TSDB_CODE_SUCCESS; - } else { - if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { - return TSDB_CODE_INVALID_PARA; - } else { - return getTableScanInfo(pOperator->pDownstream[0], order, scanFlag, inheritUsOrder); - } - } -} - -// QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN -SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) { - if (pOperator == NULL) { - qError("invalid operator, failed to find tableScanOperator %s", id); - terrno = TSDB_CODE_PAR_INTERNAL_ERROR; - return NULL; - } - - if (pOperator->operatorType == type) { - return pOperator; - } else { - if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { - qError("invalid operator, failed to find tableScanOperator %s", id); - terrno = TSDB_CODE_PAR_INTERNAL_ERROR; - return NULL; - } - - return extractOperatorInTree(pOperator->pDownstream[0], type, id); - } -} - void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { for (int32_t i = 0; i < numOfExprs; ++i) { SExprInfo* pExprInfo = &pExpr[i]; @@ -1031,37 +929,6 @@ void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs) { } } -void destroyOperatorInfo(SOperatorInfo* pOperator) { - if (pOperator == NULL) { - return; - } - - if (pOperator->fpSet.closeFn != NULL) { - pOperator->fpSet.closeFn(pOperator->info); - } - - if (pOperator->pDownstream != NULL) { - for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - destroyOperatorInfo(pOperator->pDownstream[i]); - } - - taosMemoryFreeClear(pOperator->pDownstream); - pOperator->numOfDownstream = 0; - } - - cleanupExprSupp(&pOperator->exprSupp); - taosMemoryFreeClear(pOperator); -} - -// each operator should be set their own function to return total cost buffer -int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { - if (pOperator->blocking) { - return -1; - } else { - return 0; - } -} - int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz) { *defaultPgsz = 4096; while (*defaultPgsz < rowSize * 4) { @@ -1555,45 +1422,6 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOptr; } -static int32_t extractTbscanInStreamOpTree(SOperatorInfo* pOperator, STableScanInfo** ppInfo) { - if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - if (pOperator->numOfDownstream == 0) { - qError("failed to find stream scan operator"); - return TSDB_CODE_APP_ERROR; - } - - if (pOperator->numOfDownstream > 1) { - qError("join not supported for stream block scan"); - return TSDB_CODE_APP_ERROR; - } - return extractTbscanInStreamOpTree(pOperator->pDownstream[0], ppInfo); - } else { - SStreamScanInfo* pInfo = pOperator->info; - *ppInfo = pInfo->pTableScanOp->info; - return 0; - } -} - -int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { - if (pNode->pChildren == NULL || LIST_LENGTH(pNode->pChildren) == 0) { - if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == pNode->type) { - *ppNode = (STableScanPhysiNode*)pNode; - return 0; - } else { - terrno = TSDB_CODE_APP_ERROR; - return -1; - } - } else { - if (LIST_LENGTH(pNode->pChildren) != 1) { - terrno = TSDB_CODE_APP_ERROR; - return -1; - } - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pNode->pChildren, 0); - return extractTableScanNode(pChildNode, ppNode); - } - return -1; -} - int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle) { switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { @@ -1696,45 +1524,6 @@ void doDestroyTask(SExecTaskInfo* pTaskInfo) { taosMemoryFreeClear(pTaskInfo); } -static int64_t getQuerySupportBufSize(size_t numOfTables) { - size_t s1 = sizeof(STableQueryInfo); - // size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb - return (int64_t)(s1 * 1.5 * numOfTables); -} - -int32_t checkForQueryBuf(size_t numOfTables) { - int64_t t = getQuerySupportBufSize(numOfTables); - if (tsQueryBufferSizeBytes < 0) { - return TSDB_CODE_SUCCESS; - } else if (tsQueryBufferSizeBytes > 0) { - while (1) { - int64_t s = tsQueryBufferSizeBytes; - int64_t remain = s - t; - if (remain >= 0) { - if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { - return TSDB_CODE_SUCCESS; - } - } else { - return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; - } - } - } - - // disable query processing if the value of tsQueryBufferSize is zero. - return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; -} - -void releaseQueryBuf(size_t numOfTables) { - if (tsQueryBufferSizeBytes < 0) { - return; - } - - int64_t t = getQuerySupportBufSize(numOfTables); - - // restore value is not enough buffer available - atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); -} - int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList) { SExplainExecInfo execInfo = {0}; SExplainExecInfo* pExplainInfo = taosArrayPush(pExecInfoList, &execInfo); diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c new file mode 100644 index 0000000000..b1563317cf --- /dev/null +++ b/source/libs/executor/src/operator.c @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" + +#include "tdatablock.h" +#include "tglobal.h" +#include "tmsg.h" +#include "ttime.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, + __optr_explain_fn_t explain) { + SOperatorFpSet fpSet = { + ._openFn = openFn, + .getNextFn = nextFn, + .cleanupFn = cleanup, + .closeFn = closeFn, + .reqBufFn = reqBufFn, + .getExplainFn = explain, + }; + + return fpSet; +} + +int32_t optrDummyOpenFn(SOperatorInfo* pOperator) { + OPTR_SET_OPENED(pOperator); + pOperator->cost.openCost = 0; + return TSDB_CODE_SUCCESS; +} + +int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num) { + p->pDownstream = taosMemoryCalloc(1, num * POINTER_BYTES); + if (p->pDownstream == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + memcpy(p->pDownstream, pDownstream, num * POINTER_BYTES); + p->numOfDownstream = num; + return TSDB_CODE_SUCCESS; +} + +void setOperatorCompleted(SOperatorInfo* pOperator) { + pOperator->status = OP_EXEC_DONE; + pOperator->cost.totalCost = (taosGetTimestampUs() - pOperator->pTaskInfo->cost.start) / 1000.0; + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); +} + +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, + void* pInfo, SExecTaskInfo* pTaskInfo) { + pOperator->name = (char*)name; + pOperator->operatorType = type; + pOperator->blocking = blocking; + pOperator->status = status; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; +} + +void destroyOperatorInfo(SOperatorInfo* pOperator) { + if (pOperator == NULL) { + return; + } + + if (pOperator->fpSet.closeFn != NULL) { + pOperator->fpSet.closeFn(pOperator->info); + } + + if (pOperator->pDownstream != NULL) { + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + destroyOperatorInfo(pOperator->pDownstream[i]); + } + + taosMemoryFreeClear(pOperator->pDownstream); + pOperator->numOfDownstream = 0; + } + + cleanupExprSupp(&pOperator->exprSupp); + taosMemoryFreeClear(pOperator); +} + +// each operator should be set their own function to return total cost buffer +int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { + if (pOperator->blocking) { + return -1; + } else { + return 0; + } +} + +//int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder) { +// // todo add more information about exchange operation +// int32_t type = pOperator->operatorType; +// if (type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || +// type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || +// type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { +// *order = TSDB_ORDER_ASC; +// *scanFlag = MAIN_SCAN; +// return TSDB_CODE_SUCCESS; +// } else if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { +// if (!inheritUsOrder) { +// *order = TSDB_ORDER_ASC; +// } +// *scanFlag = MAIN_SCAN; +// return TSDB_CODE_SUCCESS; +// } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { +// STableScanInfo* pTableScanInfo = pOperator->info; +// *order = pTableScanInfo->base.cond.order; +// *scanFlag = pTableScanInfo->base.scanFlag; +// return TSDB_CODE_SUCCESS; +// } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { +// STableMergeScanInfo* pTableScanInfo = pOperator->info; +// *order = pTableScanInfo->base.cond.order; +// *scanFlag = pTableScanInfo->base.scanFlag; +// return TSDB_CODE_SUCCESS; +// } else { +// if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { +// return TSDB_CODE_INVALID_PARA; +// } else { +// return getTableScanInfo(pOperator->pDownstream[0], order, scanFlag, inheritUsOrder); +// } +// } +//} + +static int64_t getQuerySupportBufSize(size_t numOfTables) { + size_t s1 = sizeof(STableQueryInfo); + // size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb + return (int64_t)(s1 * 1.5 * numOfTables); +} + +int32_t checkForQueryBuf(size_t numOfTables) { + int64_t t = getQuerySupportBufSize(numOfTables); + if (tsQueryBufferSizeBytes < 0) { + return TSDB_CODE_SUCCESS; + } else if (tsQueryBufferSizeBytes > 0) { + while (1) { + int64_t s = tsQueryBufferSizeBytes; + int64_t remain = s - t; + if (remain >= 0) { + if (atomic_val_compare_exchange_64(&tsQueryBufferSizeBytes, s, remain) == s) { + return TSDB_CODE_SUCCESS; + } + } else { + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; + } + } + } + + // disable query processing if the value of tsQueryBufferSize is zero. + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; +} + +void releaseQueryBuf(size_t numOfTables) { + if (tsQueryBufferSizeBytes < 0) { + return; + } + + int64_t t = getQuerySupportBufSize(numOfTables); + + // restore value is not enough buffer available + atomic_add_fetch_64(&tsQueryBufferSizeBytes, t); +} + +typedef enum { + OPTR_FN_RET_CONTINUE = 0x1, + OPTR_FN_RET_ABORT = 0x2, +} ERetType; + +typedef struct STraverParam { + void* pRet; + int32_t code; + void* pParam; +} STraverParam; + +// iterate the operator tree helper +typedef ERetType (*optr_fn_t)(SOperatorInfo *pOperator, STraverParam *pParam, const char* pIdstr); + +void traverseOperatorTree(SOperatorInfo* pOperator, optr_fn_t fn, STraverParam* pParam, const char* id) { + if (pOperator == NULL) { + return; + } + + ERetType ret = fn(pOperator, pParam, id); + if (ret == OPTR_FN_RET_ABORT || pParam->code != TSDB_CODE_SUCCESS) { + return; + } + + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + traverseOperatorTree(pOperator->pDownstream[i], fn, pParam, id); + if (pParam->code != 0) { + break; + } + } +} + +ERetType extractOperatorInfo(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) { + STraverParam* p = pParam; + if (pOperator->operatorType == *(int32_t*)p->pParam) { + p->pRet = pOperator; + return OPTR_FN_RET_ABORT; + } else { + return OPTR_FN_RET_CONTINUE; + } +} + +// QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN +SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) { + if (pOperator == NULL) { + qError("invalid operator, failed to find tableScanOperator %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; + return NULL; + } + + STraverParam p = {.pParam = &type, .pRet = NULL}; + traverseOperatorTree(pOperator, extractOperatorInfo, &p, id); + if (p.code != 0) { + terrno = p.code; + return NULL; + } else { + return p.pRet; + } +} + +typedef struct SExtScanInfo { + int32_t order; + int32_t scanFlag; + int32_t inheritUsOrder; +} SExtScanInfo; + +static ERetType extractScanInfo(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) { + int32_t type = pOperator->operatorType; + SExtScanInfo* pInfo = pParam->pParam; + + if (type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || + type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || + type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { + pInfo->order = TSDB_ORDER_ASC; + pInfo->scanFlag= MAIN_SCAN; + return OPTR_FN_RET_ABORT; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { + if (!pInfo->inheritUsOrder) { + pInfo->order = TSDB_ORDER_ASC; + } + pInfo->scanFlag= MAIN_SCAN; + return OPTR_FN_RET_ABORT; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pTableScanInfo = pOperator->info; + pInfo->order = pTableScanInfo->base.cond.order; + pInfo->scanFlag= pTableScanInfo->base.scanFlag; + return OPTR_FN_RET_ABORT; + } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { + STableMergeScanInfo* pTableScanInfo = pOperator->info; + pInfo->order = pTableScanInfo->base.cond.order; + pInfo->scanFlag= pTableScanInfo->base.scanFlag; + return OPTR_FN_RET_ABORT; + } else { + return OPTR_FN_RET_CONTINUE; + } +} + +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder) { + SExtScanInfo info = {.inheritUsOrder = inheritUsOrder}; + STraverParam p = {.pParam = &info}; + + extractScanInfo(pOperator, &p, NULL); + *order = info.order; + *scanFlag = info.scanFlag; + + return p.code; +} + +static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, const char* pIdStr) { + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pInfo = pOperator->info; + + if (pInfo->base.dataReader != NULL) { + tsdbReaderSetCloseFlag(pInfo->base.dataReader); + } + return OPTR_FN_RET_ABORT; + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pOperator->info; + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + + if (pTableScanInfo->base.dataReader != NULL) { + tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader); + } + + return OPTR_FN_RET_ABORT; + } + + return OPTR_FN_RET_CONTINUE; +} + +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) { + STraverParam p = {0}; + traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr); + return p.code; +} From 1ee1b0422c710f035f2e2dcfc9d00eac030b1e19 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 27 Apr 2023 23:44:27 +0800 Subject: [PATCH 093/145] fix:change push mgr from SArray to Hash --- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/inc/vnodeInt.h | 5 ++-- source/dnode/vnode/src/tq/tq.c | 28 ++++++++++------------- source/dnode/vnode/src/tq/tqPush.c | 33 +++++++++++++++++++++++++++ source/dnode/vnode/src/tq/tqUtil.c | 14 +----------- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 080e72c504..1b29b34073 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -147,7 +147,7 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type); -int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); +int32_t tqPushDataRsp(STQ* pTq, STqHandle* pHandle); // tqMeta int32_t tqMetaOpen(STQ* pTq); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index b24cb7e136..7668d45108 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -193,9 +193,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode); void tqNotifyClose(STQ*); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); -int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, - int32_t type); -int tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); +int tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg); +int tqUnregisterPushHandle(STQ* pTq, void* pHandle); int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. int tqCommit(STQ*); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7f082c748b..53a40eb839 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -98,7 +98,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { taosHashSetFreeFp(pTq->pHandle, destroyTqHandle); taosInitRWLatch(&pTq->lock); - pTq->pPushMgr = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + pTq->pPushMgr = taosHashInit(64, MurmurHash3_32, false, HASH_NO_LOCK); pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); @@ -220,17 +220,19 @@ static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqData return 0; } -int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { - SMqDataRsp* pRsp = pPushEntry->pDataRsp; - SMqRspHead* pHeader = &pPushEntry->pDataRsp->head; - doSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType); +int32_t tqPushDataRsp(STQ* pTq, STqHandle* pHandle) { + SMqDataRsp dataRsp = {0}; + dataRsp.head.consumerId = pHandle->consumerId; + dataRsp.head.epoch = pHandle->epoch; + dataRsp.head.mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + doSendDataRsp(&pHandle->msg->info, &dataRsp, pHandle->epoch, pHandle->consumerId, TMQ_MSG_TYPE__POLL_RSP); char buf1[80] = {0}; char buf2[80] = {0}; - tFormatOffset(buf1, tListLen(buf1), &pRsp->reqOffset); - tFormatOffset(buf2, tListLen(buf2), &pRsp->rspOffset); + tFormatOffset(buf1, tListLen(buf1), &dataRsp.reqOffset); + tFormatOffset(buf2, tListLen(buf2), &dataRsp.rspOffset); tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", - TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); + TD_VID(pTq->pVnode), dataRsp.head.consumerId, dataRsp.head.epoch, dataRsp.blockNum, buf1, buf2); return 0; } @@ -552,14 +554,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg atomic_store_32(&pHandle->epoch, -1); // remove if it has been register in the push manager, and return one empty block to consumer -// tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); - int32_t ret = taosHashRemove(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t)); - tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); - if(pHandle->msg != NULL) { - rpcFreeCont(pHandle->msg->pCont); - taosMemoryFree(pHandle->msg); - pHandle->msg = NULL; - } + tqUnregisterPushHandle(pTq, pHandle); + atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index dca988cbbd..1ee19b7a7b 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -300,3 +300,36 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v return 0; } + + +int32_t tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg) { + int32_t vgId = TD_VID(pTq->pVnode); + STqHandle* pHandle = (STqHandle*) handle; + if(pHandle->msg == NULL){ + pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + } + + memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); + pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); + memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); + pHandle->msg->contLen = pMsg->contLen; + int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES); + tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen); + return 0; +} + +int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) { + STqHandle *pHandle = (STqHandle*)handle; + int32_t vgId = TD_VID(pTq->pVnode); + + int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); + tqError("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); + if(pHandle->msg != NULL) { + tqPushDataRsp(pTq, pHandle); + + rpcFreeCont(pHandle->msg->pCont); + taosMemoryFree(pHandle->msg); + pHandle->msg = NULL; + } + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index da8dc1d379..1f06132d2f 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -178,21 +178,9 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, // till now, all data has been transferred to consumer, new data needs to push client once arrived. if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { -// code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); // lock taosWLockLatch(&pTq->lock); -// tqDebug("data is over, register to handle:%p, msg:%p", pHandle, pHandle->msg); - if(pHandle->msg == NULL){ - pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg)); - } - - memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); - pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); - memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); - pHandle->msg->contLen = pMsg->contLen; - int32_t ret = taosHashPut(pTq->pPushMgr, &pHandle->consumerId, sizeof(int64_t), &pHandle, POINTER_BYTES); - tqDebug("vgId:%d data is over, ret:%d, consumerId:0x%" PRIx64", register to pHandle:%p, pCont:%p, len:%d", vgId, ret, pHandle->consumerId, pHandle, pHandle->msg->pCont, pHandle->msg->contLen); - + code = tqRegisterPushEntry(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); tDeleteSMqDataRsp(&dataRsp); return code; From 7adc21028aac6f4208c0be2bfe7341625d424850 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 00:23:38 +0800 Subject: [PATCH 094/145] refactor: do some internal refactor. --- source/libs/executor/inc/executil.h | 2 - source/libs/executor/inc/executorimpl.h | 207 +------- source/libs/executor/inc/operator.h | 165 ++++++ source/libs/executor/inc/querytask.h | 83 ++++ source/libs/executor/src/aggregateoperator.c | 7 +- source/libs/executor/src/cachescanoperator.c | 2 + .../libs/executor/src/eventwindowoperator.c | 2 + source/libs/executor/src/exchangeoperator.c | 2 + source/libs/executor/src/executil.c | 1 + source/libs/executor/src/executor.c | 5 +- source/libs/executor/src/executorimpl.c | 469 +----------------- source/libs/executor/src/filloperator.c | 4 +- source/libs/executor/src/groupoperator.c | 2 + source/libs/executor/src/joinoperator.c | 2 + source/libs/executor/src/operator.c | 286 ++++++++++- source/libs/executor/src/projectoperator.c | 2 + source/libs/executor/src/querytask.c | 240 +++++++++ source/libs/executor/src/scanoperator.c | 4 +- source/libs/executor/src/sortoperator.c | 2 + source/libs/executor/src/sysscanoperator.c | 3 + source/libs/executor/src/timesliceoperator.c | 2 + source/libs/executor/src/timewindowoperator.c | 8 +- source/libs/executor/test/executorTests.cpp | 5 +- 23 files changed, 808 insertions(+), 697 deletions(-) create mode 100644 source/libs/executor/inc/operator.h create mode 100644 source/libs/executor/inc/querytask.h create mode 100644 source/libs/executor/src/querytask.c diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 2e92f9e396..5a9f079c15 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -39,8 +39,6 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) -#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) - typedef struct SGroupResInfo { int32_t index; SArray* pRows; // SArray diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 6bace8092a..4d19b8bb76 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -85,39 +85,6 @@ typedef struct SLimit { typedef struct STableScanAnalyzeInfo SFileBlockLoadRecorder; -typedef struct STaskCostInfo { - int64_t created; - int64_t start; - uint64_t elapsedTime; - double extractListTime; - double groupIdMapTime; - SFileBlockLoadRecorder* pRecoder; -} STaskCostInfo; - -typedef struct SOperatorCostInfo { - double openCost; - double totalCost; -} SOperatorCostInfo; - -struct SOperatorInfo; - -typedef int32_t (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char** result, int32_t* length); -typedef int32_t (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char* result); - -typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); -typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); -typedef void (*__optr_close_fn_t)(void* param); -typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); -typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr); - -typedef struct STaskIdInfo { - uint64_t queryId; // this is also a request id - uint64_t subplanId; - uint64_t templateId; - char* str; - int32_t vgId; -} STaskIdInfo; - enum { STREAM_RECOVER_STEP__NONE = 0, STREAM_RECOVER_STEP__PREPARE1, @@ -156,51 +123,6 @@ typedef struct SExchangeOpStopInfo { int64_t refId; } SExchangeOpStopInfo; -typedef struct STaskStopInfo { - SRWLatch lock; - SArray* pStopInfo; -} STaskStopInfo; - -struct SExecTaskInfo { - STaskIdInfo id; - uint32_t status; - STimeWindow window; - STaskCostInfo cost; - int64_t owner; // if it is in execution - int32_t code; - int32_t qbufQuota; // total available buffer (in KB) during execution query - int64_t version; // used for stream to record wal version, why not move to sschemainfo - SStreamTaskInfo streamInfo; - SSchemaInfo schemaInfo; - const char* sql; // query sql string - jmp_buf env; // jump to this position when error happens. - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] - SSubplan* pSubplan; - struct SOperatorInfo* pRoot; - SLocalFetch localFetch; - SArray* pResultBlockList; // result block list - STaskStopInfo stopInfo; - SRWLatch lock; // secure the access of STableListInfo -}; - -enum { - OP_NOT_OPENED = 0x0, - OP_OPENED = 0x1, - OP_RES_TO_RETURN = 0x5, - OP_EXEC_DONE = 0x9, -}; - -typedef struct SOperatorFpSet { - __optr_open_fn_t _openFn; // DO NOT invoke this function directly - __optr_fn_t getNextFn; - __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP - __optr_close_fn_t closeFn; - __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator - __optr_encode_fn_t encodeResultRow; - __optr_decode_fn_t decodeResultRow; - __optr_explain_fn_t getExplainFn; -} SOperatorFpSet; - typedef struct SExprSupp { SExprInfo* pExprInfo; int32_t numOfExprs; // the number of scalar expression in group operator @@ -209,22 +131,6 @@ typedef struct SExprSupp { SFilterInfo* pFilterInfo; } SExprSupp; -typedef struct SOperatorInfo { - uint16_t operatorType; - int16_t resultDataBlockId; - bool blocking; // block operator or not - uint8_t status; // denote if current operator is completed - char* name; // name, for debug purpose - void* info; // extension attribution - SExprSupp exprSupp; - SExecTaskInfo* pTaskInfo; - SOperatorCostInfo cost; - SResultInfo resultInfo; - struct SOperatorInfo** pDownstream; // downstram pointer list - int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - SOperatorFpSet fpSet; -} SOperatorInfo; - typedef enum { EX_SOURCE_DATA_NOT_READY = 0x1, EX_SOURCE_DATA_READY = 0x2, @@ -449,8 +355,8 @@ typedef struct SStreamScanInfo { SUpdateInfo* pUpdateInfo; EStreamScanMode scanMode; - SOperatorInfo* pStreamScanOp; - SOperatorInfo* pTableScanOp; + struct SOperatorInfo* pStreamScanOp; + struct SOperatorInfo* pTableScanOp; SArray* childIds; SWindowSupporter windowSup; SPartitionBySupporter partitionSup; @@ -676,18 +582,8 @@ typedef struct SStreamFillOperatorInfo { #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) -SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, - char* dbFName); - -SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, - __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); -int32_t optrDummyOpenFn(SOperatorInfo* pOperator); -int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); -void setOperatorCompleted(SOperatorInfo* pOperator); -void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, - void* pInfo, SExecTaskInfo* pTaskInfo); -void destroyOperatorInfo(SOperatorInfo* pOperator); -int32_t optrDefaultBufFn(SOperatorInfo* pOperator); +SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); +int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); @@ -703,9 +599,9 @@ void cleanupAggSup(SAggSupporter* pAggSup); void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); -void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, +void doBuildStreamResBlock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); -void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, +void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); @@ -719,12 +615,10 @@ void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pC int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, - SOperatorInfo* pOperator); + struct SOperatorInfo* pOperator); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); -SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); extern void doDestroyExchangeOperatorInfo(void* param); @@ -742,76 +636,6 @@ void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup, bool keepGroup); -// operator creater functions -// clang-format off -SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); - -SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); - -SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo); -// clang-format on int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, int32_t numOfOutput, SArray* pPseudoList); @@ -819,21 +643,11 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); int32_t checkForQueryBuf(size_t numOfTables); -int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); - -bool isTaskKilled(SExecTaskInfo* pTaskInfo); -void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); -void doDestroyTask(SExecTaskInfo* pTaskInfo); -void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); - -void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); -int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle); -int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); +int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); @@ -855,17 +669,16 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); bool groupbyTbname(SNodeList* pGroupList); -int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); -int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, +int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); -int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, int64_t* pData); void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h new file mode 100644 index 0000000000..92018b6e3e --- /dev/null +++ b/source/libs/executor/inc/operator.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OPERATOR_H +#define TDENGINE_OPERATOR_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SOperatorCostInfo { + double openCost; + double totalCost; +} SOperatorCostInfo; + +struct SOperatorInfo; + +typedef int32_t (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char** result, int32_t* length); +typedef int32_t (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char* result); + +typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); +typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr); +typedef void (*__optr_close_fn_t)(void* param); +typedef int32_t (*__optr_explain_fn_t)(struct SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); +typedef int32_t (*__optr_reqBuf_fn_t)(struct SOperatorInfo* pOptr); + +typedef struct SOperatorFpSet { + __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_fn_t getNextFn; + __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP + __optr_close_fn_t closeFn; + __optr_reqBuf_fn_t reqBufFn; // total used buffer for blocking operator + __optr_encode_fn_t encodeResultRow; + __optr_decode_fn_t decodeResultRow; + __optr_explain_fn_t getExplainFn; +} SOperatorFpSet; + +enum { + OP_NOT_OPENED = 0x0, + OP_OPENED = 0x1, + OP_RES_TO_RETURN = 0x5, + OP_EXEC_DONE = 0x9, +}; + +typedef struct SOperatorInfo { + uint16_t operatorType; + int16_t resultDataBlockId; + bool blocking; // block operator or not + uint8_t status; // denote if current operator is completed + char* name; // name, for debug purpose + void* info; // extension attribution + SExprSupp exprSupp; + SExecTaskInfo* pTaskInfo; + SOperatorCostInfo cost; + SResultInfo resultInfo; + struct SOperatorInfo** pDownstream; // downstram pointer list + int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + SOperatorFpSet fpSet; +} SOperatorInfo; + +// operator creater functions +// clang-format off +SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createTableCountScanOperatorInfo(SReadHandle* handle, STableCountScanPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); + +SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); + +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo); +// clang-format on + +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, + __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, __optr_explain_fn_t explain); +int32_t optrDummyOpenFn(SOperatorInfo* pOperator); +int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); +void setOperatorCompleted(SOperatorInfo* pOperator); +void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status, + void* pInfo, SExecTaskInfo* pTaskInfo); +int32_t optrDefaultBufFn(SOperatorInfo* pOperator); + +SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond, + SNode* pTagIndexCond, const char* pUser, const char* dbname); +void destroyOperator(SOperatorInfo* pOperator); + +SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); +int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_OPERATOR_H \ No newline at end of file diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h new file mode 100644 index 0000000000..8193426b83 --- /dev/null +++ b/source/libs/executor/inc/querytask.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_QUERYTASK_H +#define TDENGINE_QUERYTASK_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) + +typedef struct STaskIdInfo { + uint64_t queryId; // this is also a request id + uint64_t subplanId; + uint64_t templateId; + char* str; + int32_t vgId; +} STaskIdInfo; + +typedef struct STaskCostInfo { + int64_t created; + int64_t start; + uint64_t elapsedTime; + double extractListTime; + double groupIdMapTime; + SFileBlockLoadRecorder* pRecoder; +} STaskCostInfo; + +typedef struct STaskStopInfo { + SRWLatch lock; + SArray* pStopInfo; +} STaskStopInfo; + +struct SExecTaskInfo { + STaskIdInfo id; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + int32_t code; + int32_t qbufQuota; // total available buffer (in KB) during execution query + int64_t version; // used for stream to record wal version, why not move to sschemainfo + SStreamTaskInfo streamInfo; + SSchemaInfo schemaInfo; + const char* sql; // query sql string + jmp_buf env; // jump to this position when error happens. + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + SSubplan* pSubplan; + struct SOperatorInfo* pRoot; + SLocalFetch localFetch; + SArray* pResultBlockList; // result block list + STaskStopInfo stopInfo; + SRWLatch lock; // secure the access of STableListInfo +}; + +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, char* dbFName); +void doDestroyTask(SExecTaskInfo* pTaskInfo); +bool isTaskKilled(SExecTaskInfo* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); +void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); +int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, + int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); +int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_QUERYTASK_H diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index ec8060348d..c00c167d8e 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -15,7 +15,6 @@ #include "filter.h" #include "function.h" -#include "functionMgt.h" #include "os.h" #include "querynodes.h" #include "tfill.h" @@ -23,16 +22,14 @@ #include "tdatablock.h" #include "tglobal.h" -#include "tmsg.h" -#include "ttime.h" - #include "executorimpl.h" #include "index.h" #include "query.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "vnode.h" +#include "operator.h" +#include "querytask.h" typedef struct { bool hasAgg; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index f6fc332b37..6fee0899ce 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -24,6 +24,8 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" +#include "operator.h" +#include "querytask.h" typedef struct SCacheRowsScanInfo { SSDataBlock* pRes; diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 559fce5224..8ec7ed91a3 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -21,6 +21,8 @@ #include "tcompare.h" #include "tdatablock.h" #include "ttime.h" +#include "operator.h" +#include "querytask.h" typedef struct SEventWindowOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index c855a104b2..31445dd568 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -24,6 +24,8 @@ #include "index.h" #include "query.h" #include "thash.h" +#include "operator.h" +#include "querytask.h" typedef struct SFetchRspHandleWrapper { uint32_t exchangeId; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d79ed0176d..75e1d374e4 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -26,6 +26,7 @@ #include "executil.h" #include "executorimpl.h" #include "tcompression.h" +#include "querytask.h" typedef struct STableListIdInfo { uint64_t suid; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 546cd18cda..7e534ab9d8 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,13 +14,14 @@ */ #include "executor.h" -#include #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" #include "tref.h" #include "tudf.h" #include "vnode.h" +#include "operator.h" +#include "querytask.h" static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; int32_t exchangeObjRefPool = -1; @@ -249,7 +250,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id) { if (msg == NULL) { // create raw scan - SExecTaskInfo* pTaskInfo = doCreateExecTaskInfo(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, ""); + SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, ""); if (NULL == pTaskInfo) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b67a475244..3ab929cad0 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -22,7 +22,6 @@ #include "tname.h" #include "tdatablock.h" -#include "tglobal.h" #include "tmsg.h" #include "ttime.h" @@ -33,6 +32,8 @@ #include "thash.h" #include "ttypes.h" #include "vnode.h" +#include "operator.h" +#include "querytask.h" #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) @@ -71,8 +72,6 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { #define realloc u_realloc #endif -#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) - static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock); static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); @@ -84,7 +83,6 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int bool createDummyCol); static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo); -static SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize) { SFilePage* pData = NULL; @@ -443,13 +441,6 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } } -bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); } - -void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { - pTaskInfo->code = rspCode; - stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str); -} - ///////////////////////////////////////////////////////////////////////////////////////////// STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key) { STimeWindow win = {0}; @@ -467,16 +458,6 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int return win; } -void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { - if (status == TASK_NOT_COMPLETED) { - pTaskInfo->status = status; - } else { - // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED); - pTaskInfo->status |= status; - } -} - void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) { bool init = false; for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1018,136 +999,6 @@ void cleanupExprSupp(SExprSupp* pSupp) { void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst) { - char* p = dst; - - int32_t offset = 6; - memcpy(p, "TID:0x", offset); - offset += tintToHex(taskId, &p[offset]); - - memcpy(&p[offset], " QID:0x", 7); - offset += 7; - offset += tintToHex(queryId, &p[offset]); - - p[offset] = 0; -} - -SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, - char* dbFName) { - SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); - if (pTaskInfo == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); - pTaskInfo->cost.created = taosGetTimestampUs(); - - pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName); - pTaskInfo->execModel = model; - pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); - pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); - - taosInitRWLatch(&pTaskInfo->lock); - pTaskInfo->id.vgId = vgId; - pTaskInfo->id.queryId = queryId; - - pTaskInfo->id.str = taosMemoryMalloc(64); - buildTaskId(taskId, queryId, pTaskInfo->id.str); - return pTaskInfo; -} - -int32_t extractTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, SExecTaskInfo* pTaskInfo) { - SMetaReader mr = {0}; - if (pHandle == NULL) { - terrno = TSDB_CODE_INVALID_PARA; - return terrno; - } - - metaReaderInit(&mr, pHandle->meta, 0); - int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, - GET_TASKID(pTaskInfo)); - - metaReaderClear(&mr); - return terrno; - } - - SSchemaInfo* pSchemaInfo = &pTaskInfo->schemaInfo; - pSchemaInfo->tablename = taosStrdup(mr.me.name); - - if (mr.me.type == TSDB_SUPER_TABLE) { - pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); - pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version; - } else if (mr.me.type == TSDB_CHILD_TABLE) { - tDecoderClear(&mr.coder); - - tb_uid_t suid = mr.me.ctbEntry.suid; - code = metaGetTableEntryByUidCache(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - metaReaderClear(&mr); - return terrno; - } - - pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); - pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version; - } else { - pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow); - } - - metaReaderClear(&mr); - - pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode); - return TSDB_CODE_SUCCESS; -} - -SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { - int32_t numOfCols = LIST_LENGTH(pScanNode->pScanCols); - int32_t numOfTags = LIST_LENGTH(pScanNode->pScanPseudoCols); - - SSchemaWrapper* pqSw = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - pqSw->pSchema = taosMemoryCalloc(numOfCols + numOfTags, sizeof(SSchema)); - - for (int32_t i = 0; i < numOfCols; ++i) { - STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanCols, i); - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - - SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; - pSchema->colId = pColNode->colId; - pSchema->type = pColNode->node.resType.type; - pSchema->bytes = pColNode->node.resType.bytes; - tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); - } - - // this the tags and pseudo function columns, we only keep the tag columns - for (int32_t i = 0; i < numOfTags; ++i) { - STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanPseudoCols, i); - - int32_t type = nodeType(pNode->pExpr); - if (type == QUERY_NODE_COLUMN) { - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - - SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; - pSchema->colId = pColNode->colId; - pSchema->type = pColNode->node.resType.type; - pSchema->bytes = pColNode->node.resType.bytes; - tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); - } - } - - return pqSw; -} - -static void cleanupTableSchemaInfo(SSchemaInfo* pSchemaInfo) { - taosMemoryFreeClear(pSchemaInfo->dbname); - taosMemoryFreeClear(pSchemaInfo->tablename); - tDeleteSSchemaWrapper(pSchemaInfo->sw); - tDeleteSSchemaWrapper(pSchemaInfo->qsw); -} - -static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); } - bool groupbyTbname(SNodeList* pGroupList) { bool bytbname = false; if (LIST_LENGTH(pGroupList) == 1) { @@ -1161,267 +1012,6 @@ bool groupbyTbname(SNodeList* pGroupList) { return bytbname; } -SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond, - SNode* pTagIndexCond, const char* pUser) { - int32_t type = nodeType(pPhyNode); - const char* idstr = GET_TASKID(pTaskInfo); - - if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { - SOperatorInfo* pOperator = NULL; - if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { - STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - - // NOTE: this is an patch to fix the physical plan - // TODO remove it later - if (pTableScanNode->scan.node.pLimit != NULL) { - pTableScanNode->groupSort = true; - } - - STableListInfo* pTableListInfo = tableListCreate(); - int32_t code = - createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, - pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); - if (code) { - pTaskInfo->code = code; - tableListDestroy(pTableListInfo); - qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr); - return NULL; - } - - code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo); - if (code) { - pTaskInfo->code = terrno; - tableListDestroy(pTableListInfo); - return NULL; - } - - pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); - if (NULL == pOperator) { - pTaskInfo->code = terrno; - return NULL; - } - - STableScanInfo* pScanInfo = pOperator->info; - pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; - } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { - STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - STableListInfo* pTableListInfo = tableListCreate(); - - int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, - pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); - if (code) { - pTaskInfo->code = code; - tableListDestroy(pTableListInfo); - qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); - return NULL; - } - - code = extractTableSchemaInfo(pHandle, &pTableScanNode->scan, pTaskInfo); - if (code) { - pTaskInfo->code = terrno; - tableListDestroy(pTableListInfo); - return NULL; - } - - pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); - if (NULL == pOperator) { - pTaskInfo->code = terrno; - tableListDestroy(pTableListInfo); - return NULL; - } - - STableScanInfo* pScanInfo = pOperator->info; - pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; - } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { - pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode, - pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { - STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; - STableListInfo* pTableListInfo = tableListCreate(); - - if (pHandle->vnode) { - int32_t code = - createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, - pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); - if (code) { - pTaskInfo->code = code; - tableListDestroy(pTableListInfo); - qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); - return NULL; - } - } - - pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); - pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { - SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN == type) { - STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode; - pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { - STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - STableListInfo* pTableListInfo = tableListCreate(); - int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, - pTagIndexCond, pTaskInfo); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - qError("failed to getTableList, code: %s", tstrerror(code)); - return NULL; - } - - pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { - SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; - STableListInfo* pTableListInfo = tableListCreate(); - - if (pBlockNode->tableType == TSDB_SUPER_TABLE) { - SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); - int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = terrno; - return NULL; - } - - size_t num = taosArrayGetSize(pList); - for (int32_t i = 0; i < num; ++i) { - STableKeyInfo* p = taosArrayGet(pList, i); - tableListAddTableInfo(pTableListInfo, p->uid, 0); - } - - taosArrayDestroy(pList); - } else { // Create group with only one table - tableListAddTableInfo(pTableListInfo, pBlockNode->uid, 0); - } - - pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTableListInfo, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { - SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; - STableListInfo* pTableListInfo = tableListCreate(); - - int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, - pTagCond, pTagIndexCond, pTaskInfo); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - return NULL; - } - - code = extractTableSchemaInfo(pHandle, &pScanNode->scan, pTaskInfo); - if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = code; - return NULL; - } - - pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTableListInfo, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { - pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); - } else { - terrno = TSDB_CODE_INVALID_PARA; - return NULL; - } - - if (pOperator != NULL) { // todo moved away - pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; - } - - return pOperator; - } - - size_t size = LIST_LENGTH(pPhyNode->pChildren); - SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); - if (ops == NULL) { - return NULL; - } - - for (int32_t i = 0; i < size; ++i) { - SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, pTagCond, pTagIndexCond, pUser); - if (ops[i] == NULL) { - taosMemoryFree(ops); - return NULL; - } - } - - SOperatorInfo* pOptr = NULL; - if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { - pOptr = createProjectOperatorInfo(ops[0], (SProjectPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_AGG == type) { - SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; - if (pAggNode->pGroupKeys != NULL) { - pOptr = createGroupOperatorInfo(ops[0], pAggNode, pTaskInfo); - } else { - pOptr = createAggregateOperatorInfo(ops[0], pAggNode, pTaskInfo); - } - } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { - SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { - pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { - SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; - pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { - SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; - pOptr = createMergeIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { - int32_t children = 0; - pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) { - int32_t children = pHandle->numOfVgroups; - pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); - } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { - pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT == type) { - pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { - SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; - pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) { - SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - pOptr = createSessionAggOperatorInfo(ops[0], pSessionNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) { - pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) { - int32_t children = 0; - pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION == type) { - int32_t children = pHandle->numOfVgroups; - pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); - } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { - pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) { - pOptr = createStreamPartitionOperatorInfo(ops[0], (SStreamPartitionPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) { - SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; - pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { - pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { - pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { - pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) { - pOptr = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { - pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { - pOptr = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo); - } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT == type) { - pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo); - } else { - terrno = TSDB_CODE_INVALID_PARA; - taosMemoryFree(ops); - return NULL; - } - - taosMemoryFree(ops); - if (pOptr) { - pOptr->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; - } - - return pOptr; -} - int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle) { switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { @@ -1469,61 +1059,6 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* return TSDB_CODE_SUCCESS; } -int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, - int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { - *pTaskInfo = doCreateExecTaskInfo(pPlan->id.queryId, taskId, vgId, model, pPlan->dbFName); - if (*pTaskInfo == NULL) { - goto _complete; - } - - if (pHandle) { - if (pHandle->pStateBackend) { - (*pTaskInfo)->streamInfo.pState = pHandle->pStateBackend; - } - } - - (*pTaskInfo)->sql = sql; - sql = NULL; - - (*pTaskInfo)->pSubplan = pPlan; - (*pTaskInfo)->pRoot = - createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, pPlan->pTagCond, pPlan->pTagIndexCond, pPlan->user); - - if (NULL == (*pTaskInfo)->pRoot) { - terrno = (*pTaskInfo)->code; - goto _complete; - } - - return TSDB_CODE_SUCCESS; - -_complete: - taosMemoryFree(sql); - doDestroyTask(*pTaskInfo); - return terrno; -} - -static void freeBlock(void* pParam) { - SSDataBlock* pBlock = *(SSDataBlock**)pParam; - blockDataDestroy(pBlock); -} - -void doDestroyTask(SExecTaskInfo* pTaskInfo) { - qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); - destroyOperatorInfo(pTaskInfo->pRoot); - cleanupTableSchemaInfo(&pTaskInfo->schemaInfo); - cleanupStreamInfo(&pTaskInfo->streamInfo); - - if (!pTaskInfo->localFetch.localExec) { - nodesDestroyNode((SNode*)pTaskInfo->pSubplan); - } - - taosArrayDestroyEx(pTaskInfo->pResultBlockList, freeBlock); - taosArrayDestroy(pTaskInfo->stopInfo.pStopInfo); - taosMemoryFreeClear(pTaskInfo->sql); - taosMemoryFreeClear(pTaskInfo->id.str); - taosMemoryFreeClear(pTaskInfo); -} - int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList) { SExplainExecInfo execInfo = {0}; SExplainExecInfo* pExplainInfo = taosArrayPush(pExecInfoList, &execInfo); diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index 234f1a666c..e465e6a525 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -25,11 +25,13 @@ #include "thash.h" #include "ttime.h" -#include "executorInt.h" #include "function.h" #include "querynodes.h" #include "tdatablock.h" #include "tfill.h" +#include "operator.h" +#include "querytask.h" + #define FILL_POS_INVALID 0 #define FILL_POS_START 1 diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index aaab95e361..821db4244e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -26,6 +26,8 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" +#include "operator.h" +#include "querytask.h" typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 31ff11eec5..b4862a490f 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -23,6 +23,8 @@ #include "thash.h" #include "tmsg.h" #include "ttypes.h" +#include "operator.h" +#include "querytask.h" typedef struct SJoinRowCtx { bool rowRemains; diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index b1563317cf..21d5d1b639 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -15,24 +15,17 @@ #include "filter.h" #include "function.h" -#include "functionMgt.h" #include "os.h" -#include "querynodes.h" -#include "tfill.h" #include "tname.h" -#include "tdatablock.h" #include "tglobal.h" -#include "tmsg.h" -#include "ttime.h" #include "executorimpl.h" #include "index.h" #include "query.h" -#include "tcompare.h" -#include "thash.h" -#include "ttypes.h" #include "vnode.h" +#include "operator.h" +#include "querytask.h" SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, @@ -82,7 +75,7 @@ void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, b pOperator->pTaskInfo = pTaskInfo; } -void destroyOperatorInfo(SOperatorInfo* pOperator) { +void destroyOperator(SOperatorInfo* pOperator) { if (pOperator == NULL) { return; } @@ -93,7 +86,7 @@ void destroyOperatorInfo(SOperatorInfo* pOperator) { if (pOperator->pDownstream != NULL) { for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - destroyOperatorInfo(pOperator->pDownstream[i]); + destroyOperator(pOperator->pDownstream[i]); } taosMemoryFreeClear(pOperator->pDownstream); @@ -287,7 +280,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan SExtScanInfo info = {.inheritUsOrder = inheritUsOrder}; STraverParam p = {.pParam = &info}; - extractScanInfo(pOperator, &p, NULL); + traverseOperatorTree(pOperator, extractScanInfo, &p, NULL); *order = info.order; *scanFlag = info.scanFlag; @@ -304,10 +297,12 @@ static ERetType doStopDataReader(SOperatorInfo* pOperator, STraverParam* pParam, return OPTR_FN_RET_ABORT; } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pInfo = pOperator->info; - STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; - if (pTableScanInfo->base.dataReader != NULL) { - tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader); + if (pInfo->pTableScanOp != NULL) { + STableScanInfo* pTableScanInfo = pInfo->pTableScanOp->info; + if (pTableScanInfo != NULL && pTableScanInfo->base.dataReader != NULL) { + tsdbReaderSetCloseFlag(pTableScanInfo->base.dataReader); + } } return OPTR_FN_RET_ABORT; @@ -321,3 +316,264 @@ int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) { traverseOperatorTree(pOperator, doStopDataReader, &p, pIdStr); return p.code; } + +SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond, + SNode* pTagIndexCond, const char* pUser, const char* dbname) { + int32_t type = nodeType(pPhyNode); + const char* idstr = GET_TASKID(pTaskInfo); + + if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { + SOperatorInfo* pOperator = NULL; + if (QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == type) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; + + // NOTE: this is an patch to fix the physical plan + // TODO remove it later + if (pTableScanNode->scan.node.pLimit != NULL) { + pTableScanNode->groupSort = true; + } + + STableListInfo* pTableListInfo = tableListCreate(); + int32_t code = + createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, pHandle, + pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); + if (code) { + pTaskInfo->code = code; + tableListDestroy(pTableListInfo); + qError("failed to createScanTableListInfo, code:%s, %s", tstrerror(code), idstr); + return NULL; + } + + code = initQueriedTableSchemaInfo(pHandle, &pTableScanNode->scan, dbname, pTaskInfo); + if (code) { + pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); + return NULL; + } + + pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); + if (NULL == pOperator) { + pTaskInfo->code = terrno; + return NULL; + } + + STableScanInfo* pScanInfo = pOperator->info; + pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { + STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); + + int32_t code = createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, true, pHandle, + pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); + if (code) { + pTaskInfo->code = code; + tableListDestroy(pTableListInfo); + qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); + return NULL; + } + + code = initQueriedTableSchemaInfo(pHandle, &pTableScanNode->scan, dbname, pTaskInfo); + if (code) { + pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); + return NULL; + } + + pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); + if (NULL == pOperator) { + pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); + return NULL; + } + + STableScanInfo* pScanInfo = pOperator->info; + pTaskInfo->cost.pRecoder = &pScanInfo->base.readRecorder; + } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { + pOperator = createExchangeOperatorInfo(pHandle ? pHandle->pMsgCb->clientRpc : NULL, (SExchangePhysiNode*)pPhyNode, + pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); + + if (pHandle->vnode) { + int32_t code = + createScanTableListInfo(&pTableScanNode->scan, pTableScanNode->pGroupTags, pTableScanNode->groupSort, + pHandle, pTableListInfo, pTagCond, pTagIndexCond, pTaskInfo); + if (code) { + pTaskInfo->code = code; + tableListDestroy(pTableListInfo); + qError("failed to createScanTableListInfo, code: %s", tstrerror(code)); + return NULL; + } + } + + pTaskInfo->schemaInfo.qsw = extractQueriedColumnSchema(&pTableScanNode->scan); + pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTagCond, pTableListInfo, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { + SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; + pOperator = createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN == type) { + STableCountScanPhysiNode* pTblCountScanNode = (STableCountScanPhysiNode*)pPhyNode; + pOperator = createTableCountScanOperatorInfo(pHandle, pTblCountScanNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { + STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); + int32_t code = createScanTableListInfo(pScanPhyNode, NULL, false, pHandle, pTableListInfo, pTagCond, + pTagIndexCond, pTaskInfo); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + qError("failed to getTableList, code: %s", tstrerror(code)); + return NULL; + } + + pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { + SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); + + if (pBlockNode->tableType == TSDB_SUPER_TABLE) { + SArray* pList = taosArrayInit(4, sizeof(STableKeyInfo)); + int32_t code = vnodeGetAllTableList(pHandle->vnode, pBlockNode->uid, pList); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = terrno; + return NULL; + } + + size_t num = taosArrayGetSize(pList); + for (int32_t i = 0; i < num; ++i) { + STableKeyInfo* p = taosArrayGet(pList, i); + tableListAddTableInfo(pTableListInfo, p->uid, 0); + } + + taosArrayDestroy(pList); + } else { // Create group with only one table + tableListAddTableInfo(pTableListInfo, pBlockNode->uid, 0); + } + + pOperator = createDataBlockInfoScanOperator(pHandle, pBlockNode, pTableListInfo, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN == type) { + SLastRowScanPhysiNode* pScanNode = (SLastRowScanPhysiNode*)pPhyNode; + STableListInfo* pTableListInfo = tableListCreate(); + + int32_t code = createScanTableListInfo(&pScanNode->scan, pScanNode->pGroupTags, true, pHandle, pTableListInfo, + pTagCond, pTagIndexCond, pTaskInfo); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } + + code = initQueriedTableSchemaInfo(pHandle, &pScanNode->scan, dbname, pTaskInfo); + if (code != TSDB_CODE_SUCCESS) { + pTaskInfo->code = code; + return NULL; + } + + pOperator = createCacherowsScanOperator(pScanNode, pHandle, pTableListInfo, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { + pOperator = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pPhyNode, pTaskInfo); + } else { + terrno = TSDB_CODE_INVALID_PARA; + return NULL; + } + + if (pOperator != NULL) { // todo moved away + pOperator->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + } + + return pOperator; + } + + size_t size = LIST_LENGTH(pPhyNode->pChildren); + SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); + if (ops == NULL) { + return NULL; + } + + for (int32_t i = 0; i < size; ++i) { + SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); + ops[i] = createOperator(pChildNode, pTaskInfo, pHandle, pTagCond, pTagIndexCond, pUser, dbname); + if (ops[i] == NULL) { + taosMemoryFree(ops); + return NULL; + } + } + + SOperatorInfo* pOptr = NULL; + if (QUERY_NODE_PHYSICAL_PLAN_PROJECT == type) { + pOptr = createProjectOperatorInfo(ops[0], (SProjectPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_AGG == type) { + SAggPhysiNode* pAggNode = (SAggPhysiNode*)pPhyNode; + if (pAggNode->pGroupKeys != NULL) { + pOptr = createGroupOperatorInfo(ops[0], pAggNode, pTaskInfo); + } else { + pOptr = createAggregateOperatorInfo(ops[0], pAggNode, pTaskInfo); + } + } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { + SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; + pOptr = createIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { + SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; + pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { + SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; + pOptr = createMergeIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL == type) { + int32_t children = 0; + pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == type) { + int32_t children = pHandle->numOfVgroups; + pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { + pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT == type) { + pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { + SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; + pOptr = createMultiwayMergeOperatorInfo(ops, size, pMergePhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION == type) { + SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; + pOptr = createSessionAggOperatorInfo(ops[0], pSessionNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION == type) { + pOptr = createStreamSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION == type) { + int32_t children = 0; + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION == type) { + int32_t children = pHandle->numOfVgroups; + pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); + } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { + pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) { + pOptr = createStreamPartitionOperatorInfo(ops[0], (SStreamPartitionPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) { + SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; + pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { + pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { + pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_FILL == type) { + pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL == type) { + pOptr = createStreamFillOperatorInfo(ops[0], (SStreamFillPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { + pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { + pOptr = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT == type) { + pOptr = createEventwindowOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else { + terrno = TSDB_CODE_INVALID_PARA; + taosMemoryFree(ops); + return NULL; + } + + taosMemoryFree(ops); + if (pOptr) { + pOptr->resultDataBlockId = pPhyNode->pOutputDataBlockDesc->dataBlockId; + } + + return pOptr; +} diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 86c49e0fc8..01ee18b41f 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -16,6 +16,8 @@ #include "executorimpl.h" #include "filter.h" #include "functionMgt.h" +#include "operator.h" +#include "querytask.h" typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c new file mode 100644 index 0000000000..b9add35502 --- /dev/null +++ b/source/libs/executor/src/querytask.c @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "tfill.h" +#include "tname.h" + +#include "tdatablock.h" +#include "tmsg.h" + +#include "executorimpl.h" +#include "index.h" +#include "query.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" +#include "operator.h" +#include "querytask.h" + +#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) + +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, + char* dbFName) { + SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); + if (pTaskInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + pTaskInfo->cost.created = taosGetTimestampUs(); + + pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName); + pTaskInfo->execModel = model; + pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); + pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); + + taosInitRWLatch(&pTaskInfo->lock); + pTaskInfo->id.vgId = vgId; + pTaskInfo->id.queryId = queryId; + + pTaskInfo->id.str = taosMemoryMalloc(64); + buildTaskId(taskId, queryId, pTaskInfo->id.str); + return pTaskInfo; +} + +bool isTaskKilled(SExecTaskInfo* pTaskInfo) { return (0 != pTaskInfo->code); } + +void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode) { + pTaskInfo->code = rspCode; + stopTableScanOperator(pTaskInfo->pRoot, pTaskInfo->id.str); +} + +void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { + if (status == TASK_NOT_COMPLETED) { + pTaskInfo->status = status; + } else { + // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first + CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED); + pTaskInfo->status |= status; + } +} + +int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, + int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { + *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, pPlan->dbFName); + if (*pTaskInfo == NULL) { + goto _complete; + } + + if (pHandle) { + if (pHandle->pStateBackend) { + (*pTaskInfo)->streamInfo.pState = pHandle->pStateBackend; + } + } + + (*pTaskInfo)->sql = sql; + sql = NULL; + + (*pTaskInfo)->pSubplan = pPlan; + (*pTaskInfo)->pRoot = createOperator(pPlan->pNode, *pTaskInfo, pHandle, pPlan->pTagCond, pPlan->pTagIndexCond, + pPlan->user, pPlan->dbFName); + + if (NULL == (*pTaskInfo)->pRoot) { + terrno = (*pTaskInfo)->code; + goto _complete; + } + + return TSDB_CODE_SUCCESS; + + _complete: + taosMemoryFree(sql); + doDestroyTask(*pTaskInfo); + return terrno; +} + +void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo) { + taosMemoryFreeClear(pSchemaInfo->dbname); + taosMemoryFreeClear(pSchemaInfo->tablename); + tDeleteSSchemaWrapper(pSchemaInfo->sw); + tDeleteSSchemaWrapper(pSchemaInfo->qsw); +} + +int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo) { + SMetaReader mr = {0}; + if (pHandle == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + metaReaderInit(&mr, pHandle->meta, 0); + int32_t code = metaGetTableEntryByUidCache(&mr, pScanNode->uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get the table meta, uid:0x%" PRIx64 ", suid:0x%" PRIx64 ", %s", pScanNode->uid, pScanNode->suid, + GET_TASKID(pTaskInfo)); + + metaReaderClear(&mr); + return terrno; + } + + SSchemaInfo* pSchemaInfo = &pTaskInfo->schemaInfo; + + pSchemaInfo->tablename = taosStrdup(mr.me.name); + pSchemaInfo->dbname = taosStrdup(dbName); + + if (mr.me.type == TSDB_SUPER_TABLE) { + pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); + pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version; + } else if (mr.me.type == TSDB_CHILD_TABLE) { + tDecoderClear(&mr.coder); + + tb_uid_t suid = mr.me.ctbEntry.suid; + code = metaGetTableEntryByUidCache(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + metaReaderClear(&mr); + return terrno; + } + + pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.stbEntry.schemaRow); + pSchemaInfo->tversion = mr.me.stbEntry.schemaTag.version; + } else { + pSchemaInfo->sw = tCloneSSchemaWrapper(&mr.me.ntbEntry.schemaRow); + } + + metaReaderClear(&mr); + + pSchemaInfo->qsw = extractQueriedColumnSchema(pScanNode); + return TSDB_CODE_SUCCESS; +} + +SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { + int32_t numOfCols = LIST_LENGTH(pScanNode->pScanCols); + int32_t numOfTags = LIST_LENGTH(pScanNode->pScanPseudoCols); + + SSchemaWrapper* pqSw = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + pqSw->pSchema = taosMemoryCalloc(numOfCols + numOfTags, sizeof(SSchema)); + + for (int32_t i = 0; i < numOfCols; ++i) { + STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanCols, i); + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + + SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; + pSchema->colId = pColNode->colId; + pSchema->type = pColNode->node.resType.type; + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + } + + // this the tags and pseudo function columns, we only keep the tag columns + for (int32_t i = 0; i < numOfTags; ++i) { + STargetNode* pNode = (STargetNode*)nodesListGetNode(pScanNode->pScanPseudoCols, i); + + int32_t type = nodeType(pNode->pExpr); + if (type == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + + SSchema* pSchema = &pqSw->pSchema[pqSw->nCols++]; + pSchema->colId = pColNode->colId; + pSchema->type = pColNode->node.resType.type; + pSchema->bytes = pColNode->node.resType.bytes; + tstrncpy(pSchema->name, pColNode->colName, tListLen(pSchema->name)); + } + } + + return pqSw; +} + +static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); } + +static void freeBlock(void* pParam) { + SSDataBlock* pBlock = *(SSDataBlock**)pParam; + blockDataDestroy(pBlock); +} + +void doDestroyTask(SExecTaskInfo* pTaskInfo) { + qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); + destroyOperator(pTaskInfo->pRoot); + cleanupQueriedTableScanInfo(&pTaskInfo->schemaInfo); + cleanupStreamInfo(&pTaskInfo->streamInfo); + + if (!pTaskInfo->localFetch.localExec) { + nodesDestroyNode((SNode*)pTaskInfo->pSubplan); + } + + taosArrayDestroyEx(pTaskInfo->pResultBlockList, freeBlock); + taosArrayDestroy(pTaskInfo->stopInfo.pStopInfo); + taosMemoryFreeClear(pTaskInfo->sql); + taosMemoryFreeClear(pTaskInfo->id.str); + taosMemoryFreeClear(pTaskInfo); +} + +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst) { + char* p = dst; + + int32_t offset = 6; + memcpy(p, "TID:0x", offset); + offset += tintToHex(taskId, &p[offset]); + + memcpy(&p[offset], " QID:0x", 7); + offset += 7; + offset += tintToHex(queryId, &p[offset]); + + p[offset] = 0; +} diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f583c021d..a717c8b7c0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -30,6 +30,8 @@ #include "tcompare.h" #include "thash.h" #include "ttypes.h" +#include "operator.h" +#include "querytask.h" int32_t scanDebug = 0; @@ -2304,7 +2306,7 @@ static void destroyStreamScanOperatorInfo(void* param) { SStreamScanInfo* pStreamScan = (SStreamScanInfo*)param; if (pStreamScan->pTableScanOp && pStreamScan->pTableScanOp->info) { - destroyOperatorInfo(pStreamScan->pTableScanOp); + destroyOperator(pStreamScan->pTableScanOp); } if (pStreamScan->tqReader) { diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index cb0f1aa068..0b23a803c6 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -16,6 +16,8 @@ #include "filter.h" #include "executorimpl.h" #include "tdatablock.h" +#include "operator.h" +#include "querytask.h" typedef struct SSortOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index c78e6002cd..bc850b5333 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -31,6 +31,9 @@ #include "thash.h" #include "ttypes.h" #include "vnode.h" +#include "operator.h" +#include "querytask.h" + typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index f0e25d8cc5..a688920a22 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -21,6 +21,8 @@ #include "tdatablock.h" #include "tfill.h" #include "ttime.h" +#include "operator.h" +#include "querytask.h" typedef struct STimeSliceOperatorInfo { SSDataBlock* pRes; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ce2d4b27dd..ab9b0987fa 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -21,6 +21,8 @@ #include "tdatablock.h" #include "tfill.h" #include "ttime.h" +#include "operator.h" +#include "querytask.h" #define IS_FINAL_OP(op) ((op)->isFinal) #define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL); @@ -1606,7 +1608,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, i); - destroyOperatorInfo(pChildOp); + destroyOperator(pChildOp); } taosArrayDestroy(pInfo->pChildren); } @@ -2835,7 +2837,7 @@ void destroyStreamSessionAggOperatorInfo(void* param) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i); - destroyOperatorInfo(pChild); + destroyOperator(pChild); } taosArrayDestroy(pInfo->pChildren); } @@ -3812,7 +3814,7 @@ void destroyStreamStateOperatorInfo(void* param) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i); - destroyOperatorInfo(pChild); + destroyOperator(pChild); } taosArrayDestroy(pInfo->pChildren); } diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index b9a696170a..db0678f214 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -29,11 +29,8 @@ #include "taos.h" #include "tdatablock.h" #include "tdef.h" -#include "tglobal.h" -#include "tmsg.h" -#include "tname.h" -#include "trpc.h" #include "tvariant.h" +#include "operator.h" namespace { From 8205168428aafd06e19898b63961086e4d6d596c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 09:16:11 +0800 Subject: [PATCH 095/145] fix(query): fix memory leak. --- source/libs/executor/inc/querytask.h | 2 +- source/libs/executor/src/executor.c | 2 +- source/libs/executor/src/querytask.c | 12 +++++------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index 8193426b83..aa42cd2c14 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -67,7 +67,7 @@ struct SExecTaskInfo { }; void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, char* dbFName); +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model); void doDestroyTask(SExecTaskInfo* pTaskInfo); bool isTaskKilled(SExecTaskInfo* pTaskInfo); void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7e534ab9d8..2b9ab6c09a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -250,7 +250,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int32_t vgId, int32_t* numOfCols, uint64_t id) { if (msg == NULL) { // create raw scan - SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE, ""); + SExecTaskInfo* pTaskInfo = doCreateTask(0, id, vgId, OPTR_EXEC_MODEL_QUEUE); if (NULL == pTaskInfo) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index b9add35502..318deab1d8 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -35,8 +35,7 @@ #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) -SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, - char* dbFName) { +SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); if (pTaskInfo == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -46,17 +45,17 @@ SExecTaskInfo* doCreateTask(uint64_t queryId, uint64_t taskId, int32_t vgId, EOP setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->cost.created = taosGetTimestampUs(); - pTaskInfo->schemaInfo.dbname = taosStrdup(dbFName); pTaskInfo->execModel = model; pTaskInfo->stopInfo.pStopInfo = taosArrayInit(4, sizeof(SExchangeOpStopInfo)); pTaskInfo->pResultBlockList = taosArrayInit(128, POINTER_BYTES); taosInitRWLatch(&pTaskInfo->lock); + pTaskInfo->id.vgId = vgId; pTaskInfo->id.queryId = queryId; - pTaskInfo->id.str = taosMemoryMalloc(64); buildTaskId(taskId, queryId, pTaskInfo->id.str); + return pTaskInfo; } @@ -79,7 +78,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { - *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model, pPlan->dbFName); + *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model); if (*pTaskInfo == NULL) { goto _complete; } @@ -90,8 +89,7 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand } } - (*pTaskInfo)->sql = sql; - sql = NULL; + TSWAP((*pTaskInfo)->sql, sql); (*pTaskInfo)->pSubplan = pPlan; (*pTaskInfo)->pRoot = createOperator(pPlan->pNode, *pTaskInfo, pHandle, pPlan->pTagCond, pPlan->pTagIndexCond, From 6482ef9ad136caad58c624315e591855ae40bf77 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 09:19:53 +0800 Subject: [PATCH 096/145] refactor: do some internal refactor. --- source/libs/executor/src/querytask.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index 318deab1d8..ed209efed0 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -80,7 +80,8 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand int32_t vgId, char* sql, EOPTR_EXEC_MODEL model) { *pTaskInfo = doCreateTask(pPlan->id.queryId, taskId, vgId, model); if (*pTaskInfo == NULL) { - goto _complete; + taosMemoryFree(sql); + return terrno; } if (pHandle) { @@ -96,16 +97,12 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand pPlan->user, pPlan->dbFName); if (NULL == (*pTaskInfo)->pRoot) { - terrno = (*pTaskInfo)->code; - goto _complete; + int32_t code = (*pTaskInfo)->code; + doDestroyTask(*pTaskInfo); + return code; + } else { + return TSDB_CODE_SUCCESS; } - - return TSDB_CODE_SUCCESS; - - _complete: - taosMemoryFree(sql); - doDestroyTask(*pTaskInfo); - return terrno; } void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo) { From cce868d140249866698d80aa5e84c487299279aa Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 28 Apr 2023 09:45:16 +0800 Subject: [PATCH 097/145] fix:offset encode assert error --- source/client/src/clientTmq.c | 5 +++-- source/common/src/tmsg.c | 8 ++------ source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 3 +-- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 33324552dc..0a31eac09c 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1790,8 +1790,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { pVg->epSet = *pollRspWrapper->pEpset; } - // update the local offset value only for the returned values. - pVg->currentOffset = pDataRsp->rspOffset; + if(pDataRsp->rspOffset.type != 0){ // if offset is validate + pVg->currentOffset = pDataRsp->rspOffset; // update the local offset value only for the returned values. + } atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); char buf[80]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index cbf856a799..2639b81c39 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6839,10 +6839,8 @@ int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal) if (tEncodeI64(pEncoder, pOffsetVal->ts) < 0) return -1; } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { if (tEncodeI64(pEncoder, pOffsetVal->version) < 0) return -1; - } else if (pOffsetVal->type < 0) { - // do nothing } else { - ASSERT(0); + // do nothing } return 0; } @@ -6854,10 +6852,8 @@ int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) { if (tDecodeI64(pDecoder, &pOffsetVal->ts) < 0) return -1; } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { if (tDecodeI64(pDecoder, &pOffsetVal->version) < 0) return -1; - } else if (pOffsetVal->type < 0) { - // do nothing } else { - ASSERT(0); + // do nothing } return 0; } diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 1b29b34073..30b2fb74ca 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -114,7 +114,7 @@ struct STQ { char* path; int64_t walLogLastVer; SRWLatch lock; - SHashObj* pPushMgr; // consumerId -> STqHandle + SHashObj* pPushMgr; // subKey -> STqHandle SHashObj* pHandle; // subKey -> STqHandle SHashObj* pCheckInfo; // topic -> SAlterCheckInfo STqOffsetStore* pOffsetStore; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 53a40eb839..bd29c8019d 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -551,13 +551,12 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } taosWLockLatch(&pTq->lock); - atomic_store_32(&pHandle->epoch, -1); + atomic_store_32(&pHandle->epoch, 0); // remove if it has been register in the push manager, and return one empty block to consumer tqUnregisterPushHandle(pTq, pHandle); atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_add_fetch_32(&pHandle->epoch, 1); if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { qStreamCloseTsdbReader(pTaskInfo); From 3ac1ce69b64ddafb13cb7f9383d84b967230e1ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 10:02:37 +0800 Subject: [PATCH 098/145] fix(query): set correct scan flag. --- source/libs/executor/src/operator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 21d5d1b639..5e384ca6dd 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -277,13 +277,14 @@ static ERetType extractScanInfo(SOperatorInfo* pOperator, STraverParam* pParam, } int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder) { - SExtScanInfo info = {.inheritUsOrder = inheritUsOrder}; + SExtScanInfo info = {.inheritUsOrder = inheritUsOrder, .order = *order}; STraverParam p = {.pParam = &info}; traverseOperatorTree(pOperator, extractScanInfo, &p, NULL); *order = info.order; *scanFlag = info.scanFlag; + ASSERT(*order == TSDB_ORDER_ASC || *order == TSDB_ORDER_DESC); return p.code; } From da0d9c78efa5e0fbfa05865c2273a325043d86f9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 11:42:34 +0800 Subject: [PATCH 099/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tq/tqRestore.c | 2 +- source/dnode/vnode/src/tq/tqScan.c | 1 + source/libs/executor/inc/executorInt.h | 644 +++++++++++++++- source/libs/executor/inc/executorimpl.h | 699 ------------------ source/libs/executor/inc/operator.h | 1 + source/libs/executor/inc/querytask.h | 29 + source/libs/executor/src/aggregateoperator.c | 12 +- source/libs/executor/src/cachescanoperator.c | 6 +- source/libs/executor/src/dataDeleter.c | 2 +- source/libs/executor/src/dataDispatcher.c | 2 +- source/libs/executor/src/dataInserter.c | 2 +- .../libs/executor/src/eventwindowoperator.c | 6 +- source/libs/executor/src/exchangeoperator.c | 16 +- source/libs/executor/src/executil.c | 4 +- source/libs/executor/src/executor.c | 30 +- .../src/{executorimpl.c => executorInt.c} | 59 +- source/libs/executor/src/filloperator.c | 2 +- source/libs/executor/src/groupoperator.c | 5 +- source/libs/executor/src/joinoperator.c | 6 +- source/libs/executor/src/operator.c | 120 ++- source/libs/executor/src/projectoperator.c | 2 +- source/libs/executor/src/querytask.c | 6 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/sortoperator.c | 4 +- source/libs/executor/src/sysscanoperator.c | 2 +- source/libs/executor/src/tfill.c | 2 +- source/libs/executor/src/timesliceoperator.c | 6 +- source/libs/executor/src/timewindowoperator.c | 12 +- source/libs/executor/test/executorTests.cpp | 4 +- source/libs/executor/test/lhashTests.cpp | 2 +- source/libs/executor/test/sortTests.cpp | 2 +- 31 files changed, 815 insertions(+), 877 deletions(-) delete mode 100644 source/libs/executor/inc/executorimpl.h rename source/libs/executor/src/{executorimpl.c => executorInt.c} (95%) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index c164d037e0..8cda5a21c9 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -107,7 +107,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE || status == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, status); + tqDebug("s-task:%s skip push data, not ready for processing, status:%d", pTask->id.idStr, status); streamMetaReleaseTask(pStreamMeta, pTask); continue; } diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 0f00a5acb8..8e243a8bd1 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -130,6 +130,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta tqError("vgId:%d, task exec error since %s", pTq->pVnode->config.vgId, terrstr()); return -1; } + tqDebug("tmqsnap task execute end, get %p", pDataBlock); if (pDataBlock != NULL && pDataBlock->info.rows > 0) { diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index d22a7460bb..a4f1e2ef94 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -12,14 +12,75 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ - -#ifndef _TD_EXECUTOR_INT_H -#define _TD_EXECUTOR_INT_H +#ifndef TDENGINE_EXECUTORINT_H +#define TDENGINE_EXECUTORINT_H #ifdef __cplusplus extern "C" { #endif +#include "os.h" +#include "tcommon.h" +#include "tlosertree.h" +#include "tsort.h" +#include "ttszip.h" +#include "tvariant.h" + +#include "dataSinkMgt.h" +#include "executil.h" +#include "executor.h" +#include "planner.h" +#include "scalar.h" +#include "taosdef.h" +#include "tarray.h" +#include "tfill.h" +#include "thash.h" +#include "tlockfree.h" +#include "tmsg.h" +#include "tpagedbuf.h" +#include "tstream.h" +#include "tstreamUpdate.h" + +#include "vnode.h" + +typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); + +#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) +#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) +#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) +#define SET_SESSION_WIN_KEY_INVALID(pWinKey) ((pWinKey)->win.skey = INT64_MIN) + +/** + * If the number of generated results is greater than this value, + * query query will be halt and return results to client immediate. + */ +typedef struct SResultInfo { // TODO refactor + int64_t totalRows; // total generated result size in rows + int64_t totalBytes; // total results in bytes. + int32_t capacity; // capacity of current result output buffer + int32_t threshold; // result size threshold in rows. +} SResultInfo; + +typedef struct STableQueryInfo { + TSKEY lastKey; // last check ts, todo remove it later + SResultRowPosition pos; // current active time window +} STableQueryInfo; + +typedef struct SLimit { + int64_t limit; + int64_t offset; +} SLimit; + +typedef struct STableScanAnalyzeInfo SFileBlockLoadRecorder; + +enum { + STREAM_RECOVER_STEP__NONE = 0, + STREAM_RECOVER_STEP__PREPARE1, + STREAM_RECOVER_STEP__PREPARE2, + STREAM_RECOVER_STEP__SCAN1, + STREAM_RECOVER_STEP__SCAN2, +}; + extern int32_t exchangeObjRefPool; typedef struct { @@ -29,9 +90,584 @@ typedef struct { int32_t bytes; } SGroupKeys, SStateKeys; +typedef struct { + char* tablename; + char* dbname; + int32_t tversion; + SSchemaWrapper* sw; + SSchemaWrapper* qsw; +} SSchemaInfo; + +typedef struct SExchangeOpStopInfo { + int32_t operatorType; + int64_t refId; +} SExchangeOpStopInfo; + +typedef struct SExprSupp { + SExprInfo* pExprInfo; + int32_t numOfExprs; // the number of scalar expression in group operator + SqlFunctionCtx* pCtx; + int32_t* rowEntryInfoOffset; // offset value for each row result cell info + SFilterInfo* pFilterInfo; +} SExprSupp; + +typedef enum { + EX_SOURCE_DATA_NOT_READY = 0x1, + EX_SOURCE_DATA_READY = 0x2, + EX_SOURCE_DATA_EXHAUSTED = 0x3, +} EX_SOURCE_STATUS; + +#define COL_MATCH_FROM_COL_ID 0x1 +#define COL_MATCH_FROM_SLOT_ID 0x2 + +typedef struct SLoadRemoteDataInfo { + uint64_t totalSize; // total load bytes from remote + uint64_t totalRows; // total number of rows + uint64_t totalElapsed; // total elapsed time +} SLoadRemoteDataInfo; + +typedef struct SLimitInfo { + SLimit limit; + SLimit slimit; + uint64_t currentGroupId; + int64_t remainGroupOffset; + int64_t numOfOutputGroups; + int64_t remainOffset; + int64_t numOfOutputRows; +} SLimitInfo; + +typedef struct SExchangeInfo { + SArray* pSources; + SArray* pSourceDataInfo; + tsem_t ready; + void* pTransporter; + + // SArray, result block list, used to keep the multi-block that + // passed by downstream operator + SArray* pResultBlockList; + SArray* pRecycledBlocks; // build a pool for small data block to avoid to repeatly create and then destroy. + SSDataBlock* pDummyBlock; // dummy block, not keep data + bool seqLoadData; // sequential load data or not, false by default + int32_t current; + SLoadRemoteDataInfo loadInfo; + uint64_t self; + SLimitInfo limitInfo; + int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo +} SExchangeInfo; + +typedef struct SScanInfo { + int32_t numOfAsc; + int32_t numOfDesc; +} SScanInfo; + +typedef struct SSampleExecInfo { + double sampleRatio; // data block sample ratio, 1 by default + uint32_t seed; // random seed value +} SSampleExecInfo; + +enum { + TABLE_SCAN__TABLE_ORDER = 1, + TABLE_SCAN__BLOCK_ORDER = 2, +}; + +typedef struct SAggSupporter { + SSHashObj* pResultRowHashTable; // quick locate the window object for each result + char* keyBuf; // window key buffer + SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + int32_t currentPageId; // current write page id +} SAggSupporter; + +typedef struct { + // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if + // current data block needs to be loaded. + SInterval interval; + SAggSupporter* pAggSup; + SExprSupp* pExprSup; // expr supporter of aggregate operator +} SAggOptrPushDownInfo; + +typedef struct STableMetaCacheInfo { + SLRUCache* pTableMetaEntryCache; // 100 by default + uint64_t metaFetch; + uint64_t cacheHit; +} STableMetaCacheInfo; + +typedef struct STableScanBase { + STsdbReader* dataReader; + SFileBlockLoadRecorder readRecorder; + SQueryTableDataCond cond; + SAggOptrPushDownInfo pdInfo; + SColMatchInfo matchInfo; + SReadHandle readHandle; + SExprSupp pseudoSup; + STableMetaCacheInfo metaCache; + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + SLimitInfo limitInfo; + // there are more than one table list exists in one task, if only one vnode exists. + STableListInfo* pTableListInfo; +} STableScanBase; + +typedef struct STableScanInfo { + STableScanBase base; + SScanInfo scanInfo; + int32_t scanTimes; + SSDataBlock* pResBlock; + SSampleExecInfo sample; // sample execution info + int32_t currentGroupId; + int32_t currentTable; + int8_t scanMode; + int8_t assignBlockUid; + bool hasGroupByTag; + bool countOnly; +} STableScanInfo; + +typedef struct STableMergeScanInfo { + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; + SArray* queryConds; // array of queryTableDataCond + STableScanBase base; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SSDataBlock* pSortInputBlock; + int64_t startTs; // sort start time + SArray* sortSourceParams; + SLimitInfo limitInfo; + int64_t numOfRows; + SScanInfo scanInfo; + int32_t scanTimes; + SSDataBlock* pResBlock; + SSampleExecInfo sample; // sample execution info + SSortExecInfo sortExecInfo; +} STableMergeScanInfo; + +typedef struct STagScanInfo { + SColumnInfo* pCols; + SSDataBlock* pRes; + SColMatchInfo matchInfo; + int32_t curPos; + SReadHandle readHandle; + STableListInfo* pTableListInfo; +} STagScanInfo; + +typedef enum EStreamScanMode { + STREAM_SCAN_FROM_READERHANDLE = 1, + STREAM_SCAN_FROM_RES, + STREAM_SCAN_FROM_UPDATERES, + STREAM_SCAN_FROM_DELETE_DATA, + STREAM_SCAN_FROM_DATAREADER_RETRIEVE, + STREAM_SCAN_FROM_DATAREADER_RANGE, +} EStreamScanMode; + +enum { + PROJECT_RETRIEVE_CONTINUE = 0x1, + PROJECT_RETRIEVE_DONE = 0x2, +}; + +typedef struct SStreamAggSupporter { + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + SSDataBlock* pScanBlock; + SStreamState* pState; + int64_t gap; // stream session window gap + SqlFunctionCtx* pDummyCtx; // for combine + SSHashObj* pResultRows; + int32_t stateKeySize; + int16_t stateKeyType; + SDiskbasedBuf* pResultBuf; +} SStreamAggSupporter; + +typedef struct SWindowSupporter { + SStreamAggSupporter* pStreamAggSup; + int64_t gap; + uint16_t parentType; + SAggSupporter* pIntervalAggSup; +} SWindowSupporter; + +typedef struct SPartitionBySupporter { + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + bool needCalc; // partition by column +} SPartitionBySupporter; + +typedef struct SPartitionDataInfo { + uint64_t groupId; + char* tbname; + SArray* tags; + SArray* rowIds; +} SPartitionDataInfo; + +typedef struct STimeWindowAggSupp { + int8_t calTrigger; + int8_t calTriggerSaved; + int64_t deleteMark; + int64_t deleteMarkSaved; + int64_t waterMark; + TSKEY maxTs; + TSKEY minTs; + SColumnInfoData timeWindowData; // query time window info for scalar function execution. +} STimeWindowAggSupp; + +typedef struct SStreamScanInfo { + SExprInfo* pPseudoExpr; + int32_t numOfPseudoExpr; + SExprSupp tbnameCalSup; + SExprSupp tagCalSup; + int32_t primaryTsIndex; // primary time stamp slot id + SReadHandle readHandle; + SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. + SColMatchInfo matchInfo; + + SArray* pBlockLists; // multiple SSDatablock. + SSDataBlock* pRes; // result SSDataBlock + SSDataBlock* pUpdateRes; // update SSDataBlock + int32_t updateResIndex; + int32_t blockType; // current block type + int32_t validBlockIndex; // Is current data has returned? + uint64_t numOfExec; // execution times + STqReader* tqReader; + + uint64_t groupId; + SUpdateInfo* pUpdateInfo; + + EStreamScanMode scanMode; + struct SOperatorInfo* pStreamScanOp; + struct SOperatorInfo* pTableScanOp; + SArray* childIds; + SWindowSupporter windowSup; + SPartitionBySupporter partitionSup; + SExprSupp* pPartScalarSup; + bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock + SSDataBlock* pDeleteDataRes; // delete data SSDataBlock + int32_t deleteDataIndex; + STimeWindow updateWin; + STimeWindowAggSupp twAggSup; + SSDataBlock* pUpdateDataRes; + // status for tmq + SNodeList* pGroupTags; + SNode* pTagCond; + SNode* pTagIndexCond; + + // recover + int32_t blockRecoverContiCnt; + int32_t blockRecoverTotCnt; + SSDataBlock* pRecoverRes; + + SSDataBlock* pCreateTbRes; + int8_t igCheckUpdate; + int8_t igExpired; +} SStreamScanInfo; + +typedef struct { + SVnode* vnode; + SSDataBlock pRes; // result SSDataBlock + STsdbReader* dataReader; + SSnapContext* sContext; + STableListInfo* pTableListInfo; +} SStreamRawScanInfo; + +typedef struct STableCountScanSupp { + int16_t dbNameSlotId; + int16_t stbNameSlotId; + int16_t tbCountSlotId; + bool groupByDbName; + bool groupByStbName; + char dbNameFilter[TSDB_DB_NAME_LEN]; + char stbNameFilter[TSDB_TABLE_NAME_LEN]; +} STableCountScanSupp; + +typedef struct SOptrBasicInfo { + SResultRowInfo resultRowInfo; + SSDataBlock* pRes; + bool mergeResultBlock; +} SOptrBasicInfo; + +typedef struct SIntervalAggOperatorInfo { + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + STimeWindow win; // query time range + bool timeWindowInterpo; // interpolation needed or not + SArray* pInterpCols; // interpolation columns + int32_t resultTsOrder; // result timestamp order + int32_t inputOrder; // input data ts order + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + STimeWindowAggSupp twAggSup; + SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. +} SIntervalAggOperatorInfo; + +typedef struct SMergeAlignedIntervalAggOperatorInfo { + SIntervalAggOperatorInfo* intervalAggOperatorInfo; + + uint64_t groupId; // current groupId + int64_t curTs; // current ts + SSDataBlock* prefetchedBlock; + SResultRow* pResultRow; +} SMergeAlignedIntervalAggOperatorInfo; + +typedef struct SStreamIntervalOperatorInfo { + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + STimeWindowAggSupp twAggSup; + bool invertible; + bool ignoreExpiredData; + bool ignoreExpiredDataSaved; + SArray* pDelWins; // SWinRes + int32_t delIndex; + SSDataBlock* pDelRes; + SPhysiNode* pPhyNode; // create new child + SHashObj* pPullDataMap; + SArray* pPullWins; // SPullWindowInfo + int32_t pullIndex; + SSDataBlock* pPullDataRes; + bool isFinal; + SArray* pChildren; + SStreamState* pState; + SWinKey delKey; + uint64_t numOfDatapack; + SArray* pUpdated; + SSHashObj* pUpdatedMap; + int64_t dataVersion; +} SStreamIntervalOperatorInfo; + +typedef struct SDataGroupInfo { + uint64_t groupId; + int64_t numOfRows; + SArray* pPageList; +} SDataGroupInfo; + +typedef struct SWindowRowsSup { + STimeWindow win; + TSKEY prevTs; + int32_t startRowIndex; + int32_t numOfRows; + uint64_t groupId; +} SWindowRowsSup; + +typedef struct SResultWindowInfo { + void* pOutputBuf; + SSessionKey sessionWin; + bool isOutput; +} SResultWindowInfo; + +typedef struct SStreamSessionAggOperatorInfo { + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + int32_t endTsIndex; // window end timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SSDataBlock* pWinBlock; // window result + SSDataBlock* pDelRes; // delete result + SSDataBlock* pUpdateRes; // update window + bool returnUpdate; + SSHashObj* pStDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; final stream operator + SPhysiNode* pPhyNode; // create new child + bool isFinal; + bool ignoreExpiredData; + bool ignoreExpiredDataSaved; + SArray* pUpdated; + SSHashObj* pStUpdated; + int64_t dataVersion; +} SStreamSessionAggOperatorInfo; + +typedef struct SStreamStateAggOperatorInfo { + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + STimeWindowAggSupp twAggSup; + SColumn stateCol; + SSDataBlock* pDelRes; + SSHashObj* pSeDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; + bool ignoreExpiredData; + bool ignoreExpiredDataSaved; + SArray* pUpdated; + SSHashObj* pSeUpdated; + int64_t dataVersion; +} SStreamStateAggOperatorInfo; + +typedef struct SStreamPartitionOperatorInfo { + SOptrBasicInfo binfo; + SPartitionBySupporter partitionSup; + SExprSupp scalarSup; + SExprSupp tbnameCalSup; + SExprSupp tagCalSup; + SHashObj* pPartitions; + void* parIte; + void* pTbNameIte; + SSDataBlock* pInputDataBlock; + int32_t tsColIndex; + SSDataBlock* pDelRes; + SSDataBlock* pCreateTbRes; +} SStreamPartitionOperatorInfo; + +typedef struct SStreamFillSupporter { + int32_t type; // fill type + SInterval interval; + SResultRowData prev; + SResultRowData cur; + SResultRowData next; + SResultRowData nextNext; + SFillColInfo* pAllColInfo; // fill exprs and not fill exprs + SExprSupp notFillExprSup; + int32_t numOfAllCols; // number of all exprs, including the tags columns + int32_t numOfFillCols; + int32_t numOfNotFillCols; + int32_t rowSize; + SSHashObj* pResMap; + bool hasDelete; +} SStreamFillSupporter; + +typedef struct SStreamFillOperatorInfo { + SStreamFillSupporter* pFillSup; + SSDataBlock* pRes; + SSDataBlock* pSrcBlock; + int32_t srcRowIndex; + SSDataBlock* pSrcDelBlock; + int32_t srcDelRowIndex; + SSDataBlock* pDelRes; + SColMatchInfo matchInfo; + int32_t primaryTsCol; + int32_t primarySrcSlotId; + SStreamFillInfo* pFillInfo; +} SStreamFillOperatorInfo; + +#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) +#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) + +SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); +int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo); +void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo); + +void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); +void cleanupBasicInfo(SOptrBasicInfo* pInfo); + +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); +void cleanupExprSupp(SExprSupp* pSup); + +void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); + +int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, + const char* pkey, void* pState); +void cleanupAggSup(SAggSupporter* pAggSup); + +void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); + +void doBuildStreamResBlock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf); +void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf); + +bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); +bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo); +void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); +void resetLimitInfoForNextGroup(SLimitInfo* pLimitInfo); +bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); + +void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, + int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); + +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, + struct SOperatorInfo* pOperator); + +STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); +int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); + +extern void doDestroyExchangeOperatorInfo(void* param); + +void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); +int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, + int32_t rows, const char* idStr, STableMetaCacheInfo* pCache); + +void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); +void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name); + +void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); +void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); + +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, + int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, + bool isIntervalQuery, SAggSupporter* pSup, bool keepGroup); + +int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, + int32_t numOfOutput, SArray* pPseudoList); + +void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); + +int32_t checkForQueryBuf(size_t numOfTables); + +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle); + +STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, + int32_t order); +int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); +int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); +void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SSessionKey* pKey); +bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); +bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); +void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, + uint64_t* pGp, void* pTbName); +uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); + +int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, + SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); + +bool groupbyTbname(SNodeList* pGroupList); +int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo); +int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); +int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, + SExprSupp* pSup, SGroupResInfo* pGroupResInfo); +int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, + SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); +int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); +int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); +int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, + int64_t* pData); +void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, + SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); + +SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); +SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); + +void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx, + SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo); +void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset); +void doClearBufferedBlocks(SStreamScanInfo* pInfo); + uint64_t calcGroupId(char* pData, int32_t len); + #ifdef __cplusplus } #endif -#endif /*_TD_EXECUTOR_INT_H*/ \ No newline at end of file +#endif // TDENGINE_EXECUTORINT_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h deleted file mode 100644 index 4d19b8bb76..0000000000 --- a/source/libs/executor/inc/executorimpl.h +++ /dev/null @@ -1,699 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#ifndef TDENGINE_EXECUTORIMPL_H -#define TDENGINE_EXECUTORIMPL_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "os.h" -#include "tcommon.h" -#include "tlosertree.h" -#include "tsort.h" -#include "ttszip.h" -#include "tvariant.h" - -#include "dataSinkMgt.h" -#include "executil.h" -#include "executor.h" -#include "planner.h" -#include "scalar.h" -#include "taosdef.h" -#include "tarray.h" -#include "tfill.h" -#include "thash.h" -#include "tlockfree.h" -#include "tmsg.h" -#include "tpagedbuf.h" -#include "tstream.h" -#include "tstreamUpdate.h" - -#include "executorInt.h" -#include "vnode.h" - -typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); - -#define IS_VALID_SESSION_WIN(winInfo) ((winInfo).sessionWin.win.skey > 0) -#define SET_SESSION_WIN_INVALID(winInfo) ((winInfo).sessionWin.win.skey = INT64_MIN) -#define IS_INVALID_SESSION_WIN_KEY(winKey) ((winKey).win.skey <= 0) -#define SET_SESSION_WIN_KEY_INVALID(pWinKey) ((pWinKey)->win.skey = INT64_MIN) - -enum { - // when this task starts to execute, this status will set - TASK_NOT_COMPLETED = 0x1u, - - /* Task is over - * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. - * 2. when all data within queried time window, it is also denoted as query_completed - */ - TASK_COMPLETED = 0x2u, -}; - -/** - * If the number of generated results is greater than this value, - * query query will be halt and return results to client immediate. - */ -typedef struct SResultInfo { // TODO refactor - int64_t totalRows; // total generated result size in rows - int64_t totalBytes; // total results in bytes. - int32_t capacity; // capacity of current result output buffer - int32_t threshold; // result size threshold in rows. -} SResultInfo; - -typedef struct STableQueryInfo { - TSKEY lastKey; // last check ts, todo remove it later - SResultRowPosition pos; // current active time window -} STableQueryInfo; - -typedef struct SLimit { - int64_t limit; - int64_t offset; -} SLimit; - -typedef struct STableScanAnalyzeInfo SFileBlockLoadRecorder; - -enum { - STREAM_RECOVER_STEP__NONE = 0, - STREAM_RECOVER_STEP__PREPARE1, - STREAM_RECOVER_STEP__PREPARE2, - STREAM_RECOVER_STEP__SCAN1, - STREAM_RECOVER_STEP__SCAN2, -}; - -typedef struct { - STqOffsetVal currentOffset; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int64_t snapshotVer; - SPackedData submit; - SSchemaWrapper* schema; - char tbName[TSDB_TABLE_NAME_LEN]; - int8_t recoverStep; - int8_t recoverScanFinished; - SQueryTableDataCond tableCond; - int64_t fillHistoryVer1; - int64_t fillHistoryVer2; - SStreamState* pState; - int64_t dataVersion; - int64_t checkPointId; -} SStreamTaskInfo; - -typedef struct { - char* tablename; - char* dbname; - int32_t tversion; - SSchemaWrapper* sw; - SSchemaWrapper* qsw; -} SSchemaInfo; - -typedef struct SExchangeOpStopInfo { - int32_t operatorType; - int64_t refId; -} SExchangeOpStopInfo; - -typedef struct SExprSupp { - SExprInfo* pExprInfo; - int32_t numOfExprs; // the number of scalar expression in group operator - SqlFunctionCtx* pCtx; - int32_t* rowEntryInfoOffset; // offset value for each row result cell info - SFilterInfo* pFilterInfo; -} SExprSupp; - -typedef enum { - EX_SOURCE_DATA_NOT_READY = 0x1, - EX_SOURCE_DATA_READY = 0x2, - EX_SOURCE_DATA_EXHAUSTED = 0x3, -} EX_SOURCE_STATUS; - -#define COL_MATCH_FROM_COL_ID 0x1 -#define COL_MATCH_FROM_SLOT_ID 0x2 - -typedef struct SLoadRemoteDataInfo { - uint64_t totalSize; // total load bytes from remote - uint64_t totalRows; // total number of rows - uint64_t totalElapsed; // total elapsed time -} SLoadRemoteDataInfo; - -typedef struct SLimitInfo { - SLimit limit; - SLimit slimit; - uint64_t currentGroupId; - int64_t remainGroupOffset; - int64_t numOfOutputGroups; - int64_t remainOffset; - int64_t numOfOutputRows; -} SLimitInfo; - -typedef struct SExchangeInfo { - SArray* pSources; - SArray* pSourceDataInfo; - tsem_t ready; - void* pTransporter; - - // SArray, result block list, used to keep the multi-block that - // passed by downstream operator - SArray* pResultBlockList; - SArray* pRecycledBlocks; // build a pool for small data block to avoid to repeatly create and then destroy. - SSDataBlock* pDummyBlock; // dummy block, not keep data - bool seqLoadData; // sequential load data or not, false by default - int32_t current; - SLoadRemoteDataInfo loadInfo; - uint64_t self; - SLimitInfo limitInfo; - int64_t openedTs; // start exec time stamp, todo: move to SLoadRemoteDataInfo -} SExchangeInfo; - -typedef struct SScanInfo { - int32_t numOfAsc; - int32_t numOfDesc; -} SScanInfo; - -typedef struct SSampleExecInfo { - double sampleRatio; // data block sample ratio, 1 by default - uint32_t seed; // random seed value -} SSampleExecInfo; - -enum { - TABLE_SCAN__TABLE_ORDER = 1, - TABLE_SCAN__BLOCK_ORDER = 2, -}; - -typedef struct SAggSupporter { - SSHashObj* pResultRowHashTable; // quick locate the window object for each result - char* keyBuf; // window key buffer - SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - int32_t currentPageId; // current write page id -} SAggSupporter; - -typedef struct { - // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if - // current data block needs to be loaded. - SInterval interval; - SAggSupporter* pAggSup; - SExprSupp* pExprSup; // expr supporter of aggregate operator -} SAggOptrPushDownInfo; - -typedef struct STableMetaCacheInfo { - SLRUCache* pTableMetaEntryCache; // 100 by default - uint64_t metaFetch; - uint64_t cacheHit; -} STableMetaCacheInfo; - -typedef struct STableScanBase { - STsdbReader* dataReader; - SFileBlockLoadRecorder readRecorder; - SQueryTableDataCond cond; - SAggOptrPushDownInfo pdInfo; - SColMatchInfo matchInfo; - SReadHandle readHandle; - SExprSupp pseudoSup; - STableMetaCacheInfo metaCache; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - SLimitInfo limitInfo; - // there are more than one table list exists in one task, if only one vnode exists. - STableListInfo* pTableListInfo; -} STableScanBase; - -typedef struct STableScanInfo { - STableScanBase base; - SScanInfo scanInfo; - int32_t scanTimes; - SSDataBlock* pResBlock; - SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; - int8_t scanMode; - int8_t assignBlockUid; - bool hasGroupByTag; - bool countOnly; -} STableScanInfo; - -typedef struct STableMergeScanInfo { - int32_t tableStartIndex; - int32_t tableEndIndex; - bool hasGroupId; - uint64_t groupId; - SArray* queryConds; // array of queryTableDataCond - STableScanBase base; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SSDataBlock* pSortInputBlock; - int64_t startTs; // sort start time - SArray* sortSourceParams; - SLimitInfo limitInfo; - int64_t numOfRows; - SScanInfo scanInfo; - int32_t scanTimes; - SSDataBlock* pResBlock; - SSampleExecInfo sample; // sample execution info - SSortExecInfo sortExecInfo; -} STableMergeScanInfo; - -typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; - SColMatchInfo matchInfo; - int32_t curPos; - SReadHandle readHandle; - STableListInfo* pTableListInfo; -} STagScanInfo; - -typedef enum EStreamScanMode { - STREAM_SCAN_FROM_READERHANDLE = 1, - STREAM_SCAN_FROM_RES, - STREAM_SCAN_FROM_UPDATERES, - STREAM_SCAN_FROM_DELETE_DATA, - STREAM_SCAN_FROM_DATAREADER_RETRIEVE, - STREAM_SCAN_FROM_DATAREADER_RANGE, -} EStreamScanMode; - -enum { - PROJECT_RETRIEVE_CONTINUE = 0x1, - PROJECT_RETRIEVE_DONE = 0x2, -}; - -typedef struct SStreamAggSupporter { - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - SSDataBlock* pScanBlock; - SStreamState* pState; - int64_t gap; // stream session window gap - SqlFunctionCtx* pDummyCtx; // for combine - SSHashObj* pResultRows; - int32_t stateKeySize; - int16_t stateKeyType; - SDiskbasedBuf* pResultBuf; -} SStreamAggSupporter; - -typedef struct SWindowSupporter { - SStreamAggSupporter* pStreamAggSup; - int64_t gap; - uint16_t parentType; - SAggSupporter* pIntervalAggSup; -} SWindowSupporter; - -typedef struct SPartitionBySupporter { - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - char* keyBuf; // group by keys for hash - bool needCalc; // partition by column -} SPartitionBySupporter; - -typedef struct SPartitionDataInfo { - uint64_t groupId; - char* tbname; - SArray* tags; - SArray* rowIds; -} SPartitionDataInfo; - -typedef struct STimeWindowAggSupp { - int8_t calTrigger; - int8_t calTriggerSaved; - int64_t deleteMark; - int64_t deleteMarkSaved; - int64_t waterMark; - TSKEY maxTs; - TSKEY minTs; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. -} STimeWindowAggSupp; - -typedef struct SStreamScanInfo { - SExprInfo* pPseudoExpr; - int32_t numOfPseudoExpr; - SExprSupp tbnameCalSup; - SExprSupp tagCalSup; - int32_t primaryTsIndex; // primary time stamp slot id - SReadHandle readHandle; - SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. - SColMatchInfo matchInfo; - - SArray* pBlockLists; // multiple SSDatablock. - SSDataBlock* pRes; // result SSDataBlock - SSDataBlock* pUpdateRes; // update SSDataBlock - int32_t updateResIndex; - int32_t blockType; // current block type - int32_t validBlockIndex; // Is current data has returned? - uint64_t numOfExec; // execution times - STqReader* tqReader; - - uint64_t groupId; - SUpdateInfo* pUpdateInfo; - - EStreamScanMode scanMode; - struct SOperatorInfo* pStreamScanOp; - struct SOperatorInfo* pTableScanOp; - SArray* childIds; - SWindowSupporter windowSup; - SPartitionBySupporter partitionSup; - SExprSupp* pPartScalarSup; - bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. - int32_t scanWinIndex; // for state operator - int32_t pullDataResIndex; - SSDataBlock* pPullDataRes; // pull data SSDataBlock - SSDataBlock* pDeleteDataRes; // delete data SSDataBlock - int32_t deleteDataIndex; - STimeWindow updateWin; - STimeWindowAggSupp twAggSup; - SSDataBlock* pUpdateDataRes; - // status for tmq - SNodeList* pGroupTags; - SNode* pTagCond; - SNode* pTagIndexCond; - - // recover - int32_t blockRecoverContiCnt; - int32_t blockRecoverTotCnt; - SSDataBlock* pRecoverRes; - - SSDataBlock* pCreateTbRes; - int8_t igCheckUpdate; - int8_t igExpired; -} SStreamScanInfo; - -typedef struct { - SVnode* vnode; - SSDataBlock pRes; // result SSDataBlock - STsdbReader* dataReader; - SSnapContext* sContext; - STableListInfo* pTableListInfo; -} SStreamRawScanInfo; - -typedef struct STableCountScanSupp { - int16_t dbNameSlotId; - int16_t stbNameSlotId; - int16_t tbCountSlotId; - bool groupByDbName; - bool groupByStbName; - char dbNameFilter[TSDB_DB_NAME_LEN]; - char stbNameFilter[TSDB_TABLE_NAME_LEN]; -} STableCountScanSupp; - -typedef struct SOptrBasicInfo { - SResultRowInfo resultRowInfo; - SSDataBlock* pRes; - bool mergeResultBlock; -} SOptrBasicInfo; - -typedef struct SIntervalAggOperatorInfo { - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - STimeWindow win; // query time range - bool timeWindowInterpo; // interpolation needed or not - SArray* pInterpCols; // interpolation columns - int32_t resultTsOrder; // result timestamp order - int32_t inputOrder; // input data ts order - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] - STimeWindowAggSupp twAggSup; - SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. -} SIntervalAggOperatorInfo; - -typedef struct SMergeAlignedIntervalAggOperatorInfo { - SIntervalAggOperatorInfo* intervalAggOperatorInfo; - - uint64_t groupId; // current groupId - int64_t curTs; // current ts - SSDataBlock* prefetchedBlock; - SResultRow* pResultRow; -} SMergeAlignedIntervalAggOperatorInfo; - -typedef struct SStreamIntervalOperatorInfo { - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - STimeWindowAggSupp twAggSup; - bool invertible; - bool ignoreExpiredData; - bool ignoreExpiredDataSaved; - SArray* pDelWins; // SWinRes - int32_t delIndex; - SSDataBlock* pDelRes; - SPhysiNode* pPhyNode; // create new child - SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo - int32_t pullIndex; - SSDataBlock* pPullDataRes; - bool isFinal; - SArray* pChildren; - SStreamState* pState; - SWinKey delKey; - uint64_t numOfDatapack; - SArray* pUpdated; - SSHashObj* pUpdatedMap; - int64_t dataVersion; -} SStreamIntervalOperatorInfo; - -typedef struct SDataGroupInfo { - uint64_t groupId; - int64_t numOfRows; - SArray* pPageList; -} SDataGroupInfo; - -typedef struct SWindowRowsSup { - STimeWindow win; - TSKEY prevTs; - int32_t startRowIndex; - int32_t numOfRows; - uint64_t groupId; -} SWindowRowsSup; - -typedef struct SResultWindowInfo { - void* pOutputBuf; - SSessionKey sessionWin; - bool isOutput; -} SResultWindowInfo; - -typedef struct SStateWindowInfo { - SResultWindowInfo winInfo; - SStateKeys* pStateKey; -} SStateWindowInfo; - -typedef struct SStreamSessionAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int32_t primaryTsIndex; // primary timestamp slot id - int32_t endTsIndex; // window end timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SSDataBlock* pWinBlock; // window result - SSDataBlock* pDelRes; // delete result - SSDataBlock* pUpdateRes; // update window - bool returnUpdate; - SSHashObj* pStDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; final stream operator - SPhysiNode* pPhyNode; // create new child - bool isFinal; - bool ignoreExpiredData; - bool ignoreExpiredDataSaved; - SArray* pUpdated; - SSHashObj* pStUpdated; - int64_t dataVersion; -} SStreamSessionAggOperatorInfo; - -typedef struct SStreamStateAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int32_t primaryTsIndex; // primary timestamp slot id - STimeWindowAggSupp twAggSup; - SColumn stateCol; - SSDataBlock* pDelRes; - SSHashObj* pSeDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; - bool ignoreExpiredData; - bool ignoreExpiredDataSaved; - SArray* pUpdated; - SSHashObj* pSeUpdated; - int64_t dataVersion; -} SStreamStateAggOperatorInfo; - -typedef struct SStreamPartitionOperatorInfo { - SOptrBasicInfo binfo; - SPartitionBySupporter partitionSup; - SExprSupp scalarSup; - SExprSupp tbnameCalSup; - SExprSupp tagCalSup; - SHashObj* pPartitions; - void* parIte; - void* pTbNameIte; - SSDataBlock* pInputDataBlock; - int32_t tsColIndex; - SSDataBlock* pDelRes; - SSDataBlock* pCreateTbRes; -} SStreamPartitionOperatorInfo; - -typedef struct SStreamFillSupporter { - int32_t type; // fill type - SInterval interval; - SResultRowData prev; - SResultRowData cur; - SResultRowData next; - SResultRowData nextNext; - SFillColInfo* pAllColInfo; // fill exprs and not fill exprs - SExprSupp notFillExprSup; - int32_t numOfAllCols; // number of all exprs, including the tags columns - int32_t numOfFillCols; - int32_t numOfNotFillCols; - int32_t rowSize; - SSHashObj* pResMap; - bool hasDelete; -} SStreamFillSupporter; - -typedef struct SStreamFillOperatorInfo { - SStreamFillSupporter* pFillSup; - SSDataBlock* pRes; - SSDataBlock* pSrcBlock; - int32_t srcRowIndex; - SSDataBlock* pSrcDelBlock; - int32_t srcDelRowIndex; - SSDataBlock* pDelRes; - SColMatchInfo matchInfo; - int32_t primaryTsCol; - int32_t primarySrcSlotId; - SStreamFillInfo* pFillInfo; -} SStreamFillOperatorInfo; - -#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) -#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) - -SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode); -int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo); - -void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); -void cleanupBasicInfo(SOptrBasicInfo* pInfo); - -int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); -void cleanupExprSupp(SExprSupp* pSup); - -void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); - -int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, - const char* pkey, void* pState); -void cleanupAggSup(SAggSupporter* pAggSup); - -void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); - -void doBuildStreamResBlock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, - SDiskbasedBuf* pBuf); -void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, - SDiskbasedBuf* pBuf); - -bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); -bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo); -void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); -void resetLimitInfoForNextGroup(SLimitInfo* pLimitInfo); -bool applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); - -void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, - int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); - -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); -void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int64_t numOfRows, int32_t dataLen, int64_t startTs, - struct SOperatorInfo* pOperator); - -STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); - -int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); - -extern void doDestroyExchangeOperatorInfo(void* param); - -void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo); -int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock, - int32_t rows, const char* idStr, STableMetaCacheInfo* pCache); - -void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); -void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name); - -void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); -void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); - -SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, - int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, - bool isIntervalQuery, SAggSupporter* pSup, bool keepGroup); - -int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput, SArray* pPseudoList); - -void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); - -int32_t checkForQueryBuf(size_t numOfTables); - -SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); - -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle); -int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); - -STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, - int32_t order); -int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, - __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); -int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); -void getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, SSessionKey* pKey); -bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); -bool functionNeedToExecute(SqlFunctionCtx* pCtx); -bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); -bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); -void appendOneRowToStreamSpecialBlock(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, - uint64_t* pGp, void* pTbName); -uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); - -int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, - SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); - -bool groupbyTbname(SNodeList* pGroupList); -int32_t buildDataBlockFromGroupRes(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, - SGroupResInfo* pGroupResInfo); -int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size); -int32_t buildSessionResultDataBlock(struct SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, - SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, - SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); -int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); -int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); -void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); -int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int32_t pos, int32_t order, - int64_t* pData); -void appendCreateTableRow(SStreamState* pState, SExprSupp* pTableSup, SExprSupp* pTagSup, uint64_t groupId, - SSDataBlock* pSrcBlock, int32_t rowId, SSDataBlock* pDestBlock); - -SSDataBlock* buildCreateTableBlock(SExprSupp* tbName, SExprSupp* tag); -SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs); - -void copyResultrowToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx, - SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo); -void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t numOfExprs, const int32_t* rowEntryOffset); -void doClearBufferedBlocks(SStreamScanInfo* pInfo); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 92018b6e3e..632b817a07 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -157,6 +157,7 @@ void destroyOperator(SOperatorInfo* pOperator); SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id); int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder); int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr); +int32_t getOperatorExplainExecInfo(struct SOperatorInfo* operatorInfo, SArray* pExecInfoList); #ifdef __cplusplus } diff --git a/source/libs/executor/inc/querytask.h b/source/libs/executor/inc/querytask.h index aa42cd2c14..8852265da0 100644 --- a/source/libs/executor/inc/querytask.h +++ b/source/libs/executor/inc/querytask.h @@ -22,6 +22,17 @@ extern "C" { #define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) +enum { + // when this task starts to execute, this status will set + TASK_NOT_COMPLETED = 0x1u, + + /* Task is over + * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. + * 2. when all data within queried time window, it is also denoted as query_completed + */ + TASK_COMPLETED = 0x2u, +}; + typedef struct STaskIdInfo { uint64_t queryId; // this is also a request id uint64_t subplanId; @@ -44,6 +55,23 @@ typedef struct STaskStopInfo { SArray* pStopInfo; } STaskStopInfo; +typedef struct { + STqOffsetVal currentOffset; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int64_t snapshotVer; + SPackedData submit; // todo remove it + SSchemaWrapper* schema; + char tbName[TSDB_TABLE_NAME_LEN]; // this is the current scan table: todo refactor + int8_t recoverStep; + int8_t recoverScanFinished; + SQueryTableDataCond tableCond; + int64_t fillHistoryVer1; + int64_t fillHistoryVer2; + SStreamState* pState; + int64_t dataVersion; + int64_t checkPointId; +} SStreamTaskInfo; + struct SExecTaskInfo { STaskIdInfo id; uint32_t status; @@ -75,6 +103,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); int32_t qAppendTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo); +SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); #ifdef __cplusplus } diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index c00c167d8e..9b463a3dee 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -20,16 +20,16 @@ #include "tfill.h" #include "tname.h" +#include "executorInt.h" +#include "index.h" +#include "operator.h" +#include "query.h" +#include "querytask.h" +#include "tcompare.h" #include "tdatablock.h" #include "tglobal.h" -#include "executorimpl.h" -#include "index.h" -#include "query.h" -#include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "operator.h" -#include "querytask.h" typedef struct { bool hasAgg; diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 6fee0899ce..eec34a6406 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -20,12 +20,12 @@ #include "tdatablock.h" #include "tmsg.h" -#include "executorimpl.h" +#include "executorInt.h" +#include "operator.h" +#include "querytask.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "operator.h" -#include "querytask.h" typedef struct SCacheRowsScanInfo { SSDataBlock* pRes; diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 96d061fc04..11074b0e94 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -15,7 +15,7 @@ #include "dataSinkInt.h" #include "dataSinkMgt.h" -#include "executorimpl.h" +#include "executorInt.h" #include "planner.h" #include "tcompression.h" #include "tdatablock.h" diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 8e32559fac..ce8dc898a5 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -15,7 +15,7 @@ #include "dataSinkInt.h" #include "dataSinkMgt.h" -#include "executorimpl.h" +#include "executorInt.h" #include "planner.h" #include "tcompression.h" #include "tdatablock.h" diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 22f388d406..33eccf4759 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -15,7 +15,7 @@ #include "dataSinkInt.h" #include "dataSinkMgt.h" -#include "executorimpl.h" +#include "executorInt.h" #include "planner.h" #include "tcompression.h" #include "tdatablock.h" diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 8ec7ed91a3..956d5b714d 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -13,16 +13,16 @@ * along with this program. If not, see . */ -#include "executorimpl.h" +#include "executorInt.h" #include "filter.h" #include "function.h" #include "functionMgt.h" +#include "operator.h" +#include "querytask.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "ttime.h" -#include "operator.h" -#include "querytask.h" typedef struct SEventWindowOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 31445dd568..94041140d4 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -13,19 +13,19 @@ * along with this program. If not, see . */ +#include "executorInt.h" #include "filter.h" #include "function.h" +#include "index.h" +#include "operator.h" #include "os.h" +#include "query.h" +#include "querytask.h" +#include "tdatablock.h" +#include "thash.h" +#include "tmsg.h" #include "tname.h" #include "tref.h" -#include "tdatablock.h" -#include "tmsg.h" -#include "executorimpl.h" -#include "index.h" -#include "query.h" -#include "thash.h" -#include "operator.h" -#include "querytask.h" typedef struct SFetchRspHandleWrapper { uint32_t exchangeId; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 75e1d374e4..c51dc39b5b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -24,9 +24,9 @@ #include "ttime.h" #include "executil.h" -#include "executorimpl.h" -#include "tcompression.h" +#include "executorInt.h" #include "querytask.h" +#include "tcompression.h" typedef struct STableListIdInfo { uint64_t suid; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2b9ab6c09a..7a5715e5ed 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,14 +14,14 @@ */ #include "executor.h" -#include "executorimpl.h" +#include "executorInt.h" +#include "operator.h" #include "planner.h" +#include "querytask.h" #include "tdatablock.h" #include "tref.h" #include "tudf.h" #include "vnode.h" -#include "operator.h" -#include "querytask.h" static TdThreadOnce initPoolOnce = PTHREAD_ONCE_INIT; int32_t exchangeObjRefPool = -1; @@ -718,8 +718,6 @@ void qRemoveTaskStopInfo(SExecTaskInfo* pTaskInfo, SExchangeOpStopInfo* pInfo) { taosArrayRemove(pTaskInfo->stopInfo.pStopInfo, idx); } taosWUnLockLatch(&pTaskInfo->stopInfo.lock); - - return; } void qStopTaskOperators(SExecTaskInfo* pTaskInfo) { @@ -1304,3 +1302,25 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { taosArrayDestroy(plist); return pUidList; } + +static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pScanInfo = pOperator->info; + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo); + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + STableScanInfo* pScanInfo = pOperator->info; + taosArrayPush(pList, &pScanInfo->base.pTableListInfo); + } else { + if (pOperator->pDownstream != NULL && pOperator->pDownstream[0] != NULL) { + extractTableList(pList, pOperator->pDownstream[0]); + } + } +} + +SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) { + SArray* pArray = taosArrayInit(0, POINTER_BYTES); + SOperatorInfo* pOperator = pTaskInfo->pRoot; + extractTableList(pArray, pOperator); + return pArray; +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorInt.c similarity index 95% rename from source/libs/executor/src/executorimpl.c rename to source/libs/executor/src/executorInt.c index 3ab929cad0..f525f6728c 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorInt.c @@ -25,15 +25,15 @@ #include "tmsg.h" #include "ttime.h" -#include "executorimpl.h" +#include "executorInt.h" #include "index.h" +#include "operator.h" #include "query.h" +#include "querytask.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" #include "vnode.h" -#include "operator.h" -#include "querytask.h" #define SET_REVERSE_SCAN_FLAG(runtime) ((runtime)->scanFlag = REVERSE_SCAN) #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) @@ -1059,37 +1059,6 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* return TSDB_CODE_SUCCESS; } -int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList) { - SExplainExecInfo execInfo = {0}; - SExplainExecInfo* pExplainInfo = taosArrayPush(pExecInfoList, &execInfo); - - pExplainInfo->numOfRows = operatorInfo->resultInfo.totalRows; - pExplainInfo->startupCost = operatorInfo->cost.openCost; - pExplainInfo->totalCost = operatorInfo->cost.totalCost; - pExplainInfo->verboseLen = 0; - pExplainInfo->verboseInfo = NULL; - - if (operatorInfo->fpSet.getExplainFn) { - int32_t code = - operatorInfo->fpSet.getExplainFn(operatorInfo, &pExplainInfo->verboseInfo, &pExplainInfo->verboseLen); - if (code) { - qError("%s operator getExplainFn failed, code:%s", GET_TASKID(operatorInfo->pTaskInfo), tstrerror(code)); - return code; - } - } - - int32_t code = 0; - for (int32_t i = 0; i < operatorInfo->numOfDownstream; ++i) { - code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pExecInfoList); - if (code != TSDB_CODE_SUCCESS) { - // taosMemoryFreeClear(*pRes); - return TSDB_CODE_OUT_OF_MEMORY; - } - } - - return TSDB_CODE_SUCCESS; -} - int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup) { SWinKey key = { @@ -1331,25 +1300,3 @@ void qStreamCloseTsdbReader(void* task) { } } } - -static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - SStreamScanInfo* pScanInfo = pOperator->info; - STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; - taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo); - } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { - STableScanInfo* pScanInfo = pOperator->info; - taosArrayPush(pList, &pScanInfo->base.pTableListInfo); - } else { - if (pOperator->pDownstream != NULL && pOperator->pDownstream[0] != NULL) { - extractTableList(pList, pOperator->pDownstream[0]); - } - } -} - -SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo) { - SArray* pArray = taosArrayInit(0, POINTER_BYTES); - SOperatorInfo* pOperator = pTaskInfo->pRoot; - extractTableList(pArray, pOperator); - return pArray; -} \ No newline at end of file diff --git a/source/libs/executor/src/filloperator.c b/source/libs/executor/src/filloperator.c index e465e6a525..0ac9e6097f 100644 --- a/source/libs/executor/src/filloperator.c +++ b/source/libs/executor/src/filloperator.c @@ -20,7 +20,7 @@ #include "tmsg.h" #include "ttypes.h" -#include "executorimpl.h" +#include "executorInt.h" #include "tcommon.h" #include "thash.h" #include "ttime.h" diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 821db4244e..47338d4469 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -22,12 +22,11 @@ #include "tmsg.h" #include "executorInt.h" -#include "executorimpl.h" +#include "operator.h" +#include "querytask.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "operator.h" -#include "querytask.h" typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index b4862a490f..754b5f4737 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -13,18 +13,18 @@ * along with this program. If not, see . */ +#include "executorInt.h" #include "filter.h" -#include "executorimpl.h" #include "function.h" +#include "operator.h" #include "os.h" #include "querynodes.h" +#include "querytask.h" #include "tcompare.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" #include "ttypes.h" -#include "operator.h" -#include "querytask.h" typedef struct SJoinRowCtx { bool rowRemains; diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 5e384ca6dd..729178dc60 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -20,12 +20,12 @@ #include "tglobal.h" -#include "executorimpl.h" +#include "executorInt.h" #include "index.h" -#include "query.h" -#include "vnode.h" #include "operator.h" +#include "query.h" #include "querytask.h" +#include "vnode.h" SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn, @@ -75,28 +75,6 @@ void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, b pOperator->pTaskInfo = pTaskInfo; } -void destroyOperator(SOperatorInfo* pOperator) { - if (pOperator == NULL) { - return; - } - - if (pOperator->fpSet.closeFn != NULL) { - pOperator->fpSet.closeFn(pOperator->info); - } - - if (pOperator->pDownstream != NULL) { - for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - destroyOperator(pOperator->pDownstream[i]); - } - - taosMemoryFreeClear(pOperator->pDownstream); - pOperator->numOfDownstream = 0; - } - - cleanupExprSupp(&pOperator->exprSupp); - taosMemoryFreeClear(pOperator); -} - // each operator should be set their own function to return total cost buffer int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { if (pOperator->blocking) { @@ -106,40 +84,6 @@ int32_t optrDefaultBufFn(SOperatorInfo* pOperator) { } } -//int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag, bool inheritUsOrder) { -// // todo add more information about exchange operation -// int32_t type = pOperator->operatorType; -// if (type == QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN || -// type == QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN || -// type == QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN || type == QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN) { -// *order = TSDB_ORDER_ASC; -// *scanFlag = MAIN_SCAN; -// return TSDB_CODE_SUCCESS; -// } else if (type == QUERY_NODE_PHYSICAL_PLAN_EXCHANGE) { -// if (!inheritUsOrder) { -// *order = TSDB_ORDER_ASC; -// } -// *scanFlag = MAIN_SCAN; -// return TSDB_CODE_SUCCESS; -// } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { -// STableScanInfo* pTableScanInfo = pOperator->info; -// *order = pTableScanInfo->base.cond.order; -// *scanFlag = pTableScanInfo->base.scanFlag; -// return TSDB_CODE_SUCCESS; -// } else if (type == QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN) { -// STableMergeScanInfo* pTableScanInfo = pOperator->info; -// *order = pTableScanInfo->base.cond.order; -// *scanFlag = pTableScanInfo->base.scanFlag; -// return TSDB_CODE_SUCCESS; -// } else { -// if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { -// return TSDB_CODE_INVALID_PARA; -// } else { -// return getTableScanInfo(pOperator->pDownstream[0], order, scanFlag, inheritUsOrder); -// } -// } -//} - static int64_t getQuerySupportBufSize(size_t numOfTables) { size_t s1 = sizeof(STableQueryInfo); // size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb @@ -319,7 +263,7 @@ int32_t stopTableScanOperator(SOperatorInfo* pOperator, const char* pIdStr) { } SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, SNode* pTagCond, - SNode* pTagIndexCond, const char* pUser, const char* dbname) { + SNode* pTagIndexCond, const char* pUser, const char* dbname) { int32_t type = nodeType(pPhyNode); const char* idstr = GET_TASKID(pTaskInfo); @@ -347,7 +291,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR code = initQueriedTableSchemaInfo(pHandle, &pTableScanNode->scan, dbname, pTaskInfo); if (code) { - pTaskInfo->code = terrno; + pTaskInfo->code = code; tableListDestroy(pTableListInfo); return NULL; } @@ -355,6 +299,7 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pOperator = createTableScanOperatorInfo(pTableScanNode, pHandle, pTableListInfo, pTaskInfo); if (NULL == pOperator) { pTaskInfo->code = terrno; + tableListDestroy(pTableListInfo); return NULL; } @@ -578,3 +523,56 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR return pOptr; } + +void destroyOperator(SOperatorInfo* pOperator) { + if (pOperator == NULL) { + return; + } + + if (pOperator->fpSet.closeFn != NULL) { + pOperator->fpSet.closeFn(pOperator->info); + } + + if (pOperator->pDownstream != NULL) { + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { + destroyOperator(pOperator->pDownstream[i]); + } + + taosMemoryFreeClear(pOperator->pDownstream); + pOperator->numOfDownstream = 0; + } + + cleanupExprSupp(&pOperator->exprSupp); + taosMemoryFreeClear(pOperator); +} + +int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList) { + SExplainExecInfo execInfo = {0}; + SExplainExecInfo* pExplainInfo = taosArrayPush(pExecInfoList, &execInfo); + + pExplainInfo->numOfRows = operatorInfo->resultInfo.totalRows; + pExplainInfo->startupCost = operatorInfo->cost.openCost; + pExplainInfo->totalCost = operatorInfo->cost.totalCost; + pExplainInfo->verboseLen = 0; + pExplainInfo->verboseInfo = NULL; + + if (operatorInfo->fpSet.getExplainFn) { + int32_t code = + operatorInfo->fpSet.getExplainFn(operatorInfo, &pExplainInfo->verboseInfo, &pExplainInfo->verboseLen); + if (code) { + qError("%s operator getExplainFn failed, code:%s", GET_TASKID(operatorInfo->pTaskInfo), tstrerror(code)); + return code; + } + } + + int32_t code = 0; + for (int32_t i = 0; i < operatorInfo->numOfDownstream; ++i) { + code = getOperatorExplainExecInfo(operatorInfo->pDownstream[i], pExecInfoList); + if (code != TSDB_CODE_SUCCESS) { + // taosMemoryFreeClear(*pRes); + return TSDB_CODE_OUT_OF_MEMORY; + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 01ee18b41f..02f504bef0 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "executorimpl.h" +#include "executorInt.h" #include "filter.h" #include "functionMgt.h" #include "operator.h" diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index ed209efed0..b6b250a325 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -24,14 +24,14 @@ #include "tdatablock.h" #include "tmsg.h" -#include "executorimpl.h" +#include "executorInt.h" #include "index.h" +#include "operator.h" #include "query.h" +#include "querytask.h" #include "thash.h" #include "ttypes.h" #include "vnode.h" -#include "operator.h" -#include "querytask.h" #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a717c8b7c0..130cca9cbb 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "executorimpl.h" +#include "executorInt.h" #include "filter.h" #include "function.h" #include "functionMgt.h" diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0b23a803c6..10933f285c 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ +#include "executorInt.h" #include "filter.h" -#include "executorimpl.h" -#include "tdatablock.h" #include "operator.h" #include "querytask.h" +#include "tdatablock.h" typedef struct SSortOperatorInfo { SOptrBasicInfo binfo; diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index bc850b5333..c75c49fe77 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "executorimpl.h" +#include "executorInt.h" #include "filter.h" #include "function.h" #include "functionMgt.h" diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 8376caace0..fc4e82b57f 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -20,7 +20,7 @@ #include "tmsg.h" #include "ttypes.h" -#include "executorimpl.h" +#include "executorInt.h" #include "tcommon.h" #include "thash.h" #include "ttime.h" diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index a688920a22..29e3668ec4 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -12,17 +12,17 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "executorimpl.h" +#include "executorInt.h" #include "filter.h" #include "function.h" #include "functionMgt.h" +#include "operator.h" +#include "querytask.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" -#include "operator.h" -#include "querytask.h" typedef struct STimeSliceOperatorInfo { SSDataBlock* pRes; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index ab9b0987fa..bea01fa0d8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -12,21 +12,27 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "executorimpl.h" +#include "executorInt.h" #include "filter.h" #include "function.h" #include "functionMgt.h" +#include "operator.h" +#include "querytask.h" #include "tcommon.h" #include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" -#include "operator.h" -#include "querytask.h" #define IS_FINAL_OP(op) ((op)->isFinal) #define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL); +typedef struct SStateWindowInfo { + SResultWindowInfo winInfo; + SStateKeys* pStateKey; +} SStateWindowInfo; + + typedef struct SSessionAggOperatorInfo { SOptrBasicInfo binfo; SAggSupporter aggSup; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index db0678f214..cefe12990d 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -24,13 +24,13 @@ #include "os.h" #include "executor.h" -#include "executorimpl.h" +#include "executorInt.h" #include "function.h" +#include "operator.h" #include "taos.h" #include "tdatablock.h" #include "tdef.h" #include "tvariant.h" -#include "operator.h" namespace { diff --git a/source/libs/executor/test/lhashTests.cpp b/source/libs/executor/test/lhashTests.cpp index 24570ff788..92f7652d8d 100644 --- a/source/libs/executor/test/lhashTests.cpp +++ b/source/libs/executor/test/lhashTests.cpp @@ -15,7 +15,7 @@ #include #include -#include "executorimpl.h" +#include "executorInt.h" #include "tlinearhash.h" #pragma GCC diagnostic push diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index f35d07804e..8122d7d6a9 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -26,7 +26,7 @@ #include "os.h" #include "executor.h" -#include "executorimpl.h" +#include "executorInt.h" #include "taos.h" #include "tcompare.h" #include "tdatablock.h" From f9a64cbcd6c8b4b6551b58feef01880c0a308990 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 12:58:03 +0800 Subject: [PATCH 100/145] other: add some logs. --- source/dnode/vnode/src/tq/tq.c | 20 +++++++++++--------- source/dnode/vnode/src/tq/tqRestore.c | 2 +- source/libs/stream/src/streamExec.c | 12 ++++++++---- source/libs/stream/src/streamRecover.c | 6 +++++- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 7314e23145..36c35ab415 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -821,13 +821,18 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { } // do recovery step 1 - streamSourceRecoverScanStep1(pTask); + tqDebug("s-task:%s start recover step 1 scan", pTask->id.idStr); + int64_t st = taosGetTimestampMs(); + streamSourceRecoverScanStep1(pTask); if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } + double el = (taosGetTimestampMs() - st) / 1000.0; + tqDebug("s-task:%s recover step 1 ended, elapsed time:%.2fs", pTask->id.idStr, el); + // build msg to launch next step SStreamRecoverStep2Req req; code = streamBuildSourceRecover2Req(pTask, &req); @@ -853,20 +858,17 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { memcpy(serializedReq, &req, len); // dispatch msg + tqDebug("s-task:%s start recover block stage", pTask->id.idStr); + SRpcMsg rpcMsg = { - .code = 0, - .contLen = len, - .msgType = TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, - .pCont = serializedReq, - }; - + .code = 0, .contLen = len, .msgType = TDMT_VND_STREAM_RECOVER_BLOCKING_STAGE, .pCont = serializedReq}; tmsgPutToQueue(&pTq->pVnode->msgCb, WRITE_QUEUE, &rpcMsg); - return 0; } int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - int32_t code; + int32_t code = 0; + SStreamRecoverStep2Req* pReq = (SStreamRecoverStep2Req*)msg; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->taskId); if (pTask == NULL) { diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 8cda5a21c9..22903b95d9 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -107,7 +107,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { if (streamTaskShouldStop(&pTask->status) || status == TASK_STATUS__RECOVER_PREPARE || status == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("s-task:%s skip push data, not ready for processing, status:%d", pTask->id.idStr, status); + tqDebug("s-task:%s not ready for new submit block from wal, status:%d", pTask->id.idStr, status); streamMetaReleaseTask(pStreamMeta, pTask); continue; } diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 325d315262..e711700ef2 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -165,20 +165,24 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { batchCnt++; - qDebug("task %d scan exec block num %d, block limit %d", pTask->id.taskId, batchCnt, batchSz); + qDebug("s-task:%s scan exec block num %d, block limit %d", pTask->id.idStr, batchCnt, batchSz); - if (batchCnt >= batchSz) break; + if (batchCnt >= batchSz) { + break; + } } + if (taosArrayGetSize(pRes) == 0) { if (finished) { taosArrayDestroy(pRes); - qDebug("task %d finish recover exec task ", pTask->id.taskId); + qDebug("s-task:%s finish recover exec task ", pTask->id.idStr); break; } else { - qDebug("task %d continue recover exec task ", pTask->id.taskId); + qDebug("s-task:%s continue recover exec task ", pTask->id.idStr); continue; } } + SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); if (qRes == NULL) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 55c745e417..0d1440fbde 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -20,6 +20,8 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { if (pTask->taskLevel == TASK_LEVEL__SOURCE) { atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE); + qDebug("s-task:%s set task status:%d and start recover", pTask->id.idStr, pTask->status.taskStatus); + streamSetParamForRecover(pTask); streamSourceRecoverPrepareStep1(pTask, version); @@ -197,7 +199,6 @@ int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* } int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { - // return streamScanExec(pTask, 100); } @@ -210,8 +211,11 @@ int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { void* exec = pTask->exec.pExecutor; + + qDebug("s-task:%s recover step2(blocking stage) started", pTask->id.idStr); if (qStreamSourceRecoverStep2(exec, ver) < 0) { } + return streamScanExec(pTask, 100); } From 03b1de017a041ff2588307cf90ff19b43d12b3a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 14:12:17 +0800 Subject: [PATCH 101/145] refactor: remove assert. --- include/libs/executor/executor.h | 2 -- source/common/src/tname.c | 2 -- source/libs/executor/src/executor.c | 20 +++----------------- source/util/src/tarray.c | 5 ++++- source/util/src/tcache.c | 8 -------- source/util/src/thash.c | 17 ----------------- source/util/src/tlosertree.c | 2 -- source/util/src/tsched.c | 8 -------- source/util/src/tskiplist.c | 6 +++--- 9 files changed, 10 insertions(+), 60 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 34372dc2ff..61eca6cc4f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -208,8 +208,6 @@ void* qExtractReaderFromStreamScanner(void* scanner); int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); -int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem); - int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo); int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver); int32_t qStreamSourceRecoverStep2(qTaskInfo_t tinfo, int64_t ver); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index e5ed7a3728..c6210ca8c9 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -122,10 +122,8 @@ int32_t tNameLen(const SName* name) { int32_t len2 = (int32_t)strlen(name->tname); if (name->type == TSDB_DB_NAME_T) { - ASSERT(len2 == 0); return len + len1 + TSDB_NAME_DELIMITER_LEN; } else { - ASSERT(len2 > 0); return len + len1 + len2 + TSDB_NAME_DELIMITER_LEN * 2; } } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7a5715e5ed..f4c62309a2 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -139,7 +139,6 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_SUBMIT) { - ASSERT(numOfBlocks == 1); taosArrayPush(pInfo->pBlockLists, input); pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; } else if (type == STREAM_INPUT__DATA_BLOCK) { @@ -854,15 +853,6 @@ int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { } } -#if 0 -int32_t qStreamInput(qTaskInfo_t tinfo, void* pItem) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); - taosWriteQitem(pTaskInfo->streamInfo.inputQueue->queue, pItem); - return 0; -} -#endif - int32_t qStreamSourceRecoverStep1(qTaskInfo_t tinfo, int64_t ver) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); @@ -897,8 +887,7 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0); - ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); qInfo("save stream param for interval: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); @@ -914,9 +903,8 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0); - ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); qInfo("save stream param for session: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); pInfo->twAggSup.calTriggerSaved = pInfo->twAggSup.calTrigger; @@ -929,8 +917,7 @@ int32_t qStreamSetParamForRecover(qTaskInfo_t tinfo) { SStreamStateAggOperatorInfo* pInfo = pOperator->info; ASSERT(pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE || pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE); - ASSERT(pInfo->twAggSup.calTriggerSaved == 0); - ASSERT(pInfo->twAggSup.deleteMarkSaved == 0); + ASSERT(pInfo->twAggSup.calTriggerSaved == 0 && pInfo->twAggSup.deleteMarkSaved == 0); qInfo("save stream param for state: %d, %" PRId64, pInfo->twAggSup.calTrigger, pInfo->twAggSup.deleteMark); @@ -991,7 +978,6 @@ int32_t qStreamRestoreParam(qTaskInfo_t tinfo) { if (pOperator->numOfDownstream != 1 || pOperator->pDownstream[0] == NULL) { if (pOperator->numOfDownstream > 1) { qError("unexpected stream, multiple downstream"); - /*ASSERT(0);*/ return -1; } return 0; diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 8e68e78acc..f8ce680531 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -255,7 +255,10 @@ void taosArraySet(SArray* pArray, size_t index, void* pData) { } void taosArrayPopFrontBatch(SArray* pArray, size_t cnt) { - ASSERT(cnt <= pArray->size); + if (cnt > pArray->size) { + cnt = pArray->size; + } + pArray->size = pArray->size - cnt; if (pArray->size == 0 || cnt == 0) { return; diff --git a/source/util/src/tcache.c b/source/util/src/tcache.c index 22eee547d0..28d9b412a0 100644 --- a/source/util/src/tcache.c +++ b/source/util/src/tcache.c @@ -264,7 +264,6 @@ static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) { static void removeNodeInEntryList(SCacheEntry *pe, SCacheNode *prev, SCacheNode *pNode) { if (prev == NULL) { - ASSERT(pe->next == pNode); pe->next = pNode->pNext; } else { prev->pNext = pNode->pNext; @@ -464,7 +463,6 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen SCacheNode *pNode = doSearchInEntryList(pe, key, keyLen, &prev); if (pNode != NULL) { int32_t ref = T_REF_INC(pNode); - ASSERT(ref > 0); } taosRUnLockLatch(&pe->latch); @@ -607,7 +605,6 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key, pNode->data, ref); if (ref > 0) { - ASSERT(pNode->pTNodeHeader == NULL); taosAddToTrashcan(pCacheObj, pNode); } else { // ref == 0 atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size); @@ -916,7 +913,6 @@ void taosStopCacheRefreshWorker(void) { size_t taosCacheGetNumOfObj(const SCacheObj *pCacheObj) { return pCacheObj->numOfElems + pCacheObj->numOfElemsInTrash; } SCacheIter *taosCacheCreateIter(const SCacheObj *pCacheObj) { - ASSERT(pCacheObj != NULL); SCacheIter *pIter = taosMemoryCalloc(1, sizeof(SCacheIter)); pIter->pCacheObj = (SCacheObj *)pCacheObj; pIter->entryIndex = -1; @@ -966,12 +962,8 @@ bool taosCacheIterNext(SCacheIter *pIter) { SCacheNode *pNode = pEntry->next; for (int32_t i = 0; i < pEntry->num; ++i) { - ASSERT(pNode != NULL); - pIter->pCurrent[i] = pNode; int32_t ref = T_REF_INC(pIter->pCurrent[i]); - ASSERT(ref >= 1); - pNode = pNode->pNext; } diff --git a/source/util/src/thash.c b/source/util/src/thash.c index b79824e910..cf4f17bfbc 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -259,8 +259,6 @@ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTyp pHashObj->freeFp = NULL; pHashObj->callbackFp = NULL; - ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0); - pHashObj->hashList = (SHashEntry **)taosMemoryMalloc(pHashObj->capacity * sizeof(void *)); if (pHashObj->hashList == NULL) { taosMemoryFree(pHashObj); @@ -343,7 +341,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, const vo while (pNode) { if ((pNode->keyLen == keyLen) && (*(pHashObj->equalFp))(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0 && pNode->removed == 0) { - ASSERT(pNode->hashVal == hashVal); break; } @@ -701,8 +698,6 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { pNode->next = pEntry->next; pEntry->next = pNode; - - ASSERT(pNode->next != pNode); pEntry->num += 1; } @@ -816,19 +811,7 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { /*uint16_t prevRef = atomic_load_16(&pNode->refCount);*/ uint16_t afterRef = atomic_add_fetch_16(&pNode->refCount, 1); -#if 0 - ASSERT(prevRef < afterRef); - // the reference count value is overflow, which will cause the delete node operation immediately. - if (prevRef > afterRef) { - uError("hash entry ref count overflow, prev ref:%d, current ref:%d", prevRef, afterRef); - // restore the value - atomic_sub_fetch_16(&pNode->refCount, 1); - data = NULL; - } else { - data = GET_HASH_NODE_DATA(pNode); - } -#endif data = GET_HASH_NODE_DATA(pNode); if (afterRef >= MAX_WARNING_REF_COUNT) { diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index bf99212b78..c476baa790 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -115,8 +115,6 @@ void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { } void tMergeTreeRebuild(SMultiwayMergeTreeInfo* pTree) { - ASSERT((pTree->totalSources & 0x1) == 0); - tMergeTreeInit(pTree); for (int32_t i = pTree->totalSources - 1; i >= pTree->numOfSources; i--) { tMergeTreeAdjust(pTree, i); diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 467f26b362..5c1706e405 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -137,7 +137,6 @@ void *taosProcessSchedQueue(void *scheduler) { while (1) { if ((ret = tsem_wait(&pSched->fullSem)) != 0) { uFatal("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } if (atomic_load_8(&pSched->stop)) { break; @@ -145,7 +144,6 @@ void *taosProcessSchedQueue(void *scheduler) { if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } msg = pSched->queue[pSched->fullSlot]; @@ -154,12 +152,10 @@ void *taosProcessSchedQueue(void *scheduler) { if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } if ((ret = tsem_post(&pSched->emptySem)) != 0) { uFatal("post %s emptySem failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } if (msg.fp) @@ -187,12 +183,10 @@ int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { if ((ret = tsem_wait(&pSched->emptySem)) != 0) { uFatal("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } if ((ret = taosThreadMutexLock(&pSched->queueMutex)) != 0) { uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } pSched->queue[pSched->emptySlot] = *pMsg; @@ -200,12 +194,10 @@ int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { if ((ret = taosThreadMutexUnlock(&pSched->queueMutex)) != 0) { uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } if ((ret = tsem_post(&pSched->fullSem)) != 0) { uFatal("post %s fullSem failed(%s)", pSched->label, strerror(errno)); - ASSERT(0); } return ret; } diff --git a/source/util/src/tskiplist.c b/source/util/src/tskiplist.c index c72c5c70ae..222e0e8a51 100644 --- a/source/util/src/tskiplist.c +++ b/source/util/src/tskiplist.c @@ -268,8 +268,9 @@ SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { } SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) { - ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - ASSERT(pSkipList != NULL); + if (order != TSDB_ORDER_ASC && order != TSDB_ORDER_DESC) { + return NULL; + } SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order); if (val == NULL) { @@ -585,7 +586,6 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { } } - ASSERT(level <= pSkipList->maxLevel); return level; } From 18d05ff69fc159e5979695c9477720232ef39fc4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 28 Apr 2023 15:53:56 +0800 Subject: [PATCH 102/145] fix:memory leak --- source/dnode/vnode/src/tq/tqPush.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 1ee19b7a7b..68240195d7 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -307,10 +307,14 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg) { STqHandle* pHandle = (STqHandle*) handle; if(pHandle->msg == NULL){ pHandle->msg = taosMemoryCalloc(1, sizeof(SRpcMsg)); + memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); + pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); + }else{ + void *tmp = pHandle->msg->pCont; + memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); + pHandle->msg->pCont = tmp; } - memcpy(pHandle->msg, pMsg, sizeof(SRpcMsg)); - pHandle->msg->pCont = rpcMallocCont(pMsg->contLen); memcpy(pHandle->msg->pCont, pMsg->pCont, pMsg->contLen); pHandle->msg->contLen = pMsg->contLen; int32_t ret = taosHashPut(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey), &pHandle, POINTER_BYTES); From a0d08b7fe4c522342c8541965b771d8156758452 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 16:29:09 +0800 Subject: [PATCH 103/145] fix(query): add null ptr check. --- source/libs/executor/src/executor.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index f4c62309a2..655952e378 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -800,7 +800,11 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) { return; } - qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows); + if (pTaskInfo->pRoot != NULL) { + qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows); + } else { + qDebug("%s execTask completed", GET_TASKID(pTaskInfo)); + } printTaskExecCostInLog(pTaskInfo); // print the query cost summary doDestroyTask(pTaskInfo); From 70761fd9590c125608b1a1ee6b1f143cca6acb90 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 28 Apr 2023 17:56:52 +0800 Subject: [PATCH 104/145] enh: optimize tbname condition --- source/libs/catalog/src/ctgCache.c | 2 +- source/libs/parser/src/parTranslater.c | 67 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index ec087c1168..433eacd30e 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1082,7 +1082,7 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); taosMemoryFree(op); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } msg->pCtg = pCtg; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index e14b750aba..fcdc6418a0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -30,6 +30,11 @@ #define SYSTABLE_SHOW_TYPE_OFFSET QUERY_NODE_SHOW_DNODES_STMT +typedef struct SRewriteTbNameContext { + int32_t errCode; + char* pTbName; +} SRewriteTbNameContext; + typedef struct STranslateContext { SParseContext* pParseCxt; int32_t errCode; @@ -2541,6 +2546,64 @@ static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSele return code; } +static EDealRes doTranslateTbName(SNode** pNode, void* pContext) { + switch (nodeType(*pNode)) { + case QUERY_NODE_FUNCTION: { + SFunctionNode *pFunc = (SFunctionNode *)*pNode; + if (FUNCTION_TYPE_TBNAME == pFunc->funcType) { + SRewriteTbNameContext *pCxt = (SRewriteTbNameContext*)pContext; + SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pVal) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + int32_t tbLen = strlen(pCxt->pTbName); + pVal->literal = taosStrdup(pCxt->pTbName); + if (NULL == pVal->literal) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + pVal->isDuration = false; + pVal->translate = true; + pVal->node.resType.type = TSDB_DATA_TYPE_BINARY; + pVal->node.resType.bytes = tbLen + VARSTR_HEADER_SIZE; + pVal->datum.p = taosMemoryCalloc(1, tbLen + VARSTR_HEADER_SIZE + 1); + varDataSetLen(pVal->datum.p, tbLen); + strncpy(varDataVal(pVal->datum.p), pVal->literal, tbLen); + strcpy(pVal->node.userAlias, pFunc->node.userAlias); + strcpy(pVal->node.aliasName, pFunc->node.aliasName); + + nodesDestroyNode(*pNode); + *pNode = (SNode*)pVal; + } + break; + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable)) { + return TSDB_CODE_SUCCESS; + } + + SRealTableNode* pTable = (SRealTableNode*)pSelect->pFromTable; + if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType && TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) { + return TSDB_CODE_SUCCESS; + } + + SNode** pNode = NULL; + SRewriteTbNameContext pRewriteCxt = {0}; + pRewriteCxt.pTbName = pTable->table.tableName; + + nodesRewriteExprPostOrder(&pSelect->pWhere, doTranslateTbName, &pRewriteCxt); + + return pRewriteCxt.errCode; +} + static int32_t checkJoinTable(STranslateContext* pCxt, SJoinTableNode* pJoinTable) { if ((QUERY_NODE_TEMP_TABLE == nodeType(pJoinTable->pLeft) && !isTimeLineQuery(((STempTableNode*)pJoinTable->pLeft)->pSubquery)) || @@ -3649,6 +3712,10 @@ static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect if (TSDB_CODE_SUCCESS == code) { code = setTableCacheLastMode(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = replaceTbName(pCxt, pSelect); + } + return code; } From 8d84e8f8a54e69e45848ed66c20703204d9350ab Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 28 Apr 2023 19:29:17 +0800 Subject: [PATCH 105/145] fix:[TD-23788] client wait if task status error in taosx transform data --- source/client/src/clientTmq.c | 8 ++++++-- source/dnode/vnode/src/tq/tqUtil.c | 6 ++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 0a31eac09c..b5ae9116ef 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1830,7 +1830,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; + if(pollRspWrapper->metaRsp.rspOffset.type != 0){ // if offset is validate + pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; + } atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); @@ -1848,7 +1850,9 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset; + if(pollRspWrapper->taosxRsp.rspOffset.type != 0){ // if offset is validate + pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset; + } atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->taosxRsp.blockNum == 0) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 1f06132d2f..1b5e498a7f 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -211,6 +211,12 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; tqInitTaosxRsp(&taosxRsp, pRequest); + qTaskInfo_t task = pHandle->execHandle.task; + if(qTaskIsExecuting(task)){ + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } if (offset->type != TMQ_OFFSET__LOG) { if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { From 3b66e63444f1f9076ed54b23078ba2ae50b44857 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 28 Apr 2023 20:07:54 +0800 Subject: [PATCH 106/145] fix:[TD-23788] client wait if task status error in taosx transform data --- source/dnode/vnode/src/tq/tqUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 1b5e498a7f..d186c63871 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -213,7 +213,7 @@ static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, tqInitTaosxRsp(&taosxRsp, pRequest); qTaskInfo_t task = pHandle->execHandle.task; if(qTaskIsExecuting(task)){ - code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); tDeleteSTaosxRsp(&taosxRsp); return code; } From 05d416f3b960a5dd293a42a6151a261217ce2b1a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 28 Apr 2023 23:53:53 +0800 Subject: [PATCH 107/145] fix(stream): fix invalid access when handling error, not start stream if tsDisablestream is set. --- source/dnode/vnode/src/vnd/vnodeSync.c | 7 ++++++- source/libs/executor/src/executor.c | 1 - source/libs/executor/src/querytask.c | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index dc2d709d76..1f958c569f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -555,7 +555,12 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) vInfo("vgId:%d, sync restore finished, start to restore stream tasks by replay wal", pVnode->config.vgId); // start to restore all stream tasks - tqStartStreamTasks(pVnode->pTq); + if (tsDisableStream) { + vInfo("vgId:%d, not restore stream tasks, since disabled", pVnode->config.vgId); + } else { + vInfo("vgId:%d start to restore stream tasks", pVnode->config.vgId); + tqStartStreamTasks(pVnode->pTq); + } } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 655952e378..2d991a14f5 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -312,7 +312,6 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v qTaskInfo_t pTaskInfo = NULL; code = qCreateExecTask(readers, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { - nodesDestroyNode((SNode*)pPlan); qDestroyTask(pTaskInfo); terrno = code; return NULL; diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index b6b250a325..a4d8327b6a 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -99,6 +99,7 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand if (NULL == (*pTaskInfo)->pRoot) { int32_t code = (*pTaskInfo)->code; doDestroyTask(*pTaskInfo); + (*pTaskInfo) = NULL; return code; } else { return TSDB_CODE_SUCCESS; @@ -206,11 +207,14 @@ static void freeBlock(void* pParam) { void doDestroyTask(SExecTaskInfo* pTaskInfo) { qDebug("%s execTask is freed", GET_TASKID(pTaskInfo)); destroyOperator(pTaskInfo->pRoot); + pTaskInfo->pRoot = NULL; + cleanupQueriedTableScanInfo(&pTaskInfo->schemaInfo); cleanupStreamInfo(&pTaskInfo->streamInfo); if (!pTaskInfo->localFetch.localExec) { nodesDestroyNode((SNode*)pTaskInfo->pSubplan); + pTaskInfo->pSubplan = NULL; } taosArrayDestroyEx(pTaskInfo->pResultBlockList, freeBlock); From d8e0d02e6ef82dad5244133e4d8bdf5ccae7a025 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 30 Apr 2023 16:32:51 +0800 Subject: [PATCH 108/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 117 ++++++++++++++----------- 1 file changed, 68 insertions(+), 49 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index eb15400d05..465220a0c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3395,6 +3395,11 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; while (1) { + if (pReader->flag == READER_STATUS_SHOULD_STOP) { + tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr); + return TSDB_CODE_SUCCESS; + } + STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; initMemDataIterator(*pBlockScanInfo, pReader); @@ -3474,45 +3479,67 @@ static bool fileBlockPartiallyRead(SFileBlockDumpInfo* pDumpInfo, bool asc) { ((pDumpInfo->rowIndex > 0 && asc) || (pDumpInfo->rowIndex < (pDumpInfo->totalRows - 1) && (!asc))); } +typedef enum { + TSDB_READ_RETURN = 0x1, + TSDB_READ_CONTINUE = 0x2, +} ERetrieveType; + +static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; + SDataBlockIter* pBlockIter = &pReader->status.blockIter; + + while(1) { + terrno = 0; + + code = doLoadLastBlockSequentially(pReader); + if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) { + terrno = code; + return TSDB_READ_RETURN; + } + + if (pResBlock->info.rows > 0) { + return TSDB_READ_RETURN; + } + + // all data blocks are checked in this last block file, now let's try the next file + ASSERT(pReader->status.pTableIter == NULL); + code = initForFirstBlockInFile(pReader, pBlockIter); + + // error happens or all the data files are completely checked + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false) || + pReader->flag == READER_STATUS_SHOULD_STOP) { + terrno = code; + return TSDB_READ_RETURN; + } + + if (pBlockIter->numOfBlocks > 0) { // there are data blocks existed. + return TSDB_READ_CONTINUE; + } else { // all blocks in data file are checked, let's check the data in last files + resetTableListIndex(&pReader->status); + } + } +} + static int32_t buildBlockFromFiles(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; bool asc = ASCENDING_TRAVERSE(pReader->order); SDataBlockIter* pBlockIter = &pReader->status.blockIter; + SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; if (pBlockIter->numOfBlocks == 0) { - _begin: - code = doLoadLastBlockSequentially(pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pReader->resBlockInfo.pResBlock->info.rows > 0) { - return TSDB_CODE_SUCCESS; - } - - // all data blocks are checked in this last block file, now let's try the next file - if (pReader->status.pTableIter == NULL) { - code = initForFirstBlockInFile(pReader, pBlockIter); - - // error happens or all the data files are completely checked - if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { - return code; - } - - // this file does not have data files, let's start check the last block file if exists - if (pBlockIter->numOfBlocks == 0) { - resetTableListIndex(&pReader->status); - goto _begin; - } + ERetrieveType type = doReadDataFromLastFiles(pReader); + if (type != TSDB_READ_RETURN) { + return terrno; } code = doBuildDataBlock(pReader); - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) { return code; } - if (pReader->resBlockInfo.pResBlock->info.rows > 0) { + if (pResBlock->info.rows > 0) { return TSDB_CODE_SUCCESS; } } @@ -3530,30 +3557,22 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); } else { - if (pReader->status.pCurrentFileset->nSttF > 0) { - // data blocks in current file are exhausted, let's try the next file now - SBlockData* pBlockData = &pReader->status.fileBlockData; - if (pBlockData->uid != 0) { - tBlockDataClear(pBlockData); - } + // all data blocks in files are checked, let's check the data in last files. + ASSERT(pReader->status.pCurrentFileset->nSttF > 0); - tBlockDataReset(pBlockData); - resetDataBlockIterator(pBlockIter, pReader->order); - resetTableListIndex(&pReader->status); - goto _begin; - } else { - code = initForFirstBlockInFile(pReader, pBlockIter); + // data blocks in current file are exhausted, let's try the next file now + SBlockData* pBlockData = &pReader->status.fileBlockData; + if (pBlockData->uid != 0) { + tBlockDataClear(pBlockData); + } - // error happens or all the data files are completely checked - if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { - return code; - } + tBlockDataReset(pBlockData); + resetDataBlockIterator(pBlockIter, pReader->order); + resetTableListIndex(&pReader->status); - // this file does not have blocks, let's start check the last block file - if (pBlockIter->numOfBlocks == 0) { - resetTableListIndex(&pReader->status); - goto _begin; - } + ERetrieveType type = doReadDataFromLastFiles(pReader); + if (type != TSDB_READ_RETURN) { + return terrno; } } } @@ -3561,11 +3580,11 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { code = doBuildDataBlock(pReader); } - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) { return code; } - if (pReader->resBlockInfo.pResBlock->info.rows > 0) { + if (pResBlock->info.rows > 0) { return TSDB_CODE_SUCCESS; } } From 9cc4721e2acec9f6215174a16823ec4af0fcd316 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 30 Apr 2023 16:37:09 +0800 Subject: [PATCH 109/145] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 465220a0c6..1ef86f5b30 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3529,8 +3529,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; if (pBlockIter->numOfBlocks == 0) { + // let's try to extract data from stt files. ERetrieveType type = doReadDataFromLastFiles(pReader); - if (type != TSDB_READ_RETURN) { + if (type == TSDB_READ_RETURN) { return terrno; } @@ -3571,7 +3572,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { resetTableListIndex(&pReader->status); ERetrieveType type = doReadDataFromLastFiles(pReader); - if (type != TSDB_READ_RETURN) { + if (type == TSDB_READ_RETURN) { return terrno; } } From 563fca5c54f1c0fb5fca22f993d3563efb3b612e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 May 2023 14:47:01 +0800 Subject: [PATCH 110/145] fix(tmq): remove unnecessary error log. --- source/dnode/vnode/src/tq/tqOffset.c | 103 +++++++++++++++------------ source/dnode/vnode/src/tq/tqPush.c | 15 ---- 2 files changed, 58 insertions(+), 60 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index e8051a1406..34e93cec2d 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -31,57 +31,67 @@ char* tqOffsetBuildFName(const char* path, int32_t fVer) { int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) { TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); - if (pFile != NULL) { - STqOffsetHead head = {0}; - int64_t code; + if (pFile == NULL) { + return TSDB_CODE_SUCCESS; + } - while (1) { - if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { - if (code == 0) { - break; - } else { - return -1; - } - } - int32_t size = htonl(head.size); - void* memBuf = taosMemoryCalloc(1, size); - if (memBuf == NULL) { + int32_t vgId = TD_VID(pStore->pTq->pVnode); + int64_t code = 0; + + STqOffsetHead head = {0}; + + while (1) { + if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + if (code == 0) { + break; + } else { return -1; } - if ((code = taosReadFile(pFile, memBuf, size)) != size) { - taosMemoryFree(memBuf); - return -1; - } - STqOffset offset; - SDecoder decoder; - tDecoderInit(&decoder, memBuf, size); - if (tDecodeSTqOffset(&decoder, &offset) < 0) { - taosMemoryFree(memBuf); - tDecoderClear(&decoder); - return -1; - } - - tDecoderClear(&decoder); - if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { - return -1; - } - - if (offset.val.type == TMQ_OFFSET__LOG) { - STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle) { - if (walRefVer(pHandle->pRef, offset.val.version) < 0) { - tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, - pHandle->subKey, offset.val.version); - } - } - } - - taosMemoryFree(memBuf); } - taosCloseFile(&pFile); + int32_t size = htonl(head.size); + void* pMemBuf = taosMemoryCalloc(1, size); + if (pMemBuf == NULL) { + tqError("vgId:%d failed to restore offset from file, since out of memory, malloc size:%d", vgId, size); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + if ((code = taosReadFile(pFile, pMemBuf, size)) != size) { + taosMemoryFree(pMemBuf); + return -1; + } + + STqOffset offset; + SDecoder decoder; + tDecoderInit(&decoder, pMemBuf, size); + if (tDecodeSTqOffset(&decoder, &offset) < 0) { + taosMemoryFree(pMemBuf); + tDecoderClear(&decoder); + return code; + } + + tDecoderClear(&decoder); + if (taosHashPut(pStore->pHash, offset.subKey, strlen(offset.subKey), &offset, sizeof(STqOffset)) < 0) { + return -1; + } + + // todo remove this + if (offset.val.type == TMQ_OFFSET__LOG) { + STqHandle* pHandle = taosHashGet(pStore->pTq->pHandle, offset.subKey, strlen(offset.subKey)); + if (pHandle) { + if (walRefVer(pHandle->pRef, offset.val.version) < 0) { +// tqError("vgId: %d, tq handle %s ref ver %" PRId64 "error", pStore->pTq->pVnode->config.vgId, pHandle->subKey, +// offset.val.version); + } + } + } + + taosMemoryFree(pMemBuf); } - return 0; + + taosCloseFile(&pFile); + return TSDB_CODE_SUCCESS; } STqOffsetStore* tqOffsetOpen(STQ* pTq) { @@ -89,6 +99,7 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { if (pStore == NULL) { return NULL; } + pStore->pTq = pTq; pStore->needCommit = 0; pTq->pOffsetStore = pStore; @@ -98,12 +109,14 @@ STqOffsetStore* tqOffsetOpen(STQ* pTq) { taosMemoryFree(pStore); return NULL; } + char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); if (tqOffsetRestoreFromFile(pStore, fname) < 0) { taosMemoryFree(fname); taosMemoryFree(pStore); return NULL; } + taosMemoryFree(fname); return pStore; } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 7a1a6b7454..ce222fcda7 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -335,21 +335,6 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v } if (msgType == TDMT_VND_SUBMIT) { -#if 0 - void* data = taosMemoryMalloc(len); - if (data == NULL) { - // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", vgId); - return -1; - } - - memcpy(data, pReq, len); - SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver}; - - tqDebug("vgId:%d tq copy submit msg:%p len:%d ver:%" PRId64 " from %p for stream", vgId, data, len, ver, pReq); - tqProcessSubmitReq(pTq, submit); -#endif SPackedData submit = {0}; tqProcessSubmitReq(pTq, submit); } From 6c86847b12940450397a1194b7310fe1182e3421 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 May 2023 14:58:59 +0800 Subject: [PATCH 111/145] refactor: do some internal refactor. --- source/libs/stream/src/streamRecover.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 0d1440fbde..0d214661c4 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -68,11 +68,12 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; pTask->checkReqId = req.reqId; - qDebug("task %d at node %d check downstream task %d at node %d", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, + qDebug("s-task:%s at node %d check downstream task %d at node %d", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t vgSz = taosArrayGetSize(vgInfo); pTask->recoverTryingDownstream = vgSz; pTask->checkReqIds = taosArrayInit(vgSz, sizeof(int64_t)); @@ -83,14 +84,15 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { taosArrayPush(pTask->checkReqIds, &req.reqId); req.downstreamNodeId = pVgInfo->vgId; req.downstreamTaskId = pVgInfo->taskId; - qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->id.taskId, pTask->nodeId, + qDebug("s-task:%s at node %d check downstream task %d at node %d (shuffle)", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { - qDebug("task %d at node %d direct launch recover since no downstream", pTask->id.taskId, pTask->nodeId); + qDebug("s-task:%s at node %d direct launch recover since no downstream", pTask->id.idStr, pTask->nodeId); streamTaskLaunchRecover(pTask, version); } + return 0; } From 3c61932eccc7ea3a968790a24f01ff2decbb0ce2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 May 2023 19:28:54 +0800 Subject: [PATCH 112/145] refactor: do some internal refactor. --- include/libs/stream/tstream.h | 6 +++--- source/dnode/vnode/src/tq/tq.c | 28 ++++++++++++------------- source/dnode/vnode/src/tq/tqRestore.c | 13 ++++++------ source/libs/stream/inc/streamInc.h | 2 +- source/libs/stream/src/streamDispatch.c | 4 ++-- source/libs/stream/src/streamExec.c | 2 +- source/libs/stream/src/streamMeta.c | 2 +- source/libs/stream/src/streamRecover.c | 27 +++++++++++++++++------- 8 files changed, 47 insertions(+), 37 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 78fd9bed5d..48c15e9117 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -50,7 +50,6 @@ enum { TASK_STATUS__RECOVER_PREPARE, TASK_STATUS__RECOVER1, TASK_STATUS__RECOVER2, - TASK_STATUS__RESTORE, // only available for source task to replay WAL from the checkpoint }; enum { @@ -346,7 +345,7 @@ typedef struct SStreamMeta { FTaskExpand* expandFunc; int32_t vgId; SRWLatch lock; - int32_t walScan; + int32_t walScanCounter; } SStreamMeta; int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); @@ -545,8 +544,9 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz); // recover and fill history int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version); int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version); -int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq); +int32_t streamTaskCheckStatus(SStreamTask* pTask); int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version); + // common int32_t streamSetParamForRecover(SStreamTask* pTask); int32_t streamRestoreParam(SStreamTask* pTask); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 36c35ab415..fed7a8cf74 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -111,8 +111,13 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); - tqInitialize(pTq); - return pTq; + int32_t code = tqInitialize(pTq); + if (code != TSDB_CODE_SUCCESS) { + tqClose(pTq); + return NULL; + } else { + return pTq; + } } int32_t tqInitialize(STQ* pTq) { @@ -601,11 +606,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->chkInfo.currentVer = ver; // expand executor - if (pTask->fillHistory) { - pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; - } else { - pTask->status.taskStatus = TASK_STATUS__RESTORE; - } + pTask->status.taskStatus = (pTask->fillHistory)? TASK_STATUS__WAIT_DOWNSTREAM:TASK_STATUS__NORMAL; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { pTask->pState = streamStateOpen(pTq->pStreamMeta->path, pTask, false, -1, -1); @@ -664,6 +665,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } streamSetupTrigger(pTask); + tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); @@ -693,8 +695,9 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { }; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + if (pTask) { - rsp.status = (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL) ? 1 : 0; + rsp.status = streamTaskCheckStatus(pTask); streamMetaReleaseTask(pTq->pStreamMeta, pTask); tqDebug("tq recv task check req(reqId:0x%" PRIx64 @@ -1147,9 +1150,6 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask != NULL) { if (pTask->status.taskStatus == TASK_STATUS__NORMAL) { - tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr); - streamProcessRunReq(pTask); - } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) { tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, pTask->id.idStr, pTask->chkInfo.version); streamProcessRunReq(pTask); @@ -1313,10 +1313,10 @@ int32_t tqStartStreamTasks(STQ* pTq) { return 0; } - pMeta->walScan += 1; + pMeta->walScanCounter += 1; - if (pMeta->walScan > 1) { - tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScan); + if (pMeta->walScanCounter > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScanCounter); taosWUnLockLatch(&pTq->pStreamMeta->lock); return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 22903b95d9..6c8fce5b14 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -18,15 +18,14 @@ static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle); // this function should be executed by stream threads. -// there is a case that the WAL increases more fast than the restore procedure, and this restore procedure -// will not stop eventually. +// extract submit block from WAL, and add them into the input queue for the sources tasks. int32_t tqStreamTasksScanWal(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; int64_t st = taosGetTimestampMs(); while (1) { - int32_t scan = pMeta->walScan; + int32_t scan = pMeta->walScanCounter; tqDebug("vgId:%d continue check if data in wal are available, scan:%d", vgId, scan); // check all restore tasks @@ -37,12 +36,12 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { if (shouldIdle) { taosWLockLatch(&pMeta->lock); - pMeta->walScan -= 1; - times = pMeta->walScan; + pMeta->walScanCounter -= 1; + times = pMeta->walScanCounter; - ASSERT(pMeta->walScan >= 0); + ASSERT(pMeta->walScanCounter >= 0); - if (pMeta->walScan <= 0) { + if (pMeta->walScanCounter <= 0) { taosWUnLockLatch(&pMeta->lock); break; } diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 876b80697a..b17afeec98 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -39,7 +39,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); -int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); +int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, SEpSet* pEpSet); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 549374ed94..d12eca7ce3 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -208,7 +208,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis return 0; } -int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { +int32_t streamDispatchCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet) { void* buf = NULL; int32_t code = -1; SRpcMsg msg = {0}; @@ -240,7 +240,7 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* msg.pCont = buf; msg.msgType = TDMT_STREAM_TASK_CHECK; - qDebug("dispatch from s-task:%s to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr, + qDebug("s-task:%s dispatch check msg to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr, pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index e711700ef2..f33e126068 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -28,7 +28,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* while (pTask->taskLevel == TASK_LEVEL__SOURCE) { int8_t status = atomic_load_8(&pTask->status.taskStatus); - if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__RESTORE) { + if (status != TASK_STATUS__NORMAL) { qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, atomic_load_8(&pTask->status.taskStatus)); taosMsleep(2); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 065e9d280f..822ae2a485 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -287,6 +287,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { tdbTbcClose(pCur); return -1; } + tDecoderInit(&decoder, (uint8_t*)pVal, vLen); tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); @@ -305,7 +306,6 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { return -1; } - /*pTask->status.taskStatus = TASK_STATUS__NORMAL;*/ if (pTask->fillHistory) { pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; streamTaskCheckDownstream(pTask, ver); diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 0d214661c4..67f3a95827 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -54,6 +54,8 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { // checkstatus int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { + qDebug("s-taks:%s in fill history stage, ver:%"PRId64, pTask->id.idStr, version); + SStreamTaskCheckReq req = { .streamId = pTask->id.streamId, .upstreamTaskId = pTask->id.taskId, @@ -63,6 +65,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { // serialize if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { + req.reqId = tGenIdPI64(); req.downstreamNodeId = pTask->fixedEpDispatcher.nodeId; req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; @@ -70,7 +73,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { qDebug("s-task:%s at node %d check downstream task %d at node %d", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); - streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); + streamDispatchCheckMsg(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; @@ -86,7 +89,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { req.downstreamTaskId = pVgInfo->taskId; qDebug("s-task:%s at node %d check downstream task %d at node %d (shuffle)", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); - streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); + streamDispatchCheckMsg(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { qDebug("s-task:%s at node %d direct launch recover since no downstream", pTask->id.idStr, pTask->nodeId); @@ -111,14 +114,14 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp req.downstreamTaskId, req.downstreamNodeId); if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); + streamDispatchCheckMsg(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; int32_t vgSz = taosArrayGetSize(vgInfo); for (int32_t i = 0; i < vgSz; i++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); if (pVgInfo->taskId == req.downstreamTaskId) { - streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pVgInfo->epSet); + streamDispatchCheckMsg(pTask, &req, pRsp->downstreamNodeId, &pVgInfo->epSet); } } } @@ -126,8 +129,8 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp return 0; } -int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) { - return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL; +int32_t streamTaskCheckStatus(SStreamTask* pTask) { + return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL? 1:0; } int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) { @@ -137,7 +140,9 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* if (pRsp->status == 1) { if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { bool found = false; - for (int32_t i = 0; i < taosArrayGetSize(pTask->checkReqIds); i++) { + + int32_t numOfReqs = taosArrayGetSize(pTask->checkReqIds); + for (int32_t i = 0; i < numOfReqs; i++) { int64_t reqId = *(int64_t*)taosArrayGet(pTask->checkReqIds, i); if (reqId == pRsp->reqId) { found = true; @@ -151,9 +156,12 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1); ASSERT(left >= 0); + if (left == 0) { taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; + + qDebug("s-task:%s all downstream tasks:%d are ready, now enter into recover stage", pTask->id.idStr, numOfReqs); streamTaskLaunchRecover(pTask, version); } } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { @@ -165,7 +173,10 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* } else { ASSERT(0); } - } else { // not ready, it should wait for at least 100ms and then retry + } else { // not ready, wait for 100ms and retry + qDebug("s-task:%s downstream taskId:%"PRId64" (vgId:%d) not ready, wait for 100ms and retry", pTask->id.idStr, + pRsp->downstreamTaskId, pRsp->downstreamNodeId); + taosMsleep(100); streamRecheckOneDownstream(pTask, pRsp); } From a1263b8b04ef72588a4adc605eb23ad1afbd9144 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 May 2023 19:40:26 +0800 Subject: [PATCH 113/145] fix(query):fix syntax error. --- source/libs/stream/src/streamRecover.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 67f3a95827..0324580885 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -174,7 +174,7 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* ASSERT(0); } } else { // not ready, wait for 100ms and retry - qDebug("s-task:%s downstream taskId:%"PRId64" (vgId:%d) not ready, wait for 100ms and retry", pTask->id.idStr, + qDebug("s-task:%s downstream taskId:%d (vgId:%d) not ready, wait for 100ms and retry", pTask->id.idStr, pRsp->downstreamTaskId, pRsp->downstreamNodeId); taosMsleep(100); streamRecheckOneDownstream(pTask, pRsp); From 3b814f24784b237f4f161124fbeb37e9677088a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 1 May 2023 23:21:29 +0800 Subject: [PATCH 114/145] fix(stream): do some internal refactor. --- source/dnode/vnode/src/tq/tqRestore.c | 6 +++--- source/libs/stream/src/stream.c | 9 +++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 6c8fce5b14..58cb7b9e63 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -20,13 +20,13 @@ static int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle); // this function should be executed by stream threads. // extract submit block from WAL, and add them into the input queue for the sources tasks. int32_t tqStreamTasksScanWal(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); + int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampMs(); while (1) { int32_t scan = pMeta->walScanCounter; - tqDebug("vgId:%d continue check if data in wal are available, scan:%d", vgId, scan); + tqDebug("vgId:%d continue check if data in wal are available, walScanCounter:%d", vgId, scan); // check all restore tasks bool shouldIdle = true; diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 046dab380e..9ed297bd6b 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -212,9 +212,10 @@ int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { } int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { - qDebug("vgId:%d s-task:%s receive dispatch req from taskId:%d", pReq->upstreamNodeId, pTask->id.idStr, - pReq->upstreamTaskId); + qDebug("s-task:%s receive dispatch req from taskId:%d(vgId:%d)", pTask->id.idStr, pReq->upstreamTaskId, + pReq->upstreamNodeId); + // todo add the input queue buffer limitation streamTaskEnqueueBlocks(pTask, pReq, pRsp); tDeleteStreamDispatchReq(pReq); @@ -222,10 +223,6 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S if (streamTryExec(pTask) < 0) { return -1; } - - /*if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {*/ - /*streamDispatch(pTask);*/ - /*}*/ } else { streamSchedExec(pTask); } From 79342cf1f46419e95f2252a14fca150a9551c4e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 May 2023 22:51:13 +0800 Subject: [PATCH 115/145] refactor(tqm): do some internal refactor. --- source/dnode/mnode/impl/inc/mndConsumer.h | 3 +- source/dnode/mnode/impl/inc/mndDef.h | 2 +- source/dnode/mnode/impl/src/mndConsumer.c | 51 ++++++++++------------- source/dnode/mnode/impl/src/mndDef.c | 2 +- 4 files changed, 24 insertions(+), 34 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index aa38b94fd7..96401511d2 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -23,13 +23,12 @@ extern "C" { #endif enum { - MQ_CONSUMER_STATUS__MODIFY = 1, + MQ_CONSUMER_STATUS_REBALANCE = 1, // MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__LOST, // MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore MQ_CONSUMER_STATUS__LOST_REBD, - MQ_CONSUMER_STATUS__REMOVED, }; int32_t mndInitConsumer(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index fcd314d2ae..2579ff5231 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -142,7 +142,7 @@ typedef enum { CONSUMER_UPDATE__REMOVE, CONSUMER_UPDATE__LOST, CONSUMER_UPDATE__RECOVER, - CONSUMER_UPDATE__MODIFY, // subscribe req need change consume topic + CONSUMER_UPDATE__REBALANCE, // subscribe req need change consume topic } ECsmUpdateType; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 65a2fa72a2..16ed158fed 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -192,6 +192,7 @@ FAIL: return -1; } +// todo check the clear process static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqConsumerClearMsg *pClearMsg = pMsg->pCont; @@ -302,11 +303,10 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { pLostMsg->consumerId = pConsumer->consumerId; SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_CONSUMER_LOST, - .pCont = pLostMsg, - .contLen = sizeof(SMqConsumerLostMsg), - }; + .msgType = TDMT_MND_TMQ_CONSUMER_LOST, .pCont = pLostMsg, .contLen = sizeof(SMqConsumerLostMsg)}; + mDebug("consumer:0x%"PRIx64" hb not received beyond threshold %d, set to lost", pConsumer->consumerId, + MND_CONSUMER_LOST_HB_CNT); tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } } else if (status == MQ_CONSUMER_STATUS__LOST_REBD) { @@ -316,11 +316,10 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { pClearMsg->consumerId = pConsumer->consumerId; SRpcMsg rpcMsg = { - .msgType = TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, - .pCont = pClearMsg, - .contLen = sizeof(SMqConsumerClearMsg), - }; + .msgType = TDMT_MND_TMQ_LOST_CONSUMER_CLEAR, .pCont = pClearMsg, .contLen = sizeof(SMqConsumerClearMsg)}; + mDebug("consumer:0x%" PRIx64 " lost beyond threshold %d, clear it", pConsumer->consumerId, + MND_CONSUMER_LOST_CLEAR_THRESHOLD); tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } } else if (status == MQ_CONSUMER_STATUS__LOST) { @@ -334,7 +333,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); - } else { + } else { // MQ_CONSUMER_STATUS_REBALANCE taosRLockLatch(&pConsumer->lock); int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); @@ -660,7 +659,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { tstrncpy(pConsumerNew->clientId, subscribe.clientId, tListLen(pConsumerNew->clientId)); // set the update type - pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; + pConsumerNew->updateType = CONSUMER_UPDATE__REBALANCE; taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); @@ -691,7 +690,7 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { } // set the update type - pConsumerNew->updateType = CONSUMER_UPDATE__MODIFY; + pConsumerNew->updateType = CONSUMER_UPDATE__REBALANCE; taosArrayDestroy(pConsumerNew->assignedTopics); pConsumerNew->assignedTopics = taosArrayDup(pTopicList, topicNameDup); @@ -870,9 +869,10 @@ static void updateConsumerStatus(SMqConsumerObj *pConsumer) { int32_t status = pConsumer->status; if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS__MODIFY) { + if (status == MQ_CONSUMER_STATUS_REBALANCE) { pConsumer->status = MQ_CONSUMER_STATUS__READY; } else if (status == MQ_CONSUMER_STATUS__LOST) { + ASSERT(taosArrayGetSize(pConsumer->currentTopics) == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0); pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; } } @@ -913,21 +913,13 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, taosWLockLatch(&pOldConsumer->lock); - if (pNewConsumer->updateType == CONSUMER_UPDATE__MODIFY) { - SArray *tmp = pOldConsumer->rebNewTopics; - pOldConsumer->rebNewTopics = pNewConsumer->rebNewTopics; - pNewConsumer->rebNewTopics = tmp; - - tmp = pOldConsumer->rebRemovedTopics; - pOldConsumer->rebRemovedTopics = pNewConsumer->rebRemovedTopics; - pNewConsumer->rebRemovedTopics = tmp; - - tmp = pOldConsumer->assignedTopics; - pOldConsumer->assignedTopics = pNewConsumer->assignedTopics; - pNewConsumer->assignedTopics = tmp; + if (pNewConsumer->updateType == CONSUMER_UPDATE__REBALANCE) { + TSWAP(pOldConsumer->rebNewTopics, pNewConsumer->rebNewTopics); + TSWAP(pOldConsumer->rebRemovedTopics, pNewConsumer->rebRemovedTopics); + TSWAP(pOldConsumer->assignedTopics, pNewConsumer->assignedTopics); pOldConsumer->subscribeTime = pNewConsumer->upTime; - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__LOST) { int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); for (int32_t i = 0; i < sz; i++) { @@ -937,10 +929,10 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, pOldConsumer->rebalanceTime = pNewConsumer->upTime; - int32_t status = pOldConsumer->status; + int32_t prevStatus = pOldConsumer->status; pOldConsumer->status = MQ_CONSUMER_STATUS__LOST; mDebug("consumer:0x%" PRIx64 " state %s -> %s, reb-time:%" PRId64 ", reb-removed-topics:%d", - pOldConsumer->consumerId, mndConsumerStatusName(status), mndConsumerStatusName(pOldConsumer->status), + pOldConsumer->consumerId, mndConsumerStatusName(prevStatus), mndConsumerStatusName(pOldConsumer->status), pOldConsumer->rebalanceTime, (int)taosArrayGetSize(pOldConsumer->rebRemovedTopics)); } else if (pNewConsumer->updateType == CONSUMER_UPDATE__RECOVER) { int32_t sz = taosArrayGetSize(pOldConsumer->assignedTopics); @@ -950,8 +942,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } pOldConsumer->rebalanceTime = pNewConsumer->upTime; - - pOldConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pOldConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; } else if (pNewConsumer->updateType == CONSUMER_UPDATE__TOUCH) { atomic_add_fetch_32(&pOldConsumer->epoch, 1); @@ -1160,7 +1151,7 @@ static const char *mndConsumerStatusName(int status) { case MQ_CONSUMER_STATUS__LOST: case MQ_CONSUMER_STATUS__LOST_REBD: return "lost"; - case MQ_CONSUMER_STATUS__MODIFY: + case MQ_CONSUMER_STATUS_REBALANCE: return "rebalancing"; default: return "unknown"; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index c69f08eb6b..6dab018236 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -225,7 +225,7 @@ SMqConsumerObj *tNewSMqConsumerObj(int64_t consumerId, char cgroup[TSDB_CGROUP_L memcpy(pConsumer->cgroup, cgroup, TSDB_CGROUP_LEN); pConsumer->epoch = 0; - pConsumer->status = MQ_CONSUMER_STATUS__MODIFY; + pConsumer->status = MQ_CONSUMER_STATUS_REBALANCE; pConsumer->hbStatus = 0; taosInitRWLatch(&pConsumer->lock); From 5b25920fec087cdda4245d8a2d6994fa756d6f2b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 2 May 2023 23:19:32 +0800 Subject: [PATCH 116/145] refactor: do some internal refactor. --- source/dnode/mnode/impl/src/mndConsumer.c | 65 +++++++++++++--------- source/dnode/mnode/impl/src/mndSubscribe.c | 31 ++++++----- 2 files changed, 57 insertions(+), 39 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 16ed158fed..ffb343ef22 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -672,7 +672,6 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { if (mndTransPrepare(pMnode, pTrans) != 0) goto _over; } else { - /*taosRLockLatch(&pExistedConsumer->lock);*/ int32_t status = atomic_load_32(&pExistedConsumer->status); mInfo("receive subscribe request from existed consumer:0x%" PRIx64 @@ -881,7 +880,7 @@ static void updateConsumerStatus(SMqConsumerObj *pConsumer) { // remove from new topic static void removeFromNewTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { int32_t size = taosArrayGetSize(pConsumer->rebNewTopics); - for (int32_t i = 0; i < taosArrayGetSize(pConsumer->rebNewTopics); i++) { + for (int32_t i = 0; i < size; i++) { char *p = taosArrayGetP(pConsumer->rebNewTopics, i); if (strcmp(pTopic, p) == 0) { taosArrayRemove(pConsumer->rebNewTopics, i); @@ -902,11 +901,44 @@ static void removeFromRemoveTopicList(SMqConsumerObj *pConsumer, const char *pTo if (strcmp(pTopic, p) == 0) { taosArrayRemove(pConsumer->rebRemovedTopics, i); taosMemoryFree(p); + + mDebug("consumer:0x%" PRIx64 " remove topic:%s in the removed topic list, remain removedTopics:%d", + pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->rebRemovedTopics)); break; } } } +static void removeFromCurrentTopicList(SMqConsumerObj *pConsumer, const char *pTopic) { + int32_t sz = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < sz; i++) { + char *topic = taosArrayGetP(pConsumer->currentTopics, i); + if (strcmp(pTopic, topic) == 0) { + taosArrayRemove(pConsumer->currentTopics, i); + taosMemoryFree(topic); + + mDebug("consumer:0x%" PRIx64 " remove topic:%s in the current topic list, remain currentTopics:%d", + pConsumer->consumerId, pTopic, (int)taosArrayGetSize(pConsumer->currentTopics)); + break; + } + } +} + +static bool existInCurrentTopicList(const SMqConsumerObj* pConsumer, const char* pTopic) { + bool existing = false; + int32_t size = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < size; i++) { + char *topic = taosArrayGetP(pConsumer->currentTopics, i); + + if (strcmp(topic, pTopic) == 0) { + existing = true; + break; + } + } + + return existing; +} + static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) { mDebug("consumer:0x%" PRIx64 " perform update action, update type:%d, subscribe-time:%" PRId64 ", uptime:%" PRId64, pOldConsumer->consumerId, pNewConsumer->updateType, pOldConsumer->subscribeTime, pOldConsumer->upTime); @@ -951,24 +983,16 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, } else if (pNewConsumer->updateType == CONSUMER_UPDATE__ADD) { char *pNewTopic = taosStrdup(taosArrayGetP(pNewConsumer->rebNewTopics, 0)); - // not exist in current topic - bool existing = false; - int32_t numOfExistedTopics = taosArrayGetSize(pOldConsumer->currentTopics); - for (int32_t i = 0; i < numOfExistedTopics; i++) { - char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); - if (strcmp(topic, pNewTopic) == 0) { - existing = true; - } - } - + // check if exist in current topic removeFromNewTopicList(pOldConsumer, pNewTopic); // add to current topic - if (!existing) { + bool existing = existInCurrentTopicList(pOldConsumer, pNewTopic); + if (existing) { + taosMemoryFree(pNewTopic); + } else { // added into current topic list taosArrayPush(pOldConsumer->currentTopics, &pNewTopic); taosArraySort(pOldConsumer->currentTopics, taosArrayCompareString); - } else { - taosMemoryFree(pNewTopic); } // set status @@ -993,16 +1017,7 @@ static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, removeFromRemoveTopicList(pOldConsumer, removedTopic); // remove from current topic - int32_t i = 0; - int32_t sz = taosArrayGetSize(pOldConsumer->currentTopics); - for (i = 0; i < sz; i++) { - char *topic = taosArrayGetP(pOldConsumer->currentTopics, i); - if (strcmp(removedTopic, topic) == 0) { - taosArrayRemove(pOldConsumer->currentTopics, i); - taosMemoryFree(topic); - break; - } - } + removeFromCurrentTopicList(pOldConsumer, removedTopic); // set status int32_t status = pOldConsumer->status; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 015c497de1..b3cf6c9701 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -213,13 +213,9 @@ static void doRemoveExistedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, int32_t consumerVgNum = taosArrayGetSize(pConsumerEp->vgs); for (int32_t j = 0; j < consumerVgNum; j++) { - SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); - SMqRebOutputVg outputVg = { - .oldConsumerId = consumerId, - .newConsumerId = -1, - .pVgEp = pVgEp, - }; + SMqVgEp *pVgEp = taosArrayGetP(pConsumerEp->vgs, j); + SMqRebOutputVg outputVg = {.oldConsumerId = consumerId, .newConsumerId = -1, .pVgEp = pVgEp}; taosHashPut(pHash, &pVgEp->vgId, sizeof(int32_t), &outputVg, sizeof(SMqRebOutputVg)); mInfo("sub:%s mq re-balance remove vgId:%d from consumer:%" PRIx64, pSubKey, pVgEp->vgId, consumerId); } @@ -584,16 +580,11 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // here we only handle one topic rebalance requirement to ensure the atomic execution of this transaction. while (1) { -// if (rebalanceOnce) { -// break; -// } - pIter = taosHashIterate(pReq->rebSubHash, pIter); if (pIter == NULL) { break; } - // todo handle the malloc failure SMqRebInputObj rebInput = {0}; SMqRebOutputObj rebOutput = {0}; rebOutput.newConsumers = taosArrayInit(0, sizeof(int64_t)); @@ -601,6 +592,20 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { rebOutput.modifyConsumers = taosArrayInit(0, sizeof(int64_t)); rebOutput.rebVgs = taosArrayInit(0, sizeof(SMqRebOutputVg)); + if (rebOutput.newConsumers == NULL || rebOutput.removedConsumers == NULL || rebOutput.modifyConsumers == NULL || + rebOutput.rebVgs == NULL) { + taosArrayDestroy(rebOutput.newConsumers); + taosArrayDestroy(rebOutput.removedConsumers); + taosArrayDestroy(rebOutput.modifyConsumers); + taosArrayDestroy(rebOutput.rebVgs); + + terrno = TSDB_CODE_OUT_OF_MEMORY; + mInfo("mq re-balance failed, due to out of memory"); + taosHashCleanup(pReq->rebSubHash); + mndRebEnd(); + return -1; + } + SMqRebInfo *pRebInfo = (SMqRebInfo *)pIter; SMqSubscribeObj *pSub = mndAcquireSubscribeByKey(pMnode, pRebInfo->key); @@ -640,6 +645,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { rebInput.oldConsumerNum = taosHashGetSize(pSub->consumerHash); rebOutput.pSub = tCloneSubscribeObj(pSub); taosRUnLockLatch(&pSub->lock); + mInfo("sub topic:%s has %d consumers sub till now", pRebInfo->key, rebInput.oldConsumerNum); mndReleaseSubscribe(pMnode, pSub); } @@ -661,9 +667,6 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { taosArrayDestroy(rebOutput.rebVgs); tDeleteSubscribeObj(rebOutput.pSub); taosMemoryFree(rebOutput.pSub); - -// taosSsleep(100); -// rebalanceOnce = true; } // reset flag From a53c181b98f140e2d9e2d4ece604b31694896289 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 May 2023 10:04:44 +0800 Subject: [PATCH 117/145] fix(tmq): remove invalid assert --- source/dnode/mnode/impl/src/mndConsumer.c | 20 ++++++++++++--- source/dnode/mnode/impl/src/mndSubscribe.c | 29 ++++++++++++++-------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index ffb343ef22..0fc9aae59a 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -196,12 +196,14 @@ FAIL: static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { SMnode *pMnode = pMsg->info.node; SMqConsumerClearMsg *pClearMsg = pMsg->pCont; - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId); + + SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, pClearMsg->consumerId); if (pConsumer == NULL) { + mError("consumer:0x%"PRIx64" failed to be found to clear it", pClearMsg->consumerId); return 0; } - mInfo("receive consumer clear msg, consumer id %" PRId64 ", status %s", pClearMsg->consumerId, + mInfo("consumer:0x%" PRIx64 " needs to be cleared, status %s", pClearMsg->consumerId, mndConsumerStatusName(pConsumer->status)); if (pConsumer->status != MQ_CONSUMER_STATUS__LOST_REBD) { @@ -216,6 +218,8 @@ static int32_t mndProcessConsumerClearMsg(SRpcMsg *pMsg) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pMsg, "clear-csm"); if (pTrans == NULL) goto FAIL; + + // this is the drop action, not the update action if (mndSetConsumerDropLogs(pMnode, pTrans, pConsumerNew) != 0) goto FAIL; if (mndTransPrepare(pMnode, pTrans) != 0) goto FAIL; @@ -300,6 +304,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { if (status == MQ_CONSUMER_STATUS__READY) { if (hbStatus > MND_CONSUMER_LOST_HB_CNT) { SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); + if (pLostMsg == NULL) { + mError("consumer:0x%"PRIx64" failed to transfer consumer status to lost due to out of memory. alloc size:%d", + pConsumer->consumerId, sizeof(SMqConsumerLostMsg)); + continue; + } pLostMsg->consumerId = pConsumer->consumerId; SRpcMsg rpcMsg = { @@ -313,6 +322,11 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { // if the client is lost longer than one day, clear it. Otherwise, do nothing about the lost consumers. if (hbStatus > MND_CONSUMER_LOST_CLEAR_THRESHOLD) { SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg)); + if (pClearMsg == NULL) { + mError("consumer:0x%"PRIx64" failed to clear consumer due to out of memory. alloc size:%d", + pConsumer->consumerId, sizeof(SMqConsumerClearMsg)); + continue; + } pClearMsg->consumerId = pConsumer->consumerId; SRpcMsg rpcMsg = { @@ -871,7 +885,7 @@ static void updateConsumerStatus(SMqConsumerObj *pConsumer) { if (status == MQ_CONSUMER_STATUS_REBALANCE) { pConsumer->status = MQ_CONSUMER_STATUS__READY; } else if (status == MQ_CONSUMER_STATUS__LOST) { - ASSERT(taosArrayGetSize(pConsumer->currentTopics) == 0 && taosArrayGetSize(pConsumer->assignedTopics) == 0); + ASSERT(taosArrayGetSize(pConsumer->currentTopics) == 0); pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; } } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index b3cf6c9701..573c60549e 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -480,14 +480,16 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu for (int32_t i = 0; i < vgNum; i++) { SMqRebOutputVg *pRebVg = taosArrayGet(rebVgs, i); if (mndPersistSubChangeVgReq(pMnode, pTrans, pOutput->pSub, pRebVg) < 0) { - goto REB_FAIL; + mndTransDrop(pTrans); + return -1; } } // 2. redo log: subscribe and vg assignment // subscribe if (mndSetSubCommitLogs(pMnode, pTrans, pOutput->pSub) != 0) { - goto REB_FAIL; + mndTransDrop(pTrans); + return -1; } // 3. commit log: consumer to update status and epoch @@ -502,11 +504,15 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); - goto REB_FAIL; + + mndTransDrop(pTrans); + return -1; } + tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); } + // 3.2 set new consumer consumerNum = taosArrayGetSize(pOutput->newConsumers); for (int32_t i = 0; i < consumerNum; i++) { @@ -523,8 +529,11 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); - goto REB_FAIL; + + mndTransDrop(pTrans); + return -1; } + tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); } @@ -545,8 +554,11 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu if (mndSetConsumerCommitLogs(pMnode, pTrans, pConsumerNew) != 0) { tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); - goto REB_FAIL; + + mndTransDrop(pTrans); + return -1; } + tDeleteSMqConsumerObj(pConsumerNew); taosMemoryFree(pConsumerNew); } @@ -559,15 +571,12 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu // 6. execution if (mndTransPrepare(pMnode, pTrans) != 0) { mError("failed to prepare trans rebalance since %s", terrstr()); - goto REB_FAIL; + mndTransDrop(pTrans); + return -1; } mndTransDrop(pTrans); return 0; - -REB_FAIL: - mndTransDrop(pTrans); - return -1; } static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { From 0e1c261418b6eaffc61fc6c52584cb195773fe5b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 May 2023 10:10:56 +0800 Subject: [PATCH 118/145] fix(tmq): fix syntax error. --- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 0fc9aae59a..33081cd8cc 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -324,7 +324,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqConsumerClearMsg *pClearMsg = rpcMallocCont(sizeof(SMqConsumerClearMsg)); if (pClearMsg == NULL) { mError("consumer:0x%"PRIx64" failed to clear consumer due to out of memory. alloc size:%d", - pConsumer->consumerId, sizeof(SMqConsumerClearMsg)); + pConsumer->consumerId, (int32_t)sizeof(SMqConsumerClearMsg)); continue; } From 66b279abb71d3c843f7b8e2170ad6729b80d7519 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 May 2023 10:12:10 +0800 Subject: [PATCH 119/145] fix(query):fix syntax error. --- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 33081cd8cc..cd7c8beaa7 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -306,7 +306,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { SMqConsumerLostMsg *pLostMsg = rpcMallocCont(sizeof(SMqConsumerLostMsg)); if (pLostMsg == NULL) { mError("consumer:0x%"PRIx64" failed to transfer consumer status to lost due to out of memory. alloc size:%d", - pConsumer->consumerId, sizeof(SMqConsumerLostMsg)); + pConsumer->consumerId, (int32_t)sizeof(SMqConsumerLostMsg)); continue; } From b1f3a010df982b2656929cab2394f83e1db2bfe6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 4 May 2023 13:58:45 +0800 Subject: [PATCH 120/145] fix:change field bytes if length is bigger than 1024 --- source/client/src/clientSml.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 57458ff8f7..d6642dfe8d 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -558,10 +558,15 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm return 0; } +#define BOUNDARY 1024 static int32_t smlFindNearestPowerOf2(int32_t length, uint8_t type) { int32_t result = 1; - while (result <= length) { - result *= 2; + if (length >= BOUNDARY){ + result = length; + }else{ + while (result <= length) { + result *= 2; + } } if (type == TSDB_DATA_TYPE_BINARY && result > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { result = TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE; @@ -657,7 +662,7 @@ static int32_t smlBuildFieldsList(SSmlHandle *info, SSchema *schemaField, SHashO len += field->bytes; } if(len > maxLen){ - return TSDB_CODE_TSC_INVALID_VALUE; + return isTag ? TSDB_CODE_PAR_INVALID_TAGS_LENGTH : TSDB_CODE_PAR_INVALID_ROW_LENGTH; } return TSDB_CODE_SUCCESS; From 09a04052ecf8920d2f4c22e67960e806ec53a9e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 May 2023 16:15:14 +0800 Subject: [PATCH 121/145] refactor: do some internal refactor. --- include/common/tcommon.h | 8 - include/common/tmsg.h | 10 +- include/libs/executor/executor.h | 2 - include/libs/wal/wal.h | 2 - source/client/src/clientMain.c | 4 +- source/client/src/clientStmt.c | 4 +- source/client/src/clientTmq.c | 4 +- source/client/test/clientTests.cpp | 10 +- source/common/src/tdatablock.c | 4 +- source/common/src/tdataformat.c | 4 +- source/common/src/tmsg.c | 14 +- source/dnode/vnode/inc/vnode.h | 5 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 1 - source/dnode/vnode/src/meta/metaSnapshot.c | 4 +- source/dnode/vnode/src/meta/metaTable.c | 4 +- source/dnode/vnode/src/sma/smaRollup.c | 4 +- source/dnode/vnode/src/sma/smaTimeRange.c | 8 +- source/dnode/vnode/src/tq/tqPush.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 391 ++++++++++++--------- source/dnode/vnode/src/tq/tqScan.c | 4 +- source/dnode/vnode/src/tq/tqSink.c | 14 +- source/dnode/vnode/src/tq/tqUtil.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 12 +- source/libs/executor/src/dataInserter.c | 12 +- source/libs/executor/src/executor.c | 16 +- source/libs/executor/src/querytask.c | 6 +- source/libs/executor/src/scanoperator.c | 18 +- source/libs/parser/src/parInsertUtil.c | 8 +- source/libs/stream/src/streamTask.c | 2 +- 30 files changed, 305 insertions(+), 276 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 8b4d5f16df..2bc67e439f 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -212,14 +212,6 @@ enum { FETCH_TYPE__NONE, }; -typedef struct { - int8_t fetchType; - union { - SSDataBlock data; - void* meta; - }; -} SFetchRet; - typedef struct SVarColAttr { int32_t* offset; // start position for each entry in the list uint32_t length; // used buffer size that contain the valid data diff --git a/include/common/tmsg.h b/include/common/tmsg.h index dc997221e8..1ea9714bf9 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -415,7 +415,7 @@ static FORCE_INLINE SSchemaWrapper* tCloneSSchemaWrapper(const SSchemaWrapper* p return pSW; } -static FORCE_INLINE void tDeleteSSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { +static FORCE_INLINE void tDeleteSchemaWrapper(SSchemaWrapper* pSchemaWrapper) { if (pSchemaWrapper) { taosMemoryFree(pSchemaWrapper->pSchema); taosMemoryFree(pSchemaWrapper); @@ -3421,10 +3421,10 @@ typedef struct { char data[]; // SSubmitReq2 } SSubmitReq2Msg; -int32_t tEncodeSSubmitReq2(SEncoder* pCoder, const SSubmitReq2* pReq); -int32_t tDecodeSSubmitReq2(SDecoder* pCoder, SSubmitReq2* pReq); -void tDestroySSubmitTbData(SSubmitTbData* pTbData, int32_t flag); -void tDestroySSubmitReq(SSubmitReq2* pReq, int32_t flag); +int32_t tEncodeSubmitReq(SEncoder* pCoder, const SSubmitReq2* pReq); +int32_t tDecodeSubmitReq(SDecoder* pCoder, SSubmitReq2* pReq); +void tDestroySubmitTbData(SSubmitTbData* pTbData, int32_t flag); +void tDestroySubmitReq(SSubmitReq2* pReq, int32_t flag); typedef struct { int32_t affectedRows; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 61eca6cc4f..1fb00e743f 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -192,8 +192,6 @@ SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); - void qStreamSetOpen(qTaskInfo_t tinfo); void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index b51289de5e..d3e2bbb1be 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -147,8 +147,6 @@ typedef struct SWalReader { int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; -// int8_t curInvalid; -// int8_t curStopped; TdThreadMutex mutex; SWalFilterCond cond; // TODO remove it diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 2ebc8e7379..fd70598efb 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -191,7 +191,7 @@ void taos_free_result(TAOS_RES *res) { taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->rsp.blockDataLen); taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); - taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSchemaWrapper); // taosx taosArrayDestroy(pRsp->rsp.createTableLen); taosArrayDestroyP(pRsp->rsp.createTableReq, taosMemoryFree); @@ -204,7 +204,7 @@ void taos_free_result(TAOS_RES *res) { taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->rsp.blockDataLen); taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); - taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); taosMemoryFree(pRsp); diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 6e529f1a0b..975b304bf4 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -325,7 +325,7 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { taosHashCleanup(pStmt->exec.pBlockHash); pStmt->exec.pBlockHash = NULL; - tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); taosMemoryFreeClear(pStmt->exec.pCurrTbData); STMT_ERR_RET(stmtCleanBindInfo(pStmt)); @@ -895,7 +895,7 @@ int stmtExec(TAOS_STMT* stmt) { if (STMT_TYPE_QUERY == pStmt->sql.type) { launchQueryImpl(pStmt->exec.pRequest, pStmt->sql.pQuery, true, NULL); } else { - tDestroySSubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(pStmt->exec.pCurrTbData, TSDB_MSG_FLG_ENCODE); taosMemoryFreeClear(pStmt->exec.pCurrTbData); STMT_ERR_RET(qCloneCurrentTbData(pStmt->exec.pCurrBlock, &pStmt->exec.pCurrTbData)); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index b5ae9116ef..b488af9ba1 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -864,7 +864,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { taosArrayDestroyP(pRsp->dataRsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->dataRsp.blockDataLen); taosArrayDestroyP(pRsp->dataRsp.blockTbName, taosMemoryFree); - taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->dataRsp.blockSchema, (FDelete)tDeleteSchemaWrapper); } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { SMqPollRspWrapper* pRsp = (SMqPollRspWrapper*)rspWrapper; taosMemoryFreeClear(pRsp->pEpset); @@ -877,7 +877,7 @@ static void* tmqFreeRspWrapper(SMqRspWrapper* rspWrapper) { taosArrayDestroyP(pRsp->taosxRsp.blockData, taosMemoryFree); taosArrayDestroy(pRsp->taosxRsp.blockDataLen); taosArrayDestroyP(pRsp->taosxRsp.blockTbName, taosMemoryFree); - taosArrayDestroyP(pRsp->taosxRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->taosxRsp.blockSchema, (FDelete)tDeleteSchemaWrapper); // taosx taosArrayDestroy(pRsp->taosxRsp.createTableLen); taosArrayDestroyP(pRsp->taosxRsp.createTableReq, taosMemoryFree); diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b9062fc8ff..56f68e5972 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -1053,9 +1053,9 @@ TEST(clientCase, sub_db_test) { } TEST(clientCase, sub_tb_test) { - taos_options(TSDB_OPTION_CONFIGDIR, "/home/tests/dir/cfg/"); + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); - TAOS* pConn = taos_connect("vm116", "root", "taosdata", NULL, 0); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); tmq_conf_t* conf = tmq_conf_new(); @@ -1091,7 +1091,7 @@ TEST(clientCase, sub_tb_test) { int32_t precision = 0; int32_t totalRows = 0; int32_t msgCnt = 0; - int32_t timeout = 25000; + int32_t timeout = 2500000; int32_t count = 0; @@ -1117,10 +1117,10 @@ TEST(clientCase, sub_tb_test) { fields = taos_fetch_fields(pRes); numOfFields = taos_field_count(pRes); totalRows += 1; - if (totalRows % 100000 == 0) { +// if (totalRows % 100000 == 0) { taos_print_row(buf, row, fields, numOfFields); printf("row content: %s\n", buf); - } +// } } taos_free_result(pRes); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3558feaa66..fc7cbc19c0 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -2374,7 +2374,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** ppReq, const SSDataBlock* pDat } SRow* pRow = NULL; if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { - tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } ASSERT(pRow); @@ -2388,7 +2388,7 @@ _end: if (terrno != 0) { *ppReq = NULL; if (pReq) { - tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFreeClear(pReq); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index f379084cf5..b18bd882ae 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1509,7 +1509,9 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { // STSchema ======================================== STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version) { STSchema *pTSchema = taosMemoryCalloc(1, sizeof(STSchema) + sizeof(STColumn) * numOfCols); - if (pTSchema == NULL) return NULL; + if (pTSchema == NULL) { + return NULL; + } pTSchema->numOfCols = numOfCols; pTSchema->version = version; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8b14b7fbe7..7102e556cc 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -7058,7 +7058,7 @@ void tDeleteSMqDataRsp(SMqDataRsp *pRsp) { pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; - taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); pRsp->blockSchema = NULL; taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); pRsp->blockTbName = NULL; @@ -7159,7 +7159,7 @@ void tDeleteSTaosxRsp(STaosxRsp *pRsp) { pRsp->blockDataLen = NULL; taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; - taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSchemaWrapper); pRsp->blockSchema = NULL; taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); pRsp->blockTbName = NULL; @@ -7332,7 +7332,7 @@ _exit: return 0; } -int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) { +int32_t tEncodeSubmitReq(SEncoder *pCoder, const SSubmitReq2 *pReq) { if (tStartEncode(pCoder) < 0) return -1; if (tEncodeU64v(pCoder, taosArrayGetSize(pReq->aSubmitTbData)) < 0) return -1; @@ -7344,7 +7344,7 @@ int32_t tEncodeSSubmitReq2(SEncoder *pCoder, const SSubmitReq2 *pReq) { return 0; } -int32_t tDecodeSSubmitReq2(SDecoder *pCoder, SSubmitReq2 *pReq) { +int32_t tDecodeSubmitReq(SDecoder *pCoder, SSubmitReq2 *pReq) { int32_t code = 0; memset(pReq, 0, sizeof(*pReq)); @@ -7387,7 +7387,7 @@ _exit: return code; } -void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { +void tDestroySubmitTbData(SSubmitTbData *pTbData, int32_t flag) { if (NULL == pTbData) { return; } @@ -7433,14 +7433,14 @@ void tDestroySSubmitTbData(SSubmitTbData *pTbData, int32_t flag) { } } -void tDestroySSubmitReq(SSubmitReq2 *pReq, int32_t flag) { +void tDestroySubmitReq(SSubmitReq2 *pReq, int32_t flag) { if (pReq->aSubmitTbData == NULL) return; int32_t nSubmitTbData = TARRAY_SIZE(pReq->aSubmitTbData); SSubmitTbData *aSubmitTbData = (SSubmitTbData *)TARRAY_DATA(pReq->aSubmitTbData); for (int32_t i = 0; i < nSubmitTbData; i++) { - tDestroySSubmitTbData(&aSubmitTbData[i], flag); + tDestroySubmitTbData(&aSubmitTbData[i], flag); } taosArrayDestroy(pReq->aSubmitTbData); pReq->aSubmitTbData = NULL; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 002dcda488..817e6fdae4 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -231,7 +231,7 @@ typedef struct SSnapContext { } SSnapContext; typedef struct STqReader { - SPackedData msg2; + SPackedData msg; SSubmitReq2 submit; int32_t nextBlk; int64_t lastBlkUid; @@ -242,7 +242,7 @@ typedef struct STqReader { int32_t cachedSchemaVer; int64_t cachedSchemaSuid; SSchemaWrapper *pSchemaWrapper; - STSchema *pSchema; + SSDataBlock *pResBlock; } STqReader; STqReader *tqReaderOpen(SVnode *pVnode); @@ -255,6 +255,7 @@ int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); int32_t tqNextBlock(STqReader *pReader, SSDataBlock* pBlock); +int32_t tqNextBlockInWal(STqReader* pReader); int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData); int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7668d45108..eb2787595b 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -193,7 +193,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode); void tqNotifyClose(STQ*); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); -int tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg); +int tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg); int tqUnregisterPushHandle(STQ* pTq, void* pHandle); int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 2359a165b7..d464f64de3 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -639,7 +639,6 @@ tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, int lock) { STSchema *pTSchema = NULL; SSchemaWrapper *pSW = NULL; - SSchema *pSchema = NULL; pSW = metaGetTableSchema(pMeta, uid, sver, lock); if (!pSW) return NULL; diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 56e802d4fb..707dd66e30 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -217,8 +217,8 @@ typedef struct STableInfoForChildTable { static void destroySTableInfoForChildTable(void* data) { STableInfoForChildTable* pData = (STableInfoForChildTable*)data; taosMemoryFree(pData->tableName); - tDeleteSSchemaWrapper(pData->schemaRow); - tDeleteSSchemaWrapper(pData->tagRow); + tDeleteSchemaWrapper(pData->schemaRow); + tDeleteSchemaWrapper(pData->tagRow); } static void MoveToSnapShotVersion(SSnapContext* ctx) { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 96eec89127..83f2ece571 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -673,8 +673,8 @@ int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) metaUpdateUidIdx(pMeta, &nStbEntry); metaULock(pMeta); - tDeleteSSchemaWrapper(tag); - tDeleteSSchemaWrapper(row); + tDeleteSchemaWrapper(tag); + tDeleteSchemaWrapper(row); if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); tDecoderClear(&dc); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 69b3f9c3e0..20e04f122b 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -684,7 +684,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8 " failed since %s", @@ -696,7 +696,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); if (pReq) { - tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 2a26f65bf9..6a4bddc991 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -299,7 +299,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } SRow *pRow = NULL; if ((terrno = tRowBuild(pVals, (STSchema *)pTSchema, &pRow)) < 0) { - tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } taosArrayPush(tbData.aRowP, &pRow); @@ -309,7 +309,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * } // encode - tEncodeSize(tEncodeSSubmitReq2, pReq, len, terrno); + tEncodeSize(tEncodeSubmitReq, pReq, len, terrno); if (TSDB_CODE_SUCCESS == terrno) { SEncoder encoder; len += sizeof(SSubmitReq2Msg); @@ -321,7 +321,7 @@ int32_t smaBlockToSubmit(SVnode *pVnode, const SArray *pBlocks, const STSchema * ((SSubmitReq2Msg *)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg *)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); - if (tEncodeSSubmitReq2(&encoder, pReq) < 0) { + if (tEncodeSubmitReq(&encoder, pReq) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; /*vError("failed to encode submit req since %s", terrstr());*/ } @@ -332,7 +332,7 @@ _end: taosArrayDestroy(tagArray); taosArrayDestroy(pVals); if (pReq) { - tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 0575b7299d..a914517645 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -287,7 +287,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v } -int32_t tqRegisterPushEntry(STQ* pTq, void* handle, SRpcMsg* pMsg) { +int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) { int32_t vgId = TD_VID(pTq->pVnode); STqHandle* pHandle = (STqHandle*) handle; if(pHandle->msg == NULL){ diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index ead00dcc35..0c9c7b4793 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -265,9 +265,9 @@ STqReader* tqReaderOpen(SVnode* pVnode) { pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; pReader->cachedSchemaSuid = 0; - pReader->pSchema = NULL; pReader->pSchemaWrapper = NULL; pReader->tbIdHash = NULL; + pReader->pResBlock = createDataBlock(); return pReader; } @@ -276,19 +276,19 @@ void tqCloseReader(STqReader* pReader) { if (pReader->pWalReader) { walCloseReader(pReader->pWalReader); } - // free cached schema - if (pReader->pSchema) { - taosMemoryFree(pReader->pSchema); - } + if (pReader->pSchemaWrapper) { - tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + tDeleteSchemaWrapper(pReader->pSchemaWrapper); } + if (pReader->pColIdList) { taosArrayDestroy(pReader->pColIdList); } + // free hash + blockDataDestroy(pReader->pResBlock); taosHashCleanup(pReader->tbIdHash); - tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); taosMemoryFree(pReader); } @@ -322,9 +322,71 @@ int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) { return 0; } +// todo ignore the error in wal? +int32_t tqNextBlockInWal(STqReader* pReader) { + SWalReader* pWalReader = pReader->pWalReader; + + while(1) { + SArray* pBlockList = pReader->submit.aSubmitTbData; + if (pBlockList == NULL || pReader->nextBlk >= taosArrayGetSize(pBlockList)) { + + // try next message in wal file + if (walNextValidMsg(pWalReader) < 0) { + return FETCH_TYPE__NONE; + } + + void* pBody = POINTER_SHIFT(pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t bodyLen = pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pWalReader->pHead->head.version; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, pBody, bodyLen); + if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) { + tDecoderClear(&decoder); + tqError("decode wal file error, msgLen:%d, ver:%"PRId64, bodyLen, ver); + return FETCH_TYPE__NONE; + } + + tDecoderClear(&decoder); + pReader->nextBlk = 0; + } + + size_t numOfBlocks = taosArrayGetSize(pReader->submit.aSubmitTbData); + while (pReader->nextBlk < numOfBlocks) { + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg.msgStr, pReader->msg.msgLen, + pReader->msg.ver, pReader->nextBlk); + + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + + if (pReader->tbIdHash == NULL) { + int32_t code = tqRetrieveDataBlock(pReader->pResBlock, pReader, NULL); + if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { + return FETCH_TYPE__DATA; + } + } + + void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); + if (ret != NULL) { + tqDebug("tq reader return submit block, uid:%"PRId64", ver:%"PRId64, pSubmitTbData->uid, pReader->msg.ver); + + int32_t code = tqRetrieveDataBlock(pReader->pResBlock, pReader, NULL); + if (code == TSDB_CODE_SUCCESS && pReader->pResBlock->info.rows > 0) { + return FETCH_TYPE__DATA; + } + } else { + pReader->nextBlk += 1; + tqDebug("tq reader discard submit block, uid:%"PRId64", continue", pSubmitTbData->uid); + } + } + + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + pReader->msg.msgStr = NULL; + } +} + int32_t tqNextBlock(STqReader* pReader, SSDataBlock* pBlock) { while (1) { - if (pReader->msg2.msgStr == NULL) { + if (pReader->msg.msgStr == NULL) { if (walNextValidMsg(pReader->pWalReader) < 0) { return FETCH_TYPE__NONE; } @@ -337,8 +399,7 @@ int32_t tqNextBlock(STqReader* pReader, SSDataBlock* pBlock) { } while (tqNextBlockImpl(pReader)) { - memset(pBlock, 0, sizeof(SSDataBlock)); - int32_t code = tqRetrieveDataBlock(pBlock, pReader, NULL); + int32_t code = tqRetrieveDataBlock(pReader->pResBlock, pReader, NULL); if (code != TSDB_CODE_SUCCESS || pBlock->info.rows == 0) { continue; } @@ -349,31 +410,33 @@ int32_t tqNextBlock(STqReader* pReader, SSDataBlock* pBlock) { } int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { - pReader->msg2.msgStr = msgStr; - pReader->msg2.msgLen = msgLen; - pReader->msg2.ver = ver; + pReader->msg.msgStr = msgStr; + pReader->msg.msgLen = msgLen; + pReader->msg.ver = ver; tqDebug("tq reader set msg %p %d", msgStr, msgLen); SDecoder decoder; - tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); - if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { + + tDecoderInit(&decoder, pReader->msg.msgStr, pReader->msg.msgLen); + if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) { tDecoderClear(&decoder); tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver); return -1; } + tDecoderClear(&decoder); return 0; } bool tqNextBlockImpl(STqReader* pReader) { - if (pReader->msg2.msgStr == NULL) { + if (pReader->msg.msgStr == NULL) { return false; } int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { - tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, - pReader->msg2.ver, pReader->nextBlk); + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg.msgStr, pReader->msg.msgLen, + pReader->msg.ver, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); if (pReader->tbIdHash == NULL) { @@ -382,7 +445,7 @@ bool tqNextBlockImpl(STqReader* pReader) { void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { - tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg2.ver, pSubmitTbData->uid); + tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg.ver, pSubmitTbData->uid); return true; } else { tqDebug("tq reader discard submit block, uid:%"PRId64", continue", pSubmitTbData->uid); @@ -391,15 +454,15 @@ bool tqNextBlockImpl(STqReader* pReader) { pReader->nextBlk++; } - tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->nextBlk = 0; - pReader->msg2.msgStr = NULL; + pReader->msg.msgStr = NULL; return false; } bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { - if (pReader->msg2.msgStr == NULL) return false; + if (pReader->msg.msgStr == NULL) return false; int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { @@ -413,9 +476,9 @@ bool tqNextDataBlockFilterOut(STqReader* pReader, SHashObj* filterOutUids) { pReader->nextBlk++; } - tDestroySSubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); + tDestroySubmitReq(&pReader->submit, TSDB_MSG_FLG_DECODE); pReader->nextBlk = 0; - pReader->msg2.msgStr = NULL; + pReader->msg.msgStr = NULL; return false; } @@ -450,10 +513,9 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap } int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { - tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg2.msgStr, pReader->nextBlk); - SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - pReader->nextBlk++; + tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg.msgStr, pReader->nextBlk); + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk++); if (pSubmitTbDataRet) { *pSubmitTbDataRet = pSubmitTbData; } @@ -464,21 +526,11 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbDa pReader->lastBlkUid = uid; pBlock->info.id.uid = uid; - pBlock->info.version = pReader->msg2.ver; + pBlock->info.version = pReader->msg.ver; if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) { - taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 - "), version %d, possibly dropped table", - pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } + tDeleteSchemaWrapper(pReader->pSchemaWrapper); - tDeleteSSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", @@ -488,93 +540,140 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbDa return -1; } - STSchema* pTschema = pReader->pSchema; + pReader->cachedSchemaSuid = suid; + pReader->cachedSchemaVer = sversion; + SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); - - if (colNumNeed == 0) { - int32_t colMeta = 0; - while (colMeta < pSchemaWrapper->nCols) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + int32_t numOfCols = taosArrayGetSize(pReader->pColIdList); + if (numOfCols == 0) { // all columns are required + for (int32_t i = 0; i < pSchemaWrapper->nCols; ++i) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[i]; SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); if (code != TSDB_CODE_SUCCESS) { - goto FAIL; + blockDataFreeRes(pBlock); + return -1; } - colMeta++; } } else { - if (colNumNeed > pSchemaWrapper->nCols) { - colNumNeed = pSchemaWrapper->nCols; + if (numOfCols > pSchemaWrapper->nCols) { + numOfCols = pSchemaWrapper->nCols; } - int32_t colMeta = 0; - int32_t colNeed = 0; - while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; + int32_t i = 0; + int32_t j = 0; + while (i < pSchemaWrapper->nCols && j < numOfCols) { + SSchema* pColSchema = &pSchemaWrapper->pSchema[i]; col_id_t colIdSchema = pColSchema->colId; - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); + + col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, j); if (colIdSchema < colIdNeed) { - colMeta++; + i++; } else if (colIdSchema > colIdNeed) { - colNeed++; + j++; } else { SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); int32_t code = blockDataAppendColInfo(pBlock, &colInfo); if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - colMeta++; - colNeed++; + i++; + j++; } } } + } - int32_t numOfRows = 0; + int32_t numOfRows = 0; + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SColData* pCol = taosArrayGet(pSubmitTbData->aCol, 0); + numOfRows = pCol->nVal; + } else { + numOfRows = taosArrayGetSize(pSubmitTbData->aRowP); + } - if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - SArray* pCols = pSubmitTbData->aCol; - SColData* pCol = taosArrayGet(pCols, 0); - numOfRows = pCol->nVal; - } else { - SArray* pRows = pSubmitTbData->aRowP; - numOfRows = taosArrayGetSize(pRows); + if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto FAIL; + } + + pBlock->info.rows = numOfRows; + + int32_t colActual = blockDataGetNumOfCols(pBlock); + + // convert and scan one block + if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { + SArray* pCols = pSubmitTbData->aCol; + int32_t numOfCols = taosArrayGetSize(pCols); + int32_t targetIdx = 0; + int32_t sourceIdx = 0; + while (targetIdx < colActual) { + if (sourceIdx >= numOfCols) { + tqError("tqRetrieveDataBlock sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols); + goto FAIL; + } + + SColData* pCol = taosArrayGet(pCols, sourceIdx); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); + SColVal colVal; + + if (pCol->nVal != numOfRows) { + tqError("tqRetrieveDataBlock pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + goto FAIL; + } + + if (pCol->cid < pColData->info.colId) { + sourceIdx++; + } else if (pCol->cid == pColData->info.colId) { + for (int32_t i = 0; i < pCol->nVal; i++) { + tColDataGetValue(pCol, i, &colVal); + if (IS_STR_DATA_TYPE(colVal.type)) { + if (colVal.value.pData != NULL) { + char val[65535 + 2] = {0}; + memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); + varDataSetLen(val, colVal.value.nData); + if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } else { + colDataSetNULL(pColData, i); + } + } else { + if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { + goto FAIL; + } + } + } + sourceIdx++; + targetIdx++; + } else { + for (int32_t i = 0; i < pCol->nVal; i++) { + colDataSetNULL(pColData, i); + } + + targetIdx++; + } } + } else { + SArray* pRows = pSubmitTbData->aRowP; + SSchemaWrapper* pWrapper = pReader->pSchemaWrapper; + STSchema* pTSchema = tBuildTSchema(pWrapper->pSchema, pWrapper->nCols, pWrapper->version); - if (blockDataEnsureCapacity(pBlock, numOfRows) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - pBlock->info.rows = numOfRows; - - int32_t colActual = blockDataGetNumOfCols(pBlock); - - // convert and scan one block - if (pSubmitTbData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { - SArray* pCols = pSubmitTbData->aCol; - int32_t numOfCols = taosArrayGetSize(pCols); - int32_t targetIdx = 0; + for (int32_t i = 0; i < numOfRows; i++) { + SRow* pRow = taosArrayGetP(pRows, i); int32_t sourceIdx = 0; - while (targetIdx < colActual) { - if(sourceIdx >= numOfCols){ - tqError("tqRetrieveDataBlock sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols); - goto FAIL; - } - SColData* pCol = taosArrayGet(pCols, sourceIdx); - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); - SColVal colVal; - if(pCol->nVal != numOfRows){ - tqError("tqRetrieveDataBlock pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); - goto FAIL; - } - - if (pCol->cid < pColData->info.colId) { - sourceIdx++; - } else if (pCol->cid == pColData->info.colId) { - for (int32_t i = 0; i < pCol->nVal; i++) { - tColDataGetValue(pCol, i, &colVal); + for (int32_t j = 0; j < colActual; j++) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); + while (1) { + SColVal colVal; + tRowGet(pRow, pTSchema, sourceIdx, &colVal); + if (colVal.cid < pColData->info.colId) { + sourceIdx++; + continue; + } else if (colVal.cid == pColData->info.colId) { if (IS_STR_DATA_TYPE(colVal.type)) { if (colVal.value.pData != NULL) { char val[65535 + 2] = {0}; @@ -591,59 +690,18 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbDa goto FAIL; } } - } - sourceIdx++; - targetIdx++; - } else { - for (int32_t i = 0; i < pCol->nVal; i++) { + + sourceIdx++; + break; + } else { colDataSetNULL(pColData, i); - } - targetIdx++; - } - } - } else { - SArray* pRows = pSubmitTbData->aRowP; - - for (int32_t i = 0; i < numOfRows; i++) { - SRow* pRow = taosArrayGetP(pRows, i); - int32_t sourceIdx = 0; - - for (int32_t j = 0; j < colActual; j++) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); - while (1) { - SColVal colVal; - tRowGet(pRow, pTschema, sourceIdx, &colVal); - if (colVal.cid < pColData->info.colId) { - sourceIdx++; - continue; - } else if (colVal.cid == pColData->info.colId) { - if (IS_STR_DATA_TYPE(colVal.type)) { - if (colVal.value.pData != NULL) { - char val[65535 + 2] = {0}; - memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); - varDataSetLen(val, colVal.value.nData); - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - goto FAIL; - } - } else { - colDataSetNULL(pColData, i); - } - } else { - if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - goto FAIL; - } - } - - sourceIdx++; - break; - } else { - colDataSetNULL(pColData, i); - break; - } + break; } } } } + + taosMemoryFreeClear(pTSchema); } return 0; @@ -654,7 +712,7 @@ FAIL: } int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { - tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); + tqDebug("tq reader retrieve data block %p, %d", pReader->msg.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); pReader->nextBlk++; @@ -665,18 +723,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas int64_t uid = pSubmitTbData->uid; pReader->lastBlkUid = uid; - taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 - "), version %d, possibly dropped table", - pReader->pWalReader->pWal->cfg.vgId, uid, suid, sversion); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - tDeleteSSchemaWrapper(pReader->pSchemaWrapper); + tDeleteSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); if (pReader->pSchemaWrapper == NULL) { tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", @@ -686,7 +733,6 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas return -1; } - STSchema* pTschema = pReader->pSchema; SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; int32_t numOfRows = 0; @@ -743,18 +789,18 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) { blockDataFreeRes(&block); - tDeleteSSchemaWrapper(pSW); + tDeleteSchemaWrapper(pSW); goto FAIL; } tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, (int32_t)taosArrayGetSize(block.pDataBlock)); block.info.id.uid = uid; - block.info.version = pReader->msg2.ver; + block.info.version = pReader->msg.ver; if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; blockDataFreeRes(&block); - tDeleteSSchemaWrapper(pSW); + tDeleteSchemaWrapper(pSW); goto FAIL; } taosArrayPush(blocks, &block); @@ -803,14 +849,17 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas curRow++; } } else { + SSchemaWrapper* pWrapper = pReader->pSchemaWrapper; + STSchema* pTSchema = tBuildTSchema(pWrapper->pSchema, pWrapper->nCols, pWrapper->version); SArray* pRows = pSubmitTbData->aRowP; + for (int32_t i = 0; i < numOfRows; i++) { SRow* pRow = taosArrayGetP(pRows, i); bool buildNew = false; - for (int32_t j = 0; j < pTschema->numOfCols; j++) { + for (int32_t j = 0; j < pTSchema->numOfCols; j++) { SColVal colVal; - tRowGet(pRow, pTschema, j, &colVal); + tRowGet(pRow, pTSchema, j, &colVal); if (curRow == 0) { assigned[j] = !COL_VAL_IS_NONE(&colVal); buildNew = true; @@ -839,18 +888,18 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas if (tqMaskBlock(pSW, &block, pSchemaWrapper, assigned) < 0) { blockDataFreeRes(&block); - tDeleteSSchemaWrapper(pSW); + tDeleteSchemaWrapper(pSW); goto FAIL; } tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, (int32_t)taosArrayGetSize(block.pDataBlock)); block.info.id.uid = uid; - block.info.version = pReader->msg2.ver; + block.info.version = pReader->msg.ver; if (blockDataEnsureCapacity(&block, numOfRows - curRow) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; blockDataFreeRes(&block); - tDeleteSSchemaWrapper(pSW); + tDeleteSchemaWrapper(pSW); goto FAIL; } taosArrayPush(blocks, &block); @@ -868,7 +917,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas while (targetIdx < colActual) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); SColVal colVal; - tRowGet(pRow, pTschema, sourceIdx, &colVal); + tRowGet(pRow, pTSchema, sourceIdx, &colVal); if (colVal.cid < pColData->info.colId) { sourceIdx++; @@ -895,6 +944,8 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas } curRow++; } + + taosMemoryFreeClear(pTSchema); } SSDataBlock* pLastBlock = taosArrayGetLast(blocks); diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 8e243a8bd1..3d9cea54ba 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -215,7 +215,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); pSchemas = taosArrayInit(0, sizeof(void*)); continue; @@ -274,7 +274,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); - taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); + taosArrayDestroyP(pSchemas, (FDelete)tDeleteSchemaWrapper); pBlocks = taosArrayInit(0, sizeof(SSDataBlock)); pSchemas = taosArrayInit(0, sizeof(void*)); continue; diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index c2e6946b04..33d1e08c9c 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -672,7 +672,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* } SRow* pRow = NULL; if ((terrno = tRowBuild(pVals, (STSchema*)pTSchema, &pRow)) < 0) { - tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } ASSERT(pRow); @@ -681,7 +681,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* SSubmitReq2 submitReq = {0}; if (!(submitReq.aSubmitTbData = taosArrayInit(1, sizeof(SSubmitTbData)))) { - tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } @@ -690,28 +690,28 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* // encode int32_t len; int32_t code; - tEncodeSize(tEncodeSSubmitReq2, &submitReq, len, code); + tEncodeSize(tEncodeSubmitReq, &submitReq, len, code); SEncoder encoder; len += sizeof(SSubmitReq2Msg); pBuf = rpcMallocCont(len); if (NULL == pBuf) { - tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); goto _end; } ((SSubmitReq2Msg*)pBuf)->header.vgId = TD_VID(pVnode); ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); - if (tEncodeSSubmitReq2(&encoder, &submitReq) < 0) { + if (tEncodeSubmitReq(&encoder, &submitReq) < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to encode submit req since %s", terrstr()); tEncoderClear(&encoder); rpcFreeCont(pBuf); - tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); continue; } tEncoderClear(&encoder); - tDestroySSubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(&submitReq, TSDB_MSG_FLG_ENCODE); SRpcMsg msg = { .msgType = TDMT_VND_SUBMIT, diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index d186c63871..133c51a8dc 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -180,7 +180,7 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { // lock taosWLockLatch(&pTq->lock); - code = tqRegisterPushEntry(pTq, pHandle, pMsg); + code = tqRegisterPushHandle(pTq, pHandle, pMsg); taosWUnLockLatch(&pTq->lock); tDeleteSMqDataRsp(&dataRsp); return code; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 251efeab3d..f8161427db 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1007,7 +1007,7 @@ static int32_t vnodeResetTableCxt(SMeta *pMeta, SSubmitReqConvertCxt *pCxt) { } tdSTSRowIterInit(&pCxt->rowIter, pCxt->pTbSchema); - tDestroySSubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE); if (NULL == pCxt->pTbData) { pCxt->pTbData = taosMemoryCalloc(1, sizeof(SSubmitTbData)); if (NULL == pCxt->pTbData) { @@ -1039,7 +1039,7 @@ static int32_t vnodeResetTableCxt(SMeta *pMeta, SSubmitReqConvertCxt *pCxt) { static void vnodeDestroySubmitReqConvertCxt(SSubmitReqConvertCxt *pCxt) { taosMemoryFreeClear(pCxt->pTbSchema); - tDestroySSubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(pCxt->pTbData, TSDB_MSG_FLG_ENCODE); taosMemoryFreeClear(pCxt->pTbData); taosArrayDestroy(pCxt->pColValues); } @@ -1149,7 +1149,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { int32_t code = TSDB_CODE_SUCCESS; char *pMsg = NULL; uint32_t msglen = 0; - tEncodeSize(tEncodeSSubmitReq2, pSubmitReq, msglen, code); + tEncodeSize(tEncodeSubmitReq, pSubmitReq, msglen, code); if (TSDB_CODE_SUCCESS == code) { pMsg = taosMemoryMalloc(msglen); if (NULL == pMsg) { @@ -1159,7 +1159,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { if (TSDB_CODE_SUCCESS == code) { SEncoder encoder; tEncoderInit(&encoder, pMsg, msglen); - code = tEncodeSSubmitReq2(&encoder, pSubmitReq); + code = tEncodeSubmitReq(&encoder, pSubmitReq); tEncoderClear(&encoder); } if (TSDB_CODE_SUCCESS == code) { @@ -1199,7 +1199,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq len -= sizeof(SSubmitReq2Msg); SDecoder dc = {0}; tDecoderInit(&dc, pReq, len); - if (tDecodeSSubmitReq2(&dc, pSubmitReq) < 0) { + if (tDecodeSubmitReq(&dc, pSubmitReq) < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; } @@ -1388,7 +1388,7 @@ _exit: // clear taosArrayDestroy(newTbUids); - tDestroySSubmitReq(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE); + tDestroySubmitReq(pSubmitReq, 0 == pMsg->version ? TSDB_MSG_FLG_CMPT : TSDB_MSG_FLG_DECODE); tDestroySSubmitRsp2(pSubmitRsp, TSDB_MSG_FLG_ENCODE); if (code) terrno = code; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 33eccf4759..d31ac0bc51 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -126,7 +126,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int int32_t code = TSDB_CODE_SUCCESS; int32_t len = 0; void* pBuf = NULL; - tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + tEncodeSize(tEncodeSubmitReq, pReq, len, code); if (TSDB_CODE_SUCCESS == code) { SEncoder encoder; len += sizeof(SSubmitReq2Msg); @@ -138,7 +138,7 @@ static int32_t submitReqToMsg(int32_t vgId, SSubmitReq2* pReq, void** pData, int ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); - code = tEncodeSSubmitReq2(&encoder, pReq); + code = tEncodeSubmitReq(&encoder, pReq); tEncoderClear(&encoder); } @@ -281,7 +281,7 @@ int32_t buildSubmitReqFromBlock(SDataInserterHandle* pInserter, SSubmitReq2** pp SRow* pRow = NULL; if ((terrno = tRowBuild(pVals, pTSchema, &pRow)) < 0) { - tDestroySSubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(&tbData, TSDB_MSG_FLG_ENCODE); goto _end; } taosArrayPush(tbData.aRowP, &pRow); @@ -301,7 +301,7 @@ _end: if (terrno != 0) { *ppReq = NULL; if (pReq) { - tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } return terrno; @@ -326,7 +326,7 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32 code = buildSubmitReqFromBlock(pInserter, &pReq, pDataBlock, pTSchema, uid, vgId, suid); if (code) { if (pReq) { - tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); } @@ -335,7 +335,7 @@ int32_t dataBlocksToSubmitReq(SDataInserterHandle* pInserter, void** pMsg, int32 } code = submitReqToMsg(vgId, pReq, pMsg, msgLen); - tDestroySSubmitReq(pReq, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); return code; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2d991a14f5..5fc079b7c1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1052,19 +1052,6 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s return TSDB_CODE_SUCCESS; } -int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - if ((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)) { - qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr); - terrno = TSDB_CODE_PAR_INTERNAL_ERROR; - return -1; - } - qDebug("set the submit block for future scan"); - - pTaskInfo->streamInfo.submit = submit; - return 0; -} - void qStreamSetOpen(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; @@ -1086,6 +1073,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (pOperator == NULL) { return -1; } + SStreamScanInfo* pInfo = pOperator->info; STableScanInfo* pScanInfo = pInfo->pTableScanOp->info; STableScanBase* pScanBaseInfo = &pScanInfo->base; @@ -1221,7 +1209,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); - tDeleteSSchemaWrapper(pTaskInfo->streamInfo.schema); + tDeleteSchemaWrapper(pTaskInfo->streamInfo.schema); pTaskInfo->streamInfo.schema = mtInfo.schema; qDebug("tmqsnap qStreamPrepareScan snapshot data uid:%" PRId64 " ts %" PRId64 " %s", mtInfo.uid, pOffset->ts, id); diff --git a/source/libs/executor/src/querytask.c b/source/libs/executor/src/querytask.c index a4d8327b6a..7716b5976b 100644 --- a/source/libs/executor/src/querytask.c +++ b/source/libs/executor/src/querytask.c @@ -109,8 +109,8 @@ int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHand void cleanupQueriedTableScanInfo(SSchemaInfo* pSchemaInfo) { taosMemoryFreeClear(pSchemaInfo->dbname); taosMemoryFreeClear(pSchemaInfo->tablename); - tDeleteSSchemaWrapper(pSchemaInfo->sw); - tDeleteSSchemaWrapper(pSchemaInfo->qsw); + tDeleteSchemaWrapper(pSchemaInfo->sw); + tDeleteSchemaWrapper(pSchemaInfo->qsw); } int32_t initQueriedTableSchemaInfo(SReadHandle* pHandle, SScanPhysiNode* pScanNode, const char* dbName, SExecTaskInfo* pTaskInfo) { @@ -197,7 +197,7 @@ SSchemaWrapper* extractQueriedColumnSchema(SScanPhysiNode* pScanNode) { return pqSw; } -static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSSchemaWrapper(pStreamInfo->schema); } +static void cleanupStreamInfo(SStreamTaskInfo* pStreamInfo) { tDeleteSchemaWrapper(pStreamInfo->schema); } static void freeBlock(void* pParam) { SSDataBlock* pBlock = *(SSDataBlock**)pParam; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 130cca9cbb..0f4e18105c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1623,7 +1623,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock pInfo->pRes->info.dataLoad = 1; blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); - blockDataFreeRes((SSDataBlock*)pBlock); +// blockDataFreeRes((SSDataBlock*)pBlock); calBlockTbName(pInfo, pInfo->pRes); return 0; @@ -1637,7 +1637,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { qDebug("start to exec queue scan, %s", id); if (pTaskInfo->streamInfo.submit.msgStr != NULL) { - if (pInfo->tqReader->msg2.msgStr == NULL) { + if (pInfo->tqReader->msg.msgStr == NULL) { SPackedData submit = pTaskInfo->streamInfo.submit; if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { qError("submit msg messed up when initing stream submit block %p", submit.msgStr); @@ -1663,7 +1663,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } } - pInfo->tqReader->msg2 = (SPackedData){0}; + pInfo->tqReader->msg = (SPackedData){0}; pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } @@ -1689,17 +1689,17 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { - SSDataBlock block = {0}; - int32_t type = tqNextBlock(pInfo->tqReader, &block); + int32_t type = tqNextBlockInWal(pInfo->tqReader); + SSDataBlock* pRes = pInfo->tqReader->pResBlock; // curVersion move to next, so currentOffset = curVersion - 1 tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pInfo->tqReader->pWalReader->curVersion - 1); if (type == FETCH_TYPE__DATA) { - qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, block.info.rows, + qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); - setBlockIntoRes(pInfo, &block, true); + setBlockIntoRes(pInfo, pRes, true); if (pInfo->pRes->info.rows > 0) { qDebug("doQueueScan get data from log %" PRId64 " rows, return, version:%" PRId64, pInfo->pRes->info.rows, pTaskInfo->streamInfo.currentOffset.version); @@ -2055,7 +2055,7 @@ FETCH_NEXT_BLOCK: NEXT_SUBMIT_BLK: while (1) { - if (pInfo->tqReader->msg2.msgStr == NULL) { + if (pInfo->tqReader->msg.msgStr == NULL) { if (pInfo->validBlockIndex >= totBlockNum) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); doClearBufferedBlocks(pInfo); @@ -2191,7 +2191,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); } qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); - tDeleteSSchemaWrapper(mtInfo.schema); + tDeleteSchemaWrapper(mtInfo.schema); return NULL; } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) { SSnapContext* sContext = pInfo->sContext; diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index a3b067b94d..f921094752 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -313,7 +313,7 @@ void insDestroyTableDataCxt(STableDataCxt* pTableCxt) { insDestroyBoundColInfo(&pTableCxt->boundColsInfo); taosArrayDestroyEx(pTableCxt->pValues, destroyColVal); if (pTableCxt->pData) { - tDestroySSubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitTbData(pTableCxt->pData, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pTableCxt->pData); } taosMemoryFree(pTableCxt); @@ -324,7 +324,7 @@ void insDestroyVgroupDataCxt(SVgroupDataCxt* pVgCxt) { return; } - tDestroySSubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); + tDestroySubmitReq(pVgCxt->pData, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pVgCxt->pData); taosMemoryFree(pVgCxt); } @@ -499,7 +499,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin int32_t code = TSDB_CODE_SUCCESS; uint32_t len = 0; void* pBuf = NULL; - tEncodeSize(tEncodeSSubmitReq2, pReq, len, code); + tEncodeSize(tEncodeSubmitReq, pReq, len, code); if (TSDB_CODE_SUCCESS == code) { SEncoder encoder; len += sizeof(SSubmitReq2Msg); @@ -511,7 +511,7 @@ static int32_t buildSubmitReq(int32_t vgId, SSubmitReq2* pReq, void** pData, uin ((SSubmitReq2Msg*)pBuf)->header.contLen = htonl(len); ((SSubmitReq2Msg*)pBuf)->version = htobe64(1); tEncoderInit(&encoder, POINTER_SHIFT(pBuf, sizeof(SSubmitReq2Msg)), len - sizeof(SSubmitReq2Msg)); - code = tEncodeSSubmitReq2(&encoder, pReq); + code = tEncodeSubmitReq(&encoder, pReq); tEncoderClear(&encoder); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 67c60008fd..f301d9d517 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -193,7 +193,7 @@ void tFreeStreamTask(SStreamTask* pTask) { taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree); if (pTask->outputType == TASK_OUTPUT__TABLE) { - tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper); + tDeleteSchemaWrapper(pTask->tbSink.pSchemaWrapper); taosMemoryFree(pTask->tbSink.pTSchema); } From 1a8a834a2cbe3ce8a140a8848bb9d1e08c79dbf4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 May 2023 17:03:53 +0800 Subject: [PATCH 122/145] fix(tmq): fix memory leak. --- source/dnode/vnode/src/tq/tqRead.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0c9c7b4793..3ee706cd39 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -341,6 +341,8 @@ int32_t tqNextBlockInWal(STqReader* pReader) { SDecoder decoder = {0}; tDecoderInit(&decoder, pBody, bodyLen); + taosArrayDestroy(pReader->submit.aSubmitTbData); + if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) { tDecoderClear(&decoder); tqError("decode wal file error, msgLen:%d, ver:%"PRId64, bodyLen, ver); From af0ca38a897fa5d63e71f24ab1427007b73c0436 Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 4 May 2023 17:12:02 +0800 Subject: [PATCH 123/145] int16 overflow --- include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndStb.c | 20 ++++++++++++++++++++ source/util/src/terror.c | 1 + 3 files changed, 22 insertions(+) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index ab89466a19..3847757d9d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -261,6 +261,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) // 2.x #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) +#define TSDB_CODE_MND_BIG_FIELD_VALUE TAOS_DEF_ERROR_CODE(0, 0x0370) // mnode-func diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 63bcef2a5b..939080be47 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -797,6 +797,11 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat return -1; } + if(pDst->nextColId > 0 && pDst->nextColId < 0x7fff - pDst->numOfColumns - pDst->numOfTags){ + terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + return -1; + } + for (int32_t i = 0; i < pDst->numOfColumns; ++i) { SField *pField = taosArrayGet(pCreate->pColumns, i); SSchema *pSchema = &pDst->pColumns[i]; @@ -927,6 +932,11 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq return -1; } + if(pDst->nextColId > 0 && pDst->nextColId < 0x7fff - pDst->numOfColumns - pDst->numOfTags){ + terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + return -1; + } + for (int32_t i = 0; i < pDst->numOfColumns; ++i) { SField *pField = taosArrayGet(createReq->pColumns, i); SSchema *pSchema = &pDst->pColumns[i]; @@ -1154,6 +1164,11 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p return -1; } + if(pNew->nextColId > 0 && pNew->nextColId < 0x7fff - ntags){ + terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + return -1; + } + for (int32_t i = 0; i < ntags; i++) { SField *pField = taosArrayGet(pFields, i); if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { @@ -1461,6 +1476,11 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray return -1; } + if(pNew->nextColId > 0 && pNew->nextColId < 0x7fff - ncols){ + terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + return -1; + } + for (int32_t i = 0; i < ncols; i++) { SField *pField = taosArrayGet(pFields, i); if (mndFindSuperTableColumnIndex(pOld, pField->name) >= 0) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 002d605793..f3aff1a200 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -203,6 +203,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES, "Invalid row bytes") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_BIG_FIELD_VALUE, "out of range and overflow") // mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") From 766b752c18e83d1ef91e7b44b433023ed5fb7f0c Mon Sep 17 00:00:00 2001 From: dmchen Date: Thu, 4 May 2023 18:35:37 +0800 Subject: [PATCH 124/145] if statement for overflow --- source/dnode/mnode/impl/src/mndStb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 939080be47..3407d4a1dc 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -797,7 +797,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat return -1; } - if(pDst->nextColId > 0 && pDst->nextColId < 0x7fff - pDst->numOfColumns - pDst->numOfTags){ + if(pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){ terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; return -1; } @@ -932,7 +932,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq return -1; } - if(pDst->nextColId > 0 && pDst->nextColId < 0x7fff - pDst->numOfColumns - pDst->numOfTags){ + if(pDst->nextColId < 0 && pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){ terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; return -1; } @@ -1164,7 +1164,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p return -1; } - if(pNew->nextColId > 0 && pNew->nextColId < 0x7fff - ntags){ + if(pNew->nextColId < 0 && pNew->nextColId >= 0x7fff - ntags){ terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; return -1; } @@ -1476,7 +1476,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray return -1; } - if(pNew->nextColId > 0 && pNew->nextColId < 0x7fff - ncols){ + if(pNew->nextColId < 0 && pNew->nextColId >= 0x7fff - ncols){ terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; return -1; } From 25f451ba819beb71f7a8f01c0b52e60502c3bc73 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 4 May 2023 23:34:35 +0800 Subject: [PATCH 125/145] fix(tmq): fix result data block info. --- source/dnode/mnode/impl/src/mndSubscribe.c | 4 ++-- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tq/tqRead.c | 15 ++++++++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 573c60549e..f4d6e27dea 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -197,7 +197,7 @@ static SMqRebInfo *mndGetOrCreateRebSub(SHashObj *pHash, const char *key) { return pRebSub; } -static void doRemoveExistedConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { +static void doRemoveLostConsumers(SMqRebOutputObj *pOutput, SHashObj *pHash, const SMqRebInputObj *pInput) { int32_t numOfRemoved = taosArrayGetSize(pInput->pRebInfo->removedConsumers); const char *pSubKey = pOutput->pSub->key; @@ -339,7 +339,7 @@ static int32_t mndDoRebalance(SMnode *pMnode, const SMqRebInputObj *pInput, SMqR SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); // 2. check and get actual removed consumers, put their vg into pHash - doRemoveExistedConsumers(pOutput, pHash, pInput); + doRemoveLostConsumers(pOutput, pHash, pInput); // 3. if previously no consumer, there are vgs not assigned, put these vg into pHash addUnassignedVgroups(pOutput, pHash); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 817e6fdae4..c7424cd233 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -241,6 +241,7 @@ typedef struct STqReader { SArray *pColIdList; // SArray int32_t cachedSchemaVer; int64_t cachedSchemaSuid; + int64_t cachedSchemaUid; SSchemaWrapper *pSchemaWrapper; SSDataBlock *pResBlock; } STqReader; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 3ee706cd39..8622216b28 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -530,22 +530,27 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbDa pBlock->info.id.uid = uid; pBlock->info.version = pReader->msg.ver; - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || pReader->cachedSchemaSuid != suid) { + if ((suid != 0 && pReader->cachedSchemaSuid != suid) || (suid == 0 && pReader->cachedSchemaUid != uid) || (pReader->cachedSchemaVer != sversion)) { tDeleteSchemaWrapper(pReader->pSchemaWrapper); pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, uid, sversion, 1); if (pReader->pSchemaWrapper == NULL) { - tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pReader->pWalReader->pWal->cfg.vgId, uid, pReader->cachedSchemaVer); + tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", uid:%" PRId64 "version %d, possibly dropped table", + pReader->pWalReader->pWal->cfg.vgId, suid, uid, pReader->cachedSchemaVer); pReader->cachedSchemaSuid = 0; terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; return -1; } + pReader->cachedSchemaUid = uid; pReader->cachedSchemaSuid = suid; pReader->cachedSchemaVer = sversion; SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; + if (blockDataGetNumOfCols(pBlock) > 0) { + blockDataDestroy(pReader->pResBlock); + pReader->pResBlock = createDataBlock(); + } int32_t numOfCols = taosArrayGetSize(pReader->pColIdList); if (numOfCols == 0) { // all columns are required @@ -671,8 +676,12 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbDa SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); while (1) { SColVal colVal; + tqDebug("start to extract column id:%d, index:%d", pColData->info.colId, sourceIdx); + tRowGet(pRow, pTSchema, sourceIdx, &colVal); if (colVal.cid < pColData->info.colId) { + tqDebug("colIndex:%d column id:%d in row, ignore, the required colId:%d, total cols in schema:%d", + sourceIdx, colVal.cid, pColData->info.colId, pTSchema->numOfCols); sourceIdx++; continue; } else if (colVal.cid == pColData->info.colId) { From 32b4642dac9ddb3a94b6b99feb72ebd1783965d5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 00:57:43 +0800 Subject: [PATCH 126/145] fix(tmq): fix invalid free. --- source/dnode/vnode/src/tq/tqRead.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 8622216b28..7ead634f8b 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -341,7 +341,15 @@ int32_t tqNextBlockInWal(STqReader* pReader) { SDecoder decoder = {0}; tDecoderInit(&decoder, pBody, bodyLen); - taosArrayDestroy(pReader->submit.aSubmitTbData); + + { + int32_t nSubmitTbData = taosArrayGetSize(pReader->submit.aSubmitTbData); + for (int32_t i = 0; i < nSubmitTbData; i++) { + SSubmitTbData* pData = taosArrayGet(pReader->submit.aSubmitTbData, i); + pData->aRowP = taosArrayDestroy(pData->aRowP); + } + pReader->submit.aSubmitTbData = taosArrayDestroy(pReader->submit.aSubmitTbData); + } if (tDecodeSubmitReq(&decoder, &pReader->submit) < 0) { tDecoderClear(&decoder); From 42b23e6471e7c7b4cf77cb3a58b22d76d5da57f0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 10:40:55 +0800 Subject: [PATCH 127/145] fix(query): stop tsdb reader asap. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 54 +++++++++++--------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 1ef86f5b30..8f7ebc6c5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -21,10 +21,9 @@ #define getCurrentKeyInLastBlock(_r) ((_r)->currentKey) typedef enum { - READER_STATUS_SUSPEND = 0x1, - READER_STATUS_SHOULD_STOP = 0x2, - READER_STATUS_NORMAL = 0x3, -} EReaderExecStatus; + READER_STATUS_SUSPEND = 0x1, + READER_STATUS_NORMAL = 0x2, +} EReaderStatus; typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -184,6 +183,7 @@ typedef struct STsdbReaderAttr { STimeWindow window; bool freeBlock; SVersionRange verRange; + int16_t order; } STsdbReaderAttr; typedef struct SResultBlockInfo { @@ -196,7 +196,8 @@ struct STsdbReader { STsdb* pTsdb; SVersionRange verRange; TdThreadMutex readerMutex; - EReaderExecStatus flag; + EReaderStatus flag; + int32_t code; uint64_t suid; int16_t order; EReadMode readMode; @@ -2995,9 +2996,9 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr while (1) { // only check here, since the iterate data in memory is very fast. - if (pReader->flag == READER_STATUS_SHOULD_STOP) { - tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr); - return TSDB_CODE_SUCCESS; + if (pReader->code != TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; } bool hasNext = false; @@ -3093,9 +3094,9 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; while (1) { - if (pReader->flag == READER_STATUS_SHOULD_STOP) { - tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr); - return TSDB_CODE_SUCCESS; + if (pReader->code == TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; } // load the last data block of current table @@ -3246,7 +3247,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } } - return code; + return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code; } static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { @@ -3395,9 +3396,9 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; while (1) { - if (pReader->flag == READER_STATUS_SHOULD_STOP) { - tsdbWarn("tsdb reader is stopped ASAP, %s", pReader->idStr); - return TSDB_CODE_SUCCESS; + if (pReader->code == TSDB_CODE_SUCCESS) { + tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); + return pReader->code; } STableBlockScanInfo** pBlockScanInfo = pStatus->pTableIter; @@ -3493,7 +3494,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { terrno = 0; code = doLoadLastBlockSequentially(pReader); - if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) { + if (code != TSDB_CODE_SUCCESS) { terrno = code; return TSDB_READ_RETURN; } @@ -3507,8 +3508,7 @@ static ERetrieveType doReadDataFromLastFiles(STsdbReader* pReader) { code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked - if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false) || - pReader->flag == READER_STATUS_SHOULD_STOP) { + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { terrno = code; return TSDB_READ_RETURN; } @@ -3536,13 +3536,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } code = doBuildDataBlock(pReader); - if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) { + if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { return code; } - - if (pResBlock->info.rows > 0) { - return TSDB_CODE_SUCCESS; - } } while (1) { @@ -3581,13 +3577,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { code = doBuildDataBlock(pReader); } - if (code != TSDB_CODE_SUCCESS || pReader->flag == READER_STATUS_SHOULD_STOP) { + if (code != TSDB_CODE_SUCCESS || pResBlock->info.rows > 0) { return code; } - - if (pResBlock->info.rows > 0) { - return TSDB_CODE_SUCCESS; - } } } @@ -4849,8 +4841,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { *hasNext = false; - if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT) { - return code; + if (isEmptyQueryTimeWindow(&pReader->window) || pReader->step == EXTERNAL_ROWS_NEXT || pReader->code != TSDB_CODE_SUCCESS) { + return (pReader->code != TSDB_CODE_SUCCESS)? pReader->code:code; } SReaderStatus* pStatus = &pReader->status; @@ -5456,4 +5448,4 @@ void tsdbReaderSetId(STsdbReader* pReader, const char* idstr) { pReader->idStr = taosStrdup(idstr); } -void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->flag = READER_STATUS_SHOULD_STOP; } +void tsdbReaderSetCloseFlag(STsdbReader* pReader) { pReader->code = TSDB_CODE_TSC_QUERY_CANCELLED; } From 5006ecc484e239dfbc8cf1550c366d9ed1e4c30b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 May 2023 11:45:14 +0800 Subject: [PATCH 128/145] fix: stable name not responsed while stable removed issue --- source/dnode/mnode/impl/src/mndStb.c | 3 +++ source/os/src/osLocale.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 63bcef2a5b..da3c3b98a8 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2524,6 +2524,9 @@ int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp, &smaVer) != 0) { metaRsp.numOfColumns = -1; metaRsp.suid = pStbVersion->suid; + tstrncpy(metaRsp.dbFName, pStbVersion->dbFName, sizeof(metaRsp.dbFName)); + tstrncpy(metaRsp.tbName, pStbVersion->stbName, sizeof(metaRsp.tbName)); + tstrncpy(metaRsp.stbName, pStbVersion->stbName, sizeof(metaRsp.stbName)); taosArrayPush(hbRsp.pMetaRsp, &metaRsp); continue; } diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index 129faaacc8..136b8cf022 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -171,7 +171,7 @@ void taosGetSystemLocale(char *outLocale, char *outCharset) { strcpy(outLocale, "en_US.UTF-8"); } else { tstrncpy(outLocale, locale, TD_LOCALE_LEN); - printf("locale not configured, set to system default:%s\n", outLocale); + //printf("locale not configured, set to system default:%s\n", outLocale); } // if user does not specify the charset, extract it from locale From 38cbe0b768b383d2df9f1ce2b1c4b4f6c758626b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 12:52:10 +0800 Subject: [PATCH 129/145] fix(query): fix the invalid copmarison. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8f7ebc6c5c..d0a0ea7947 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3094,7 +3094,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock; while (1) { - if (pReader->code == TSDB_CODE_SUCCESS) { + if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); return pReader->code; } @@ -3396,7 +3396,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { STableUidList* pUidList = &pStatus->uidList; while (1) { - if (pReader->code == TSDB_CODE_SUCCESS) { + if (pReader->code != TSDB_CODE_SUCCESS) { tsdbWarn("tsdb reader is stopped ASAP, code:%s, %s", strerror(pReader->code), pReader->idStr); return pReader->code; } From 9b5c205498bc85480a1405ce91f9d8879613ff70 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 13:41:25 +0800 Subject: [PATCH 130/145] fix(stream): set correct ssdatablock for tqRetrieveDataBlock --- source/libs/executor/src/scanoperator.c | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0f4e18105c..5122e0e439 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1649,14 +1649,12 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; while (tqNextBlockImpl(pInfo->tqReader)) { - SSDataBlock block = {0}; - - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { + int32_t code = tqRetrieveDataBlock(pInfo->tqReader->pResBlock, pInfo->tqReader, NULL); + if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { continue; } - setBlockIntoRes(pInfo, &block, true); + setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, true); if (pBlockInfo->rows > 0) { return pInfo->pRes; @@ -2075,14 +2073,12 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pInfo->pRes); while (tqNextBlockImpl(pInfo->tqReader)) { - SSDataBlock block = {0}; - - int32_t code = tqRetrieveDataBlock(&block, pInfo->tqReader, NULL); - if (code != TSDB_CODE_SUCCESS || block.info.rows == 0) { + int32_t code = tqRetrieveDataBlock(pInfo->tqReader->pResBlock, pInfo->tqReader, NULL); + if (code != TSDB_CODE_SUCCESS || pInfo->tqReader->pResBlock->info.rows == 0) { continue; } - setBlockIntoRes(pInfo, &block, false); + setBlockIntoRes(pInfo, pInfo->tqReader->pResBlock, false); if (updateInfoIgnore(pInfo->pUpdateInfo, &pInfo->pRes->info.window, pInfo->pRes->info.id.groupId, pInfo->pRes->info.version)) { From 3976504b4758b8df0152d0916c255e4dd22cc0a6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 May 2023 14:01:15 +0800 Subject: [PATCH 131/145] fix: subtable grant is not allowed --- source/libs/parser/src/parTranslater.c | 32 ++++++++++++++++++-------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fcdc6418a0..7f77458d53 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6666,22 +6666,36 @@ static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode** } static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, SAlterUserReq* pReq) { - if (NULL == pStmt->pTagCond) { - return TSDB_CODE_SUCCESS; - } - if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, - "The With clause can only be used for table level privilege"); - } - - pCxt->pCurrStmt = (SNode*)pStmt; SRealTableNode* pTable = NULL; int32_t code = createRealTableForGrantTable(pStmt, &pTable); if (TSDB_CODE_SUCCESS == code) { SName name; code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), &(pTable->pMeta)); + if (code) { + nodesDestroyNode((SNode*)pTable); + return code; + } + + if (TSDB_SUPER_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType) { + nodesDestroyNode((SNode*)pTable); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + "Only supertable and normal table can be granted"); + } } + + if (TSDB_CODE_SUCCESS == code && NULL == pStmt->pTagCond) { + nodesDestroyNode((SNode*)pTable); + return TSDB_CODE_SUCCESS; + } + if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) { + nodesDestroyNode((SNode*)pTable); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + "The With clause can only be used for table level privilege"); + } + + pCxt->pCurrStmt = (SNode*)pStmt; + if (TSDB_CODE_SUCCESS == code) { code = addNamespace(pCxt, pTable); } From e889249c54b2dcf40e3fa4c11a90ef9f3d130079 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 May 2023 14:33:09 +0800 Subject: [PATCH 132/145] fix: grant no table specifed issue --- source/libs/parser/src/parTranslater.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7f77458d53..b598fffbc6 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6667,6 +6667,15 @@ static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode** static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, SAlterUserReq* pReq) { SRealTableNode* pTable = NULL; + if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) { + if (pStmt->pTagCond) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + "The With clause can only be used for table level privilege"); + } else { + return TSDB_CODE_SUCCESS; + } + } + int32_t code = createRealTableForGrantTable(pStmt, &pTable); if (TSDB_CODE_SUCCESS == code) { SName name; @@ -6688,11 +6697,6 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, nodesDestroyNode((SNode*)pTable); return TSDB_CODE_SUCCESS; } - if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) { - nodesDestroyNode((SNode*)pTable); - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, - "The With clause can only be used for table level privilege"); - } pCxt->pCurrStmt = (SNode*)pStmt; From 8b12d4d3dad000c91fef10a218db6231a72f3ee3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 15:02:03 +0800 Subject: [PATCH 133/145] fix(stream): secure the delete task operation. TD-1198 --- source/dnode/vnode/src/tq/tq.c | 2 ++ source/dnode/vnode/src/tq/tqRead.c | 1 - source/dnode/vnode/src/tq/tqRestore.c | 1 + source/libs/stream/src/streamExec.c | 1 + source/libs/stream/src/streamMeta.c | 4 +++- 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 12b81b6c3f..6b46a6a12f 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1326,7 +1326,9 @@ int32_t tqStartStreamTasks(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; + taosWLockLatch(&pMeta->lock); + int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks); if (numOfTasks == 0) { tqInfo("vgId:%d no stream tasks exists", vgId); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index ead00dcc35..0deac6e82c 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -1039,6 +1039,5 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } taosWUnLockLatch(&pTq->pStreamMeta->lock); - return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index 58cb7b9e63..c3c7f7ba7b 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -36,6 +36,7 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { if (shouldIdle) { taosWLockLatch(&pMeta->lock); + pMeta->walScanCounter -= 1; times = pMeta->walScanCounter; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index f33e126068..aa7aaf93d4 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -313,6 +313,7 @@ int32_t streamExecForAll(SStreamTask* pTask) { pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer}; taosWLockLatch(&pTask->pMeta->lock); + streamMetaSaveTask(pTask->pMeta, pTask); if (streamMetaCommit(pTask->pMeta) < 0) { taosWUnLockLatch(&pTask->pMeta->lock); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 822ae2a485..0d797f0bcb 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -216,12 +216,14 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { SStreamTask* pTask = *ppTask; + + taosWLockLatch(&pMeta->lock); + taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP); - taosWLockLatch(&pMeta->lock); streamMetaReleaseTask(pMeta, pTask); taosWUnLockLatch(&pMeta->lock); } From 4576e90fe608e7f9061d4f2389a69829ce576382 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 5 May 2023 15:57:06 +0800 Subject: [PATCH 134/145] Update run_case.sh --- tests/parallel_test/run_case.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index ffc23ba6d2..56e898c5b2 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -79,7 +79,7 @@ md5sum /home/TDinternal/debug/build/lib/libtaos.so #define taospy 2.7.6 pip3 list|grep taospy pip3 uninstall taospy -y -pip3 install taospy==2.7.6 +pip3 install --default-timeout=60 taospy==2.7.6 $TIMEOUT_CMD $cmd RET=$? From 3550f9b8c010606ad110957d8fa4539945b08d32 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 5 May 2023 15:57:49 +0800 Subject: [PATCH 135/145] Update run_case.sh --- tests/parallel_test/run_case.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 56e898c5b2..2d736e1414 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -79,7 +79,7 @@ md5sum /home/TDinternal/debug/build/lib/libtaos.so #define taospy 2.7.6 pip3 list|grep taospy pip3 uninstall taospy -y -pip3 install --default-timeout=60 taospy==2.7.6 +pip3 install --default-timeout=120 taospy==2.7.6 $TIMEOUT_CMD $cmd RET=$? From 8286c145b68ff352d350162741858e9a1123fa4b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 5 May 2023 16:15:15 +0800 Subject: [PATCH 136/145] fix(tdb/decoder): free decoder's value if not needed by user --- source/libs/tdb/src/db/tdbBtree.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 6df2b40000..c49b5726b6 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -1814,6 +1814,11 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { *ppVal = pVal; *vLen = cd.vLen; + } else { + if (TDB_CELLDECODER_FREE_VAL(&cd)) { + tdbTrace("tdb/btree-next2 decoder: %p pVal free: %p", &cd, cd.pVal); + tdbFree(cd.pVal); + } } ret = tdbBtcMoveToNext(pBtc); From ee0e0b4ad4ad28fa9b8da5d0b466616f189ce13d Mon Sep 17 00:00:00 2001 From: dm chen Date: Fri, 5 May 2023 17:46:24 +0800 Subject: [PATCH 137/145] Update taoserror.h --- include/util/taoserror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3847757d9d..2b116f0c82 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -261,7 +261,7 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_MND_INVALID_STABLE_NAME TAOS_DEF_ERROR_CODE(0, 0x036D) // 2.x #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) -#define TSDB_CODE_MND_BIG_FIELD_VALUE TAOS_DEF_ERROR_CODE(0, 0x0370) +#define TSDB_CODE_MND_FIELD_VALUE_OVERFLOW TAOS_DEF_ERROR_CODE(0, 0x0370) // mnode-func From 59f103a4772d15999c20eb9c97bcbf8ed42c72d8 Mon Sep 17 00:00:00 2001 From: dm chen Date: Fri, 5 May 2023 17:47:32 +0800 Subject: [PATCH 138/145] Update mndStb.c --- source/dnode/mnode/impl/src/mndStb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3407d4a1dc..6b3b69e003 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -798,7 +798,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat } if(pDst->nextColId < 0 || pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){ - terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW; return -1; } From 4242f2c4ccfa09af7079f672083c0b37c9c5a637 Mon Sep 17 00:00:00 2001 From: dm chen Date: Fri, 5 May 2023 17:49:27 +0800 Subject: [PATCH 139/145] Update mndStb.c --- source/dnode/mnode/impl/src/mndStb.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 6b3b69e003..21e2bbaf92 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -933,7 +933,7 @@ static int32_t mndBuildStbFromAlter(SStbObj *pStb, SStbObj *pDst, SMCreateStbReq } if(pDst->nextColId < 0 && pDst->nextColId >= 0x7fff - pDst->numOfColumns - pDst->numOfTags){ - terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW; return -1; } @@ -1165,7 +1165,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p } if(pNew->nextColId < 0 && pNew->nextColId >= 0x7fff - ntags){ - terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW; return -1; } @@ -1477,7 +1477,7 @@ static int32_t mndAddSuperTableColumn(const SStbObj *pOld, SStbObj *pNew, SArray } if(pNew->nextColId < 0 && pNew->nextColId >= 0x7fff - ncols){ - terrno = TSDB_CODE_MND_BIG_FIELD_VALUE; + terrno = TSDB_CODE_MND_FIELD_VALUE_OVERFLOW; return -1; } From 87696d8ab313a91fa8a7ffe515ed9f15b8b2d64c Mon Sep 17 00:00:00 2001 From: dm chen Date: Fri, 5 May 2023 17:50:06 +0800 Subject: [PATCH 140/145] Update terror.c --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f3aff1a200..a875f1d209 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -203,7 +203,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STB_OPTION, "Invalid stable options") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ROW_BYTES, "Invalid row bytes") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_BIG_FIELD_VALUE, "out of range and overflow") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_FIELD_VALUE_OVERFLOW, "out of range and overflow") // mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_NAME, "Invalid func name") From 08ce790cda0926560485963e1048f85356002ba1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 18:20:27 +0800 Subject: [PATCH 141/145] fix(query): fix memory leak. --- source/dnode/vnode/src/tq/tqRead.c | 7 +++++++ source/libs/executor/src/scanoperator.c | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 7ead634f8b..3aaf5cd60d 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tmsg.h" #include "tq.h" bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { @@ -346,6 +347,10 @@ int32_t tqNextBlockInWal(STqReader* pReader) { int32_t nSubmitTbData = taosArrayGetSize(pReader->submit.aSubmitTbData); for (int32_t i = 0; i < nSubmitTbData; i++) { SSubmitTbData* pData = taosArrayGet(pReader->submit.aSubmitTbData, i); + if (pData->pCreateTbReq != NULL) { + taosArrayDestroy(pData->pCreateTbReq->ctb.tagName); + taosMemoryFreeClear(pData->pCreateTbReq); + } pData->aRowP = taosArrayDestroy(pData->aRowP); } pReader->submit.aSubmitTbData = taosArrayDestroy(pReader->submit.aSubmitTbData); @@ -530,6 +535,8 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbDa *pSubmitTbDataRet = pSubmitTbData; } + blockDataCleanup(pBlock); + int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5122e0e439..9b1b5235cc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1699,8 +1699,6 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, pRes, true); if (pInfo->pRes->info.rows > 0) { - qDebug("doQueueScan get data from log %" PRId64 " rows, return, version:%" PRId64, pInfo->pRes->info.rows, - pTaskInfo->streamInfo.currentOffset.version); return pInfo->pRes; } } else if (type == FETCH_TYPE__NONE) { From 8958aabe4bc078c0b7d635e7b52ff99e44991abf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 19:33:30 +0800 Subject: [PATCH 142/145] fix(stream): opt task scan performance. --- include/libs/stream/tstream.h | 1 + source/dnode/snode/src/snode.c | 4 ++++ source/dnode/vnode/src/tq/tq.c | 9 ++++--- source/dnode/vnode/src/tq/tqPush.c | 5 ++-- source/dnode/vnode/src/tq/tqRestore.c | 34 ++++++++------------------- source/libs/stream/src/streamMeta.c | 26 +++++++++++++++++++- 6 files changed, 49 insertions(+), 30 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 48c15e9117..5fd9a8b12b 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -340,6 +340,7 @@ typedef struct SStreamMeta { TTB* pTaskDb; TTB* pCheckpointDb; SHashObj* pTasks; + SArray* pTaskList; // SArray void* ahandle; TXN* txn; FTaskExpand* expandFunc; diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index cefc4fa63e..7352bbc0fe 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -153,11 +153,15 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); // 2.save task + taosWLockLatch(&pSnode->pMeta->lock); code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask); if (code < 0) { + taosWUnLockLatch(&pSnode->pMeta->lock); return -1; } + taosWUnLockLatch(&pSnode->pMeta->lock); + // 3.go through recover steps to fill history if (pTask->fillHistory) { streamSetParamForRecover(pTask); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 6b46a6a12f..792ff8677e 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -781,13 +781,17 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms tDecoderClear(&decoder); // 2.save task, use the newest commit version as the initial start version of stream task. + taosWLockLatch(&pTq->pStreamMeta->lock); code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); if (code < 0) { tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr, streamMetaGetNumOfTasks(pTq->pStreamMeta)); + taosWUnLockLatch(&pTq->pStreamMeta->lock); return -1; } + taosWUnLockLatch(&pTq->pStreamMeta->lock); + // 3.go through recover steps to fill history if (pTask->fillHistory) { streamTaskCheckDownstream(pTask, sversion); @@ -1323,13 +1327,12 @@ FAIL: int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; } int32_t tqStartStreamTasks(STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); - + int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; taosWLockLatch(&pMeta->lock); - int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks); + int32_t numOfTasks = taosArrayGetSize(pMeta->pTaskList); if (numOfTasks == 0) { tqInfo("vgId:%d no stream tasks exists", vgId); taosWUnLockLatch(&pTq->pStreamMeta->lock); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 0575b7299d..6a51f74908 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -262,14 +262,15 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v // taosWUnLockLatch(&pTq->lock); } - tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, (int)taosHashGetSize(pTq->pStreamMeta->pTasks)); + int32_t numOfTasks = streamMetaGetNumOfTasks(pTq->pStreamMeta); + tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, numOfTasks); // push data for stream processing: // 1. the vnode has already been restored. // 2. the vnode should be the leader. // 3. the stream is not suspended yet. if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && pTq->pVnode->restored) { - if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) { + if (numOfTasks == 0) { return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c index c3c7f7ba7b..1e45f578f6 100644 --- a/source/dnode/vnode/src/tq/tqRestore.c +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -57,42 +57,28 @@ int32_t tqStreamTasksScanWal(STQ* pTq) { return 0; } -static SArray* extractTaskIdList(SStreamMeta* pStreamMeta, int32_t numOfTasks) { - SArray* pTaskIdList = taosArrayInit(numOfTasks, sizeof(int32_t)); - void* pIter = NULL; - - taosWLockLatch(&pStreamMeta->lock); - while(1) { - pIter = taosHashIterate(pStreamMeta->pTasks, pIter); - if (pIter == NULL) { - break; - } - - SStreamTask* pTask = *(SStreamTask**)pIter; - taosArrayPush(pTaskIdList, &pTask->id.taskId); - } - - taosWUnLockLatch(&pStreamMeta->lock); - return pTaskIdList; -} - int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = true; bool noNewDataInWal = true; int32_t vgId = pStreamMeta->vgId; - int32_t numOfTasks = taosHashGetSize(pStreamMeta->pTasks); + int32_t numOfTasks = taosArrayGetSize(pStreamMeta->pTaskList); if (numOfTasks == 0) { return TSDB_CODE_SUCCESS; } + SArray* pTaskList = NULL; + taosWLockLatch(&pStreamMeta->lock); + pTaskList = taosArrayDup(pStreamMeta->pTaskList, NULL); + taosWUnLockLatch(&pStreamMeta->lock); + tqDebug("vgId:%d start to check wal to extract new submit block for %d tasks", vgId, numOfTasks); - SArray* pTaskIdList = extractTaskIdList(pStreamMeta, numOfTasks); // update the new task number - numOfTasks = taosArrayGetSize(pTaskIdList); + numOfTasks = taosArrayGetSize(pTaskList); + for (int32_t i = 0; i < numOfTasks; ++i) { - int32_t* pTaskId = taosArrayGet(pTaskIdList, i); + int32_t* pTaskId = taosArrayGet(pTaskList, i); SStreamTask* pTask = streamMetaAcquireTask(pStreamMeta, *pTaskId); if (pTask == NULL) { continue; @@ -166,7 +152,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) { *pScanIdle = true; } - taosArrayDestroy(pTaskIdList); + taosArrayDestroy(pTaskList); return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 0d797f0bcb..aefe7885f9 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -57,6 +57,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } + // task list + pMeta->pTaskList = taosArrayInit(4, sizeof(int32_t)); + if (pMeta->pTaskList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + if (streamMetaBegin(pMeta) < 0) { goto _err; } @@ -70,6 +77,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF _err: taosMemoryFree(pMeta->path); if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks); + if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); if (pMeta->pTaskDb) tdbTbClose(pMeta->pTaskDb); if (pMeta->pCheckpointDb) tdbTbClose(pMeta->pCheckpointDb); if (pMeta->db) tdbClose(pMeta->db); @@ -100,6 +108,7 @@ void streamMetaClose(SStreamMeta* pMeta) { } taosHashCleanup(pMeta->pTasks); + pMeta->pTaskList = taosArrayDestroy(pMeta->pTaskList); taosMemoryFree(pMeta->path); taosMemoryFree(pMeta); } @@ -180,11 +189,15 @@ int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* } taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES); + taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); return 0; } int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) { - return (int32_t) taosHashGetSize(pMeta->pTasks); + size_t size = taosHashGetSize(pMeta->pTasks); + ASSERT(taosArrayGetSize(pMeta->pTaskList) == taosHashGetSize(pMeta->pTasks)); + + return (int32_t) size; } SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { @@ -224,6 +237,15 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__STOP); + int32_t num = taosArrayGetSize(pMeta->pTaskList); + for(int32_t i = 0; i < num; ++i) { + int32_t* pTaskId = taosArrayGet(pMeta->pTaskList, i); + if (*pTaskId == taskId) { + taosArrayRemove(pMeta->pTaskList, i); + break; + } + } + streamMetaReleaseTask(pMeta, pTask); taosWUnLockLatch(&pMeta->lock); } @@ -308,6 +330,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { return -1; } + taosArrayPush(pMeta->pTaskList, &pTask->id.taskId); + if (pTask->fillHistory) { pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; streamTaskCheckDownstream(pTask, ver); From 1e9bca79a420717d553931c8e9dbb9505d2e009a Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 5 May 2023 19:38:26 +0800 Subject: [PATCH 143/145] update crash gen push message --- tests/pytest/auto_crash_gen.py | 26 +++++++++---------- tests/pytest/auto_crash_gen_valgrind.py | 20 +++++++------- .../pytest/auto_crash_gen_valgrind_cluster.py | 24 ++++++++--------- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/tests/pytest/auto_crash_gen.py b/tests/pytest/auto_crash_gen.py index 56629ede13..5af2f055cd 100755 --- a/tests/pytest/auto_crash_gen.py +++ b/tests/pytest/auto_crash_gen.py @@ -1,3 +1,4 @@ +import datetime import os import socket import requests @@ -238,17 +239,7 @@ def start_taosd(): start_cmd = 'cd %s && python3 test.py >>/dev/null '%(start_path) os.system(start_cmd) -def get_cmds(args_list): - # build_path = get_path() - # if repo == "community": - # crash_gen_path = build_path[:-5]+"community/tests/pytest/" - # elif repo == "TDengine": - # crash_gen_path = build_path[:-5]+"/tests/pytest/" - # else: - # pass - - # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path) - +def get_cmds(args_list): crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) return crash_gen_cmd @@ -295,7 +286,7 @@ def check_status(): elif "Crash_Gen is now exiting with status code: 0" in run_code: return 0 else: - return 2 + return 2 def main(): @@ -310,7 +301,7 @@ def main(): build_path = get_path() - os.system("pip install git+https://github.com/taosdata/taos-connector-python.git") + if repo =="community": crash_gen_path = build_path[:-5]+"community/tests/pytest/" elif repo =="TDengine": @@ -334,7 +325,9 @@ def main(): if not os.path.exists(run_dir): os.mkdir(run_dir) print(crash_cmds) + starttime = datetime.datetime.now() run_crash_gen(crash_cmds) + endtime = datetime.datetime.now() status = check_status() print("exit status : ", status) @@ -349,7 +342,12 @@ def main(): print('======== crash_gen run sucess and exit as expected ========') try: - text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + text = f'''exit status: {msg_dict[status]} + git commit : {git_commit} + hostname: {hostname} + start time: {starttime} + end time: {endtime} + cmd: {crash_cmds}''' send_msg(get_msg(text)) except Exception as e: print("exception:", e) diff --git a/tests/pytest/auto_crash_gen_valgrind.py b/tests/pytest/auto_crash_gen_valgrind.py index 22fc5a480f..49e2c43f84 100755 --- a/tests/pytest/auto_crash_gen_valgrind.py +++ b/tests/pytest/auto_crash_gen_valgrind.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 +import datetime import os import socket import requests @@ -241,15 +242,6 @@ def start_taosd(): os.system(start_cmd +">>/dev/null") def get_cmds(args_list): - # build_path = get_path() - # if repo == "community": - # crash_gen_path = build_path[:-5]+"community/tests/pytest/" - # elif repo == "TDengine": - # crash_gen_path = build_path[:-5]+"/tests/pytest/" - # else: - # pass - - # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path) crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) return crash_gen_cmd @@ -343,7 +335,6 @@ def main(): args = limits(args) build_path = get_path() - os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null") if repo =="community": crash_gen_path = build_path[:-5]+"community/tests/pytest/" elif repo =="TDengine": @@ -368,7 +359,9 @@ def main(): if not os.path.exists(run_dir): os.mkdir(run_dir) print(crash_cmds) + starttime = datetime.datetime.now() run_crash_gen(crash_cmds) + endtime = datetime.datetime.now() status = check_status() # back_path = os.path.join(core_path,"valgrind_report") @@ -384,7 +377,12 @@ def main(): print('======== crash_gen run sucess and exit as expected ========') try: - text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + text = f'''exit status: {msg_dict[status]} + git commit : {git_commit} + hostname: {hostname} + start time: {starttime} + end time: {endtime} + cmd: {crash_cmds}''' send_msg(get_msg(text)) except Exception as e: print("exception:", e) diff --git a/tests/pytest/auto_crash_gen_valgrind_cluster.py b/tests/pytest/auto_crash_gen_valgrind_cluster.py index 547de9af47..5189ff4262 100755 --- a/tests/pytest/auto_crash_gen_valgrind_cluster.py +++ b/tests/pytest/auto_crash_gen_valgrind_cluster.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 +import datetime import os import socket import requests @@ -241,16 +242,7 @@ def start_taosd(): os.system(start_cmd +">>/dev/null") def get_cmds(args_list): - # build_path = get_path() - # if repo == "community": - # crash_gen_path = build_path[:-5]+"community/tests/pytest/" - # elif repo == "TDengine": - # crash_gen_path = build_path[:-5]+"/tests/pytest/" - # else: - # pass - - # crash_gen_cmd = 'cd %s && ./crash_gen.sh --valgrind -p -t 10 -s 1000 -g 0x32c,0x32d,0x3d3,0x18,0x2501,0x369,0x388,0x061a,0x2550 '%(crash_gen_path) - + crash_gen_cmd = get_auto_mix_cmds(args_list,valgrind=valgrind_mode) return crash_gen_cmd @@ -342,8 +334,7 @@ def main(): args = random_args(args_list) args = limits(args) - build_path = get_path() - os.system("pip install git+https://github.com/taosdata/taos-connector-python.git >>/dev/null") + build_path = get_path() if repo =="community": crash_gen_path = build_path[:-5]+"community/tests/pytest/" elif repo =="TDengine": @@ -368,7 +359,9 @@ def main(): if not os.path.exists(run_dir): os.mkdir(run_dir) print(crash_cmds) + starttime = datetime.datetime.now() run_crash_gen(crash_cmds) + endtime = datetime.datetime.now() status = check_status() # back_path = os.path.join(core_path,"valgrind_report") @@ -384,7 +377,12 @@ def main(): print('======== crash_gen run sucess and exit as expected ========') try: - text = f"crash_gen instance exit status of docker [ {hostname} ] is : {msg_dict[status]}\n " + f" and git commit : {git_commit}" + text = f'''exit status: {msg_dict[status]} + git commit : {git_commit} + hostname: {hostname} + start time: {starttime} + end time: {endtime} + cmd: {crash_cmds}''' send_msg(get_msg(text)) except Exception as e: print("exception:", e) From 50ba55561b3bcb879c5a3bf9b01b2ac60defd839 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 5 May 2023 22:11:59 +0800 Subject: [PATCH 144/145] fix(stream): set correct group id and table id. --- source/dnode/vnode/src/tq/tqRead.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 3aaf5cd60d..082e31ea91 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -565,6 +565,10 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbDa if (blockDataGetNumOfCols(pBlock) > 0) { blockDataDestroy(pReader->pResBlock); pReader->pResBlock = createDataBlock(); + pBlock = pReader->pResBlock; + + pBlock->info.id.uid = uid; + pBlock->info.version = pReader->msg.ver; } int32_t numOfCols = taosArrayGetSize(pReader->pColIdList); From ff6523fb02f72697d3b71a3342c3f9c16422eb2b Mon Sep 17 00:00:00 2001 From: Benguang Zhao Date: Sat, 6 May 2023 09:14:03 +0800 Subject: [PATCH 145/145] enh: declare mndSplitVgroup in mndVgroup.h --- source/dnode/mnode/impl/inc/mndVgroup.h | 2 ++ source/dnode/mnode/impl/src/mndVgroup.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 0229735952..94c4eae83f 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -50,6 +50,8 @@ void *mndBuildCreateVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *p void *mndBuildDropVnodeReq(SMnode *, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); bool mndVgroupInDb(SVgObj *pVgroup, int64_t dbUid); +int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index ed1fddb63f..0003d07fd6 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2006,7 +2006,7 @@ static int32_t mndAddAdjustVnodeHashRangeAction(SMnode *pMnode, STrans *pTrans, return 0; } -static int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { +int32_t mndSplitVgroup(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SVgObj *pVgroup) { int32_t code = -1; STrans *pTrans = NULL; SSdbRaw *pRaw = NULL;