From bd81d9b5fc8ea08dc86e5cfe1e033f794f490704 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 3 Aug 2022 21:08:52 +0800 Subject: [PATCH 01/32] enh: improve catalog performance --- include/common/tmsg.h | 2 + source/dnode/mnode/impl/src/mndQuery.c | 6 + source/dnode/vnode/src/vnd/vnodeQuery.c | 6 + source/libs/catalog/inc/catalogInt.h | 24 +++ source/libs/catalog/src/ctgAsync.c | 237 +++++++++++++++++------- source/libs/catalog/src/ctgCache.c | 54 ++++++ source/libs/catalog/src/ctgRemote.c | 81 +++++--- source/libs/catalog/src/ctgUtil.c | 57 ++++++ source/libs/qworker/src/qwMsg.c | 7 +- 9 files changed, 382 insertions(+), 92 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9bc2ae4268..7ebe7e465c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3053,6 +3053,7 @@ int32_t tEncodeDeleteRes(SEncoder* pCoder, const SDeleteRes* pRes); int32_t tDecodeDeleteRes(SDecoder* pCoder, SDeleteRes* pRes); typedef struct { + int32_t msgIdx; int32_t msgType; int32_t msgLen; void* msg; @@ -3066,6 +3067,7 @@ typedef struct { typedef struct { int32_t reqType; + int32_t msgIdx; int32_t msgLen; int32_t rspCode; void* msg; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index 2beeb10335..654f46ec85 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -84,6 +84,9 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + req.msgIdx = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); + offset += sizeof(req.msgIdx); + req.msgType = ntohl(*(int32_t*)((char*)pMsg->pCont + offset)); offset += sizeof(req.msgType); @@ -111,6 +114,7 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { } else { rsp.rspCode = 0; } + rsp.msgIdx = req.msgIdx; rsp.reqType = reqMsg.msgType; rsp.msgLen = reqMsg.info.rspLen; rsp.msg = reqMsg.info.rsp; @@ -136,6 +140,8 @@ int32_t mndProcessBatchMetaMsg(SRpcMsg *pMsg) { *(int32_t*)((char*)pRsp + offset) = htonl(p->reqType); offset += sizeof(p->reqType); + *(int32_t*)((char*)pRsp + offset) = htonl(p->msgIdx); + offset += sizeof(p->msgIdx); *(int32_t*)((char*)pRsp + offset) = htonl(p->msgLen); offset += sizeof(p->msgLen); *(int32_t*)((char*)pRsp + offset) = htonl(p->rspCode); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index d18ba88268..707c4bc471 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -273,6 +273,9 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { } for (int32_t i = 0; i < msgNum; ++i) { + req.msgIdx = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); + offset += sizeof(req.msgIdx); + req.msgType = ntohl(*(int32_t *)((char *)pMsg->pCont + offset)); offset += sizeof(req.msgType); @@ -301,6 +304,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { break; } + rsp.msgIdx = req.msgIdx; rsp.reqType = reqMsg.msgType; rsp.msgLen = reqMsg.contLen; rsp.rspCode = reqMsg.code; @@ -327,6 +331,8 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { *(int32_t *)((char *)pRsp + offset) = htonl(p->reqType); offset += sizeof(p->reqType); + *(int32_t *)((char *)pRsp + offset) = htonl(p->msgIdx); + offset += sizeof(p->msgIdx); *(int32_t *)((char *)pRsp + offset) = htonl(p->msgLen); offset += sizeof(p->msgLen); *(int32_t *)((char *)pRsp + offset) = htonl(p->rspCode); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 816309beb1..a4e47e8053 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -80,6 +80,7 @@ typedef enum { CTG_TASK_GET_UDF, CTG_TASK_GET_USER, CTG_TASK_GET_SVR_VER, + CTG_TASK_GET_TB_META_BATCH, } CTG_TASK_TYPE; typedef enum { @@ -110,6 +111,22 @@ typedef struct SCtgTbMetaCtx { int32_t flag; } SCtgTbMetaCtx; +typedef struct SCtgFetch { + int32_t reqIdx; + int32_t fetchIdx; + int32_t flag; + SCtgTbCacheInfo tbInfo; + int32_t vgId; +} SCtgFetch; + +typedef struct SCtgTbMetaBCtx { + int32_t fetchNum; + SArray* pNames; + SArray* pTbMetas; + SArray* pFetchs; +} SCtgTbMetaBCtx; + + typedef struct SCtgTbIndexCtx { SName* pName; } SCtgTbIndexCtx; @@ -218,6 +235,7 @@ typedef struct SCtgJob { int32_t batchId; SHashObj* pBatchs; SArray* pTasks; + int32_t subTaskNum; int32_t taskDone; SMetaData jobRes; int32_t jobResCode; @@ -276,6 +294,7 @@ typedef struct SCtgTask { int32_t taskId; SCtgJob* pJob; void* taskCtx; + SArray* msgCtxs; SCtgMsgCtx msgCtx; int32_t code; void* res; @@ -284,6 +303,7 @@ typedef struct SCtgTask { SArray* pParents; SCtgSubRes subRes; SHashObj* pBatchs; + int32_t msgIdx; } SCtgTask; typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*); @@ -487,6 +507,8 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) +#define CTG_GET_TASK_MSGCTX(_task) ((CTG_TASK_GET_TB_META_BATCH == (_task)->type) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) + #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) #define CTG_TABLE_NOT_EXIST(code) (code == CTG_ERR_CODE_TABLE_NOT_EXIST) @@ -586,6 +608,7 @@ int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas); int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); @@ -672,6 +695,7 @@ int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2); int32_t ctgDbVgVersionSearchCompare(const void* key1, const void* key2); void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput); int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target); +int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); char * ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 0184ac3a60..8aa33b759a 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -50,6 +50,31 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_META_BATCH; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetaBCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbMetaBCtx* ctx = task.taskCtx; + ctx->pNames = param; + ctx->pTbMetas = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { char *dbFName = (char*)param; SCtgTask task = {0}; @@ -328,7 +353,6 @@ int32_t ctgInitGetTbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { } - int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (NULL == pDb) { @@ -452,7 +476,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const } SCtgJob *pJob = *job; - + + pJob->subTaskNum = taskNum; pJob->queryId = pConn->requestId; pJob->userFp = fp; pJob->pCtg = pCtg; @@ -506,10 +531,16 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_INFO, dbFName, NULL)); } +#if 0 for (int32_t i = 0; i < tbMetaNum; ++i) { SName* name = taosArrayGet(pReq->pTableMeta, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_META, name, NULL)); } +#else + if (tbMetaNum > 0) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_META_BATCH, pReq->pTableMeta, NULL)); + } +#endif for (int32_t i = 0; i < tbHashNum; ++i) { SName* name = taosArrayGet(pReq->pTableHash, i); @@ -586,6 +617,15 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbMetaBRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + pJob->jobRes.pTableMeta = pTask->res; + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgDumpDbVgRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbVgroup) { @@ -847,43 +887,55 @@ _return: int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDBCache *dbCache = NULL; - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); +#if CTG_BATCH_FETCH + SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); + int32_t flag = pFetch->flag; + int32_t* vgId = &pFetch->vgId; +#else + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + SName* pName = ctx->pName; + int32_t flag = ctx->flag; + int32_t* vgId = &ctx->vgId; +#endif - CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); switch (reqType) { case TDMT_MND_USE_DB: { - SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); return TSDB_CODE_SUCCESS; } case TDMT_MND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; if (CTG_IS_META_NULL(pOut->metaType)) { - if (CTG_FLAG_IS_STB(ctx->flag)) { + if (CTG_FLAG_IS_STB(flag)) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(ctx->pName, dbFName); + tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (NULL != dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); ctgReleaseVgInfoToCache(pCtg, dbCache); } else { @@ -898,52 +950,58 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * return TSDB_CODE_SUCCESS; } - ctgError("no tbmeta got, tbName:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - if (pTask->msgCtx.lastOut) { - TSWAP(pTask->msgCtx.out, pTask->msgCtx.lastOut); - STableMetaOutput* pLastOut = (STableMetaOutput*)pTask->msgCtx.out; + if (pMsgCtx->lastOut) { + TSWAP(pMsgCtx->out, pMsgCtx->lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pMsgCtx->out; TSWAP(pLastOut->tbMeta, pOut->tbMeta); } break; } case TDMT_VND_TABLE_META: { - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; if (CTG_IS_META_NULL(pOut->metaType)) { - ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(ctx->pName)); - ctgRemoveTbMetaFromCache(pCtg, ctx->pName, false); + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); } - if (CTG_FLAG_IS_STB(ctx->flag)) { + if (CTG_FLAG_IS_STB(flag)) { break; } if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { - ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(ctx->pName)); + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); taosMemoryFreeClear(pOut->tbMeta); - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, ctx->pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); } else if (CTG_IS_META_BOTH(pOut->metaType)) { int32_t exist = 0; - if (!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) { - CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pOut->dbFName, pOut->tbName, &exist)); + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + taosMemoryFreeClear(pOut->tbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (pOut->tbMeta) { + exist = 1; + } } if (0 == exist) { - TSWAP(pTask->msgCtx.lastOut, pTask->msgCtx.out); + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); - } else { - taosMemoryFreeClear(pOut->tbMeta); - - SET_META_TYPE_CTABLE(pOut->metaType); } } break; @@ -954,17 +1012,20 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * break; } - STableMetaOutput* pOut = (STableMetaOutput*)pTask->msgCtx.out; + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; ctgUpdateTbMetaToCache(pCtg, pOut, false); if (CTG_IS_META_BOTH(pOut->metaType)) { memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); - } else if (CTG_IS_META_CTABLE(pOut->metaType)) { - SName stbName = *ctx->pName; + } + +/* + else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *pName; strcpy(stbName.tname, pOut->tbName); SCtgTbMetaCtx stbCtx = {0}; - stbCtx.flag = ctx->flag; + stbCtx.flag = flag; stbCtx.pName = &stbName; CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); @@ -977,8 +1038,19 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); } +*/ +#if CTG_BATCH_FETCH + SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + pRes->code = 0; + pRes->pRes = pOut->tbMeta; + pOut->tbMeta = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pTbMetas); + } +#else TSWAP(pTask->res, pOut->tbMeta); +#endif _return: @@ -986,8 +1058,10 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } - ctgHandleTaskEnd(pTask, code); - + if (pTask->res || code) { + ctgHandleTaskEnd(pTask, code); + } + CTG_RET(code); } @@ -1224,38 +1298,37 @@ _return: CTG_RET(code); } -int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask) { +int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32_t* vgId) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; int32_t code = 0; - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - if (CTG_FLAG_IS_SYS_DB(ctx->flag)) { - ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(ctx->pName)); + if (CTG_FLAG_IS_SYS_DB(flag)) { + ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(pName)); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)ctx->pName->dbname, (char *)ctx->pName->tname, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)pName->dbname, (char *)pName->tname, NULL, pTask)); } - if (CTG_FLAG_IS_STB(ctx->flag)) { - ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(ctx->pName)); + if (CTG_FLAG_IS_STB(flag)) { + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pName)); // if get from mnode failed, will not try vnode - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, ctx->pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); } SCtgDBCache *dbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(ctx->pName, dbFName); + tNameGetFullDbName(pName, dbFName); CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); if (dbCache) { SVgroupInfo vgInfo = {0}; - CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, ctx->pName, &vgInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); - ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(ctx->pName), ctx->flag); + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); - ctx->vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, ctx->pName, &vgInfo, NULL, pTask)); + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); } else { SBuildUseDBInput input = {0}; @@ -1284,11 +1357,40 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask)); + SCtgTbMetaCtx* pCtx = (SCtgTbMetaCtx*)pTask->taskCtx; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pCtx->flag, pCtx->pName, &pCtx->vgId)); return TSDB_CODE_SUCCESS; } +int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + + CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, (SCtgTbMetaBCtx*)pTask->taskCtx, (SArray**)&pTask->res)); + if (pTask->res) { + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + + pTask->msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgAddMsgCtx(pTask->msgCtxs, 0, NULL, NULL)); + CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); + } + + pTask->msgIdx = 0; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1597,19 +1699,20 @@ int32_t ctgCloneDbVg(SCtgTask* pTask, void** pRes) { SCtgAsyncFps gCtgAsyncFps[] = { - {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, - {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, - {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, - {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, - {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, - {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, ctgCloneTbMeta}, - {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, - {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, - {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, - {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, - {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, - {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, - {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, + {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, + {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, + {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, + {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, + {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, + {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, ctgCloneTbMeta}, + {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, + {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, + {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, + {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, + {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, + {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, + {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, + {ctgInitGetTbMetaBTask, ctgLaunchGetTbMetaBTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaBRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 2e8e259151..1b5968d426 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2152,6 +2152,60 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { + int32_t tbNum = taosArrayGetSize(ctx->pNames); + SName* fName = taosArrayGet(ctx->pNames, 0); + int32_t fIdx = 0; + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pName = taosArrayGet(ctx->pNames, i); + SCtgTbMetaCtx nctx = {0}; + nctx.flag = CTG_FLAG_UNKNOWN_STB; + nctx.pName = pName; + + if (IS_SYS_DBNAME(pName->dbname)) { + CTG_FLAG_SET_SYS_DB(nctx.flag); + } + + STableMeta *pTableMeta = NULL; + CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, &nctx, &pTableMeta)); + SMetaRes res = {0}; + + if (pTableMeta) { + if (CTG_FLAG_MATCH_STB(nctx.flag, pTableMeta->tableType) && + ((!CTG_FLAG_IS_FORCE_UPDATE(nctx.flag)) || (CTG_FLAG_IS_SYS_DB(nctx.flag)))) { + res.pRes = pTableMeta; + } else { + taosMemoryFreeClear(pTableMeta); + } + } + + if (NULL == res.pRes) { + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + if (CTG_FLAG_IS_UNKNOWN_STB(nctx.flag)) { + CTG_FLAG_SET_STB(nctx.flag, nctx.tbInfo.tbType); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = nctx.flag; + + taosArrayPush(ctx->pFetchs, &fetch); + } + + taosArrayPush(ctx->pTbMetas, &res); + } + + if (NULL == ctx->pFetchs) { + TSWAP(*pTbMetas, ctx->pTbMetas); + } + + return TSDB_CODE_SUCCESS; +} int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { int32_t code = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 0f97b5c5b1..d489496e37 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -46,6 +46,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu if (msgNum > 0) { rsp.reqType = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.reqType); + rsp.msgIdx = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); + offset += sizeof(rsp.msgIdx); rsp.msgLen = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); offset += sizeof(rsp.msgLen); rsp.rspCode = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); @@ -57,6 +59,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu taskMsg.pData = rsp.msg; taskMsg.len = rsp.msgLen; } else { + rsp.msgIdx = pTask->msgIdx++; rsp.reqType = -1; taskMsg.msgType = -1; taskMsg.pData = NULL; @@ -64,6 +67,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } pTask->pBatchs = pBatchs; + pTask->msgIdx = rsp.msgIdx; ctgDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(taskMsg.msgType + 1)); @@ -431,19 +435,19 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT SHashObj* pBatchs = pTask->pBatchs; SCtgJob* pJob = pTask->pJob; SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); - int32_t taskNum = taosArrayGetSize(pTask->pJob->pTasks); SCtgBatch newBatch = {0}; SBatchMsg req = {0}; if (NULL == pBatch) { - newBatch.pMsgs = taosArrayInit(taskNum, sizeof(SBatchMsg)); - newBatch.pTaskIds = taosArrayInit(taskNum, sizeof(int32_t)); + newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); + newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } newBatch.conn = *pConn; + req.msgIdx = pTask->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -456,16 +460,25 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT newBatch.msgSize = sizeof(SBatchReq) + sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { + SName* pName = NULL; if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { + SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + } else { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + pName = ctx->pName; + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } + + tNameGetFullDbName(pName, newBatch.dbFName); } newBatch.msgType = (vgId > 0) ? TDMT_VND_BATCH_META : TDMT_MND_BATCH_META; @@ -480,6 +493,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT return TSDB_CODE_SUCCESS; } + req.msgIdx = pTask->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -492,16 +506,25 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT pBatch->msgSize += sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { + SName* pName = NULL; if (TDMT_VND_TABLE_CFG == msgType) { SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { - SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; - tNameGetFullDbName(ctx->pName, newBatch.dbFName); + if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { + SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + } else { + SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; + pName = ctx->pName; + } } else { ctgError("invalid vnode msgType %d", msgType); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } + + tNameGetFullDbName(pName, newBatch.dbFName); } ctgDebug("task %d %s req added to batch %d, target vgId %d", pTask->taskId, TMSG_INFO(msgType), pBatch->batchId, vgId); @@ -517,7 +540,7 @@ _return: } int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { - *msg = taosMemoryMalloc(pBatch->msgSize); + *msg = taosMemoryCalloc(1, pBatch->msgSize); if (NULL == (*msg)) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -532,6 +555,8 @@ int32_t ctgBuildBatchReqMsg(SCtgBatch* pBatch, int32_t vgId, void** msg) { for (int32_t i = 0; i < num; ++i) { SBatchMsg* pReq = taosArrayGet(pBatch->pMsgs, i); + *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgIdx); + offset += sizeof(pReq->msgIdx); *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgType); offset += sizeof(pReq->msgType); *(int32_t*)((char*)(*msg) + offset) = htonl(pReq->msgLen); @@ -599,7 +624,8 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, NULL)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, NULL)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -645,7 +671,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -696,7 +722,8 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildU if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, input->db)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, input->db)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -746,7 +773,8 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)dbFName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)dbFName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -796,7 +824,8 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)indexName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)indexName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -849,7 +878,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *n CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)tbFName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -899,7 +928,8 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const ch if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)funcName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)funcName)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -949,7 +979,8 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, (char*)user)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)user)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); @@ -1004,7 +1035,9 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); + + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); #else @@ -1068,13 +1101,14 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa if (NULL == pOut) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, pOut, tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); #else @@ -1129,7 +1163,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, @@ -1188,7 +1222,8 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); + #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); #else @@ -1233,7 +1268,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **ou } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); #if CTG_BATCH_FETCH CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index bdf60b110b..9f0d0f71d5 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -88,6 +88,8 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get user]"; case CTG_TASK_GET_SVR_VER: return "[get svr ver]"; + case CTG_TASK_GET_TB_META_BATCH: + return "[bget table meta]"; default: return "unknown"; } @@ -460,6 +462,15 @@ void ctgResetTbMetaTask(SCtgTask* pTask) { taosMemoryFreeClear(pTask->res); } +void ctgFreeBatchMeta(void* meta) { + if (NULL == meta) { + return; + } + + SMetaRes* pRes = (SMetaRes*)meta; + taosMemoryFreeClear(pRes->pRes); +} + void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { switch (type) { case CTG_TASK_GET_QNODE: @@ -500,6 +511,15 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_META_BATCH: { + SArray* pArray = (SArray*)*pRes; + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + ctgFreeBatchMeta(taosArrayGet(pArray, i)); + } + *pRes = NULL; // no need to free it + break; + } default: qError("invalid task type %d", type); break; @@ -554,6 +574,11 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { taosMemoryFreeClear(*pRes); break; } + case CTG_TASK_GET_TB_META_BATCH: { + taosArrayDestroyEx(*pRes, ctgFreeBatchMeta); + *pRes = NULL; + break; + } default: qError("invalid task type %d", type); break; @@ -583,6 +608,21 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_META_BATCH: { + SCtgTbMetaBCtx* taskCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + taosArrayDestroyEx(taskCtx->pTbMetas, ctgFreeBatchMeta); + taosArrayDestroy(taskCtx->pFetchs); + // NO NEED TO FREE pNames + + taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx); + + if (pTask->msgCtx.lastOut) { + ctgFreeSTableMetaOutput((STableMetaOutput*)pTask->msgCtx.lastOut); + pTask->msgCtx.lastOut = NULL; + } + taosMemoryFreeClear(pTask->taskCtx); + break; + } case CTG_TASK_GET_TB_HASH: { SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); @@ -679,6 +719,23 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ return TSDB_CODE_SUCCESS; } +int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target) { + SCtgMsgCtx ctx = {0}; + + ctx.reqType = reqType; + ctx.out = out; + if (target) { + ctx.target = strdup(target); + if (NULL == ctx.target) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + taosArrayPush(pCtxs, &ctx); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetHashFunction(int8_t hashMethod, tableNameHashFp *fp) { switch (hashMethod) { diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index e8ffd98153..6bf2529551 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -387,10 +387,13 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); - QW_ERR_RET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); + QW_ERR_JRET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); QW_SCH_TASK_DLOG("processQuery end, node:%p", node); - return TSDB_CODE_SUCCESS; +_return: + + taosMemoryFree(sql); + return code; } int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int64_t ts) { From bdee43e3cc8ba8e11d6fab21bdbf6f7536c48098 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 09:01:35 +0800 Subject: [PATCH 02/32] fix: fix crash issue --- source/libs/catalog/src/ctgAsync.c | 13 ++++++++++++- source/libs/qworker/src/qwMsg.c | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 8aa33b759a..8dc32d306a 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1058,7 +1058,18 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } - if (pTask->res || code) { +#if CTG_BATCH_FETCH + if (code) { + SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + pRes->code = code; + pRes->pRes = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pTbMetas); + } + } +#endif + + if (pTask->res) { ctgHandleTaskEnd(pTask, code); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index 6bf2529551..2348f1ef21 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -388,11 +388,11 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int char * sql = strndup(msg->msg, msg->sqlLen); QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), pMsg->info.handle, sql); QW_ERR_JRET(qwProcessQuery(QW_FPARAMS(), &qwMsg, sql)); - QW_SCH_TASK_DLOG("processQuery end, node:%p", node); _return: - taosMemoryFree(sql); + QW_SCH_TASK_DLOG("processQuery end, node:%p, code:%d", node, code); + return code; } From bc7698366a5b712b93a71208dae8e3f4d8ced3c5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 11:11:26 +0800 Subject: [PATCH 03/32] enh: launch tables in same db --- source/libs/catalog/src/ctgCache.c | 273 +++++++++++++++++++++++++++- source/libs/catalog/src/ctgRemote.c | 2 +- source/libs/qcom/src/queryUtil.c | 2 + 3 files changed, 274 insertions(+), 3 deletions(-) diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1b5968d426..6cec690bad 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -242,7 +242,6 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S goto _return; } - int32_t sz = 0; pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); @@ -282,7 +281,6 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, goto _return; } - int32_t sz = 0; char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); @@ -2152,6 +2150,7 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet return TSDB_CODE_SUCCESS; } +#if 0 int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { int32_t tbNum = taosArrayGetSize(ctx->pNames); SName* fName = taosArrayGet(ctx->pNames, 0); @@ -2206,6 +2205,276 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe return TSDB_CODE_SUCCESS; } +#endif + +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { + int32_t tbNum = taosArrayGetSize(ctx->pNames); + int32_t fIdx = 0; + SName* pName = taosArrayGet(ctx->pNames, 0); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + int32_t flag = CTG_FLAG_UNKNOWN_STB; + uint64_t lastSuid = 0; + STableMeta* lastTableMeta = NULL; + + if (IS_SYS_DBNAME(pName->dbname)) { + CTG_FLAG_SET_SYS_DB(flag); + strcpy(dbFName, pName->dbname); + } else { + tNameGetFullDbName(pName, dbFName); + } + + SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; + ctgAcquireDBCache(pCtg, dbFName, &dbCache); + + if (NULL == dbCache) { + ctgDebug("db %s not in cache", dbFName); + for (int32_t i = 0; i < tbNum; ++i) { + SMetaRes res = {0}; + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + } + + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < tbNum; ++i) { + SName* pName = taosArrayGet(ctx->pNames, i); + SMetaRes res = {0}; + + pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); + if (NULL == pCache) { + ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + STableMeta* tbMeta = pCache->pMeta; + + SCtgTbMetaCtx nctx = {0}; + nctx.flag = flag; + nctx.tbInfo.inCache = true; + nctx.tbInfo.dbId = dbCache->dbId; + nctx.tbInfo.suid = tbMeta->suid; + nctx.tbInfo.tbType = tbMeta->tableType; + + STableMeta* pTableMeta = NULL; + if (tbMeta->tableType != TSDB_CHILD_TABLE) { + int32_t metaSize = CTG_META_SIZE(tbMeta); + pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == pTableMeta) { + //ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTableMeta, tbMeta, metaSize); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); + + res.pRes = pTableMeta; + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + // PROCESS FOR CHILD TABLE + + if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { + cloneTableMeta(lastTableMeta, &pTableMeta); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); + + res.pRes = pTableMeta; + taosArrayPush(ctx->pTbMetas, &res); + + continue; + } + + int32_t metaSize = sizeof(SCTableMeta); + pTableMeta = taosMemoryCalloc(1, metaSize); + if (NULL == pTableMeta) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(pTableMeta, tbMeta, metaSize); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", + pName->tname, nctx.tbInfo.tbType, dbFName); + + char* stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); + if (NULL == stName) { + ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + continue; + } + + pCache = taosHashAcquire(dbCache->tbCache, stName, strlen(stName)); + if (NULL == pCache) { + ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); + taosHashRelease(dbCache->stbCache, stName); + + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + continue; + } + + taosHashRelease(dbCache->stbCache, stName); + + CTG_LOCK(CTG_READ, &pCache->metaLock); + if (NULL == pCache->pMeta) { + ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", pTableMeta->suid, dbFName); + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + + continue; + } + + STableMeta* stbMeta = pCache->pMeta; + if (stbMeta->suid != nctx.tbInfo.suid) { + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, nctx.tbInfo.suid); + + if (NULL == ctx->pFetchs) { + ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.flag = flag; + + taosArrayPush(ctx->pFetchs, &fetch); + taosArrayPush(ctx->pTbMetas, &res); + + taosMemoryFreeClear(pTableMeta); + + continue; + } + + metaSize = CTG_META_SIZE(stbMeta); + pTableMeta = taosMemoryRealloc(pTableMeta, metaSize); + if (NULL == pTableMeta) { + //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); + + if (pCache) { + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); + } + + res.pRes = pTableMeta; + taosArrayPush(ctx->pTbMetas, &res); + + lastSuid = pTableMeta->suid; + lastTableMeta = pTableMeta; + } + + if (NULL == ctx->pFetchs) { + TSWAP(*pTbMetas, ctx->pTbMetas); + } + + return TSDB_CODE_SUCCESS; +} + int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { int32_t code = 0; diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index d489496e37..a8d5a7ae95 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -69,7 +69,7 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu pTask->pBatchs = pBatchs; pTask->msgIdx = rsp.msgIdx; - ctgDebug("QID:0x%" PRIx64 " ctg task %d start to handle rsp %s", pJob->queryId, pTask->taskId, TMSG_INFO(taskMsg.msgType + 1)); + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, pTask->msgIdx, TMSG_INFO(taskMsg.msgType + 1)); (*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index d8fda57791..4cad6a078b 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -462,3 +462,5 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } + + From d8d6cacd0f71297461fb4ac42d49fc079cdbcb38 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 4 Aug 2022 16:23:45 +0800 Subject: [PATCH 04/32] enh: improve getting table hash performance --- source/libs/catalog/inc/catalogInt.h | 18 ++- source/libs/catalog/src/ctgAsync.c | 205 +++++++++++++++++++++++++-- source/libs/catalog/src/ctgCache.c | 39 +++-- source/libs/catalog/src/ctgUtil.c | 136 +++++++++++++++++- 4 files changed, 360 insertions(+), 38 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index a4e47e8053..e12bcfd18f 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -81,6 +81,7 @@ typedef enum { CTG_TASK_GET_USER, CTG_TASK_GET_SVR_VER, CTG_TASK_GET_TB_META_BATCH, + CTG_TASK_GET_TB_HASH_BATCH, } CTG_TASK_TYPE; typedef enum { @@ -114,6 +115,7 @@ typedef struct SCtgTbMetaCtx { typedef struct SCtgFetch { int32_t reqIdx; int32_t fetchIdx; + int32_t resIdx; int32_t flag; SCtgTbCacheInfo tbInfo; int32_t vgId; @@ -122,11 +124,10 @@ typedef struct SCtgFetch { typedef struct SCtgTbMetaBCtx { int32_t fetchNum; SArray* pNames; - SArray* pTbMetas; + SArray* pResList; SArray* pFetchs; } SCtgTbMetaBCtx; - typedef struct SCtgTbIndexCtx { SName* pName; } SCtgTbIndexCtx; @@ -154,6 +155,14 @@ typedef struct SCtgTbHashCtx { SName* pName; } SCtgTbHashCtx; +typedef struct SCtgTbHashBCtx { + int32_t fetchNum; + SArray* pNames; + SArray* pResList; + SArray* pFetchs; +} SCtgTbHashBCtx; + + typedef struct SCtgIndexCtx { char indexFName[TSDB_INDEX_FNAME_LEN]; } SCtgIndexCtx; @@ -507,7 +516,7 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_GET_TASK_MSGCTX(_task) ((CTG_TASK_GET_TB_META_BATCH == (_task)->type) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) +#define CTG_GET_TASK_MSGCTX(_task) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) @@ -608,7 +617,7 @@ int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas); +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList); int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); @@ -687,6 +696,7 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); +int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 8dc32d306a..eed4a956b5 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -65,7 +65,7 @@ int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SCtgTbMetaBCtx* ctx = task.taskCtx; ctx->pNames = param; - ctx->pTbMetas = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); taosArrayPush(pJob->pTasks, &task); @@ -74,7 +74,6 @@ int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } - int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { char *dbFName = (char*)param; SCtgTask task = {0}; @@ -178,6 +177,31 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbHashBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_HASH_BATCH; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashBCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbHashBCtx* ctx = task.taskCtx; + ctx->pNames = param; + ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SCtgTask task = {0}; @@ -542,10 +566,16 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const } #endif +#if 0 for (int32_t i = 0; i < tbHashNum; ++i) { SName* name = taosArrayGet(pReq->pTableHash, i); CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_HASH, name, NULL)); } +#else + if (tbHashNum > 0) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_HASH_BATCH, pReq->pTableHash, NULL)); + } +#endif for (int32_t i = 0; i < tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); @@ -656,6 +686,14 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbHashBRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + + pJob->jobRes.pTableHash = pTask->res; + + return TSDB_CODE_SUCCESS; +} + int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pTableIndex) { @@ -1041,12 +1079,12 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * */ #if CTG_BATCH_FETCH - SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pTbMetas); + TSWAP(pTask->res, ctx->pResList); } #else TSWAP(pTask->res, pOut->tbMeta); @@ -1060,11 +1098,11 @@ _return: #if CTG_BATCH_FETCH if (code) { - SMetaRes* pRes = taosArrayGet(ctx->pTbMetas, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); pRes->code = code; pRes->pRes = NULL; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pTbMetas); + TSWAP(pTask->res, ctx->pResList); } } #endif @@ -1146,6 +1184,62 @@ _return: CTG_RET(code); } +int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgTbHashBCtx* ctx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCatalog* pCtg = pTask->pJob->pCtg; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, true)); + + CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); + pOut->dbVgroup = NULL; + + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + +_return: + + if (code) { + SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); // TODO + + SMetaRes res = {0}; + int32_t num = taosArrayGetSize(ctx->pNames); + for (int32_t i = 0; i < num; ++i) { + SMetaRes *pRes = taosArrayGet(ctx->pResList, i); + pRes->code = code; + pRes->pRes = NULL; + } + + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + } + + if (pTask->res) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1377,23 +1471,25 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; - CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, (SCtgTbMetaBCtx*)pTask->taskCtx, (SArray**)&pTask->res)); - if (pTask->res) { + CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, pCtx, pCtx->pNames)); + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + if (pCtx->fetchNum <= 0) { + TSWAP(pTask->res, pCtx->pResList); + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } - - SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; - pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + taosArraySetSize(pTask->msgCtxs, pCtx->fetchNum); for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); pTask->msgIdx = pFetch->fetchIdx; - CTG_ERR_RET(ctgAddMsgCtx(pTask->msgCtxs, 0, NULL, NULL)); CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); } @@ -1472,6 +1568,90 @@ _return: CTG_RET(code); } +int32_t ctgLaunchGetTbHashBTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbHashBCtx* pCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgDBCache *dbCache = NULL; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + int32_t dbNum = 1; + int32_t fIdx = 0; + int32_t tbNum = 0; + int32_t code = 0; + + for (int32_t i = 0; i < dbNum; ++i) { // TODO + SName* pName = taosArrayGet(pCtx->pNames, 0); + if (IS_SYS_DBNAME(pName->dbname)) { + strcpy(dbFName, pName->dbname); + } else { + tNameGetFullDbName(pName, dbFName); + } + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + + if (NULL != dbCache) { + CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, dbFName, false)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + dbCache = NULL; + } else { + if (NULL == pCtx->pFetchs) { + pCtx->pFetchs = taosArrayInit(dbNum, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.reqIdx = i; + fetch.fetchIdx = fIdx++; + fetch.resIdx = tbNum; + + tbNum += taosArrayGetSize(pCtx->pNames); // TODO + + taosArrayPush(pCtx->pFetchs, &fetch); + taosArraySetSize(pCtx->pResList, tbNum); + } + } + + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); + if (pCtx->fetchNum <= 0) { + TSWAP(pTask->res, pCtx->pResList); + + CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); + return TSDB_CODE_SUCCESS; + } + + pTask->msgCtxs = taosArrayInit(pCtx->fetchNum, sizeof(SCtgMsgCtx)); + taosArraySetSize(pTask->msgCtxs, pCtx->fetchNum); + + for (int32_t i = 0; i < pCtx->fetchNum; ++i) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); + SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + + pTask->msgIdx = pFetch->fetchIdx; + + SBuildUseDBInput input = {0}; + if (IS_SYS_DBNAME(pName->dbname)) { + strcpy(input.db, pName->dbname); + } else { + tNameGetFullDbName(pName, input.db); + } + + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + } + + pTask->msgIdx = 0; + +_return: + + if (dbCache) { + ctgReleaseVgInfoToCache(pCtg, dbCache); + } + + return code; +} + + int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1724,6 +1904,7 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, {ctgInitGetTbMetaBTask, ctgLaunchGetTbMetaBTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaBRes, NULL, NULL}, + {ctgInitGetTbHashBTask, ctgLaunchGetTbHashBTask, ctgHandleGetTbHashBRsp, ctgDumpTbHashBRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 6cec690bad..1a28ff3e25 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2151,7 +2151,7 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet } #if 0 -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pResList) { int32_t tbNum = taosArrayGetSize(ctx->pNames); SName* fName = taosArrayGet(ctx->pNames, 0); int32_t fIdx = 0; @@ -2196,21 +2196,21 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe taosArrayPush(ctx->pFetchs, &fetch); } - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); } if (NULL == ctx->pFetchs) { - TSWAP(*pTbMetas, ctx->pTbMetas); + TSWAP(*pResList, ctx->pResList); } return TSDB_CODE_SUCCESS; } #endif -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pTbMetas) { - int32_t tbNum = taosArrayGetSize(ctx->pNames); +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList) { + int32_t tbNum = taosArrayGetSize(pList); int32_t fIdx = 0; - SName* pName = taosArrayGet(ctx->pNames, 0); + SName* pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; uint64_t lastSuid = 0; @@ -2241,14 +2241,14 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); } return TSDB_CODE_SUCCESS; } for (int32_t i = 0; i < tbNum; ++i) { - SName* pName = taosArrayGet(ctx->pNames, i); + SName* pName = taosArrayGet(pList, i); SMetaRes res = {0}; pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); @@ -2264,7 +2264,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2282,7 +2282,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2315,7 +2315,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2324,6 +2324,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { cloneTableMeta(lastTableMeta, &pTableMeta); + memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); if (pCache) { CTG_UNLOCK(CTG_READ, &pCache->metaLock); @@ -2333,7 +2334,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); res.pRes = pTableMeta; - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); continue; } @@ -2367,7 +2368,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); continue; @@ -2388,7 +2389,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); continue; @@ -2414,7 +2415,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); @@ -2440,7 +2441,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe fetch.flag = flag; taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); taosMemoryFreeClear(pTableMeta); @@ -2462,16 +2463,12 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } res.pRes = pTableMeta; - taosArrayPush(ctx->pTbMetas, &res); + taosArrayPush(ctx->pResList, &res); lastSuid = pTableMeta->suid; lastTableMeta = pTableMeta; } - if (NULL == ctx->pFetchs) { - TSWAP(*pTbMetas, ctx->pTbMetas); - } - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 9f0d0f71d5..c841255b42 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -90,6 +90,8 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get svr ver]"; case CTG_TASK_GET_TB_META_BATCH: return "[bget table meta]"; + case CTG_TASK_GET_TB_HASH_BATCH: + return "[bget table hash]"; default: return "unknown"; } @@ -471,6 +473,16 @@ void ctgFreeBatchMeta(void* meta) { taosMemoryFreeClear(pRes->pRes); } +void ctgFreeBatchHash(void* hash) { + if (NULL == hash) { + return; + } + + SMetaRes* pRes = (SMetaRes*)hash; + taosMemoryFreeClear(pRes->pRes); +} + + void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { switch (type) { case CTG_TASK_GET_QNODE: @@ -520,6 +532,15 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { *pRes = NULL; // no need to free it break; } + case CTG_TASK_GET_TB_HASH_BATCH: { + SArray* pArray = (SArray*)*pRes; + int32_t num = taosArrayGetSize(pArray); + for (int32_t i = 0; i < num; ++i) { + ctgFreeBatchHash(taosArrayGet(pArray, i)); + } + *pRes = NULL; // no need to free it + break; + } default: qError("invalid task type %d", type); break; @@ -579,6 +600,11 @@ void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { *pRes = NULL; break; } + case CTG_TASK_GET_TB_HASH_BATCH: { + taosArrayDestroyEx(*pRes, ctgFreeBatchHash); + *pRes = NULL; + break; + } default: qError("invalid task type %d", type); break; @@ -610,7 +636,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { } case CTG_TASK_GET_TB_META_BATCH: { SCtgTbMetaBCtx* taskCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; - taosArrayDestroyEx(taskCtx->pTbMetas, ctgFreeBatchMeta); + taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchMeta); taosArrayDestroy(taskCtx->pFetchs); // NO NEED TO FREE pNames @@ -629,6 +655,17 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { taosMemoryFreeClear(pTask->taskCtx); break; } + case CTG_TASK_GET_TB_HASH_BATCH: { + SCtgTbHashBCtx* taskCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchHash); + taosArrayDestroy(taskCtx->pFetchs); + // NO NEED TO FREE pNames + + taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx); + + taosMemoryFreeClear(pTask->taskCtx); + break; + } case CTG_TASK_GET_TB_INDEX: { SCtgTbIndexCtx* taskCtx = (SCtgTbIndexCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); @@ -837,6 +874,103 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } +int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update) { + int32_t code = 0; + SMetaRes res = {0}; + int32_t vgNum = taosHashGetSize(dbInfo->vgHash); + if (vgNum <= 0) { + ctgError("db vgroup cache invalid, db:%s, vgroup number:%d", dbFName, vgNum); + CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); + } + + tableNameHashFp fp = NULL; + SVgroupInfo *vgInfo = NULL; + + CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); + + int32_t tbNum = taosArrayGetSize(pCtx->pNames); + + if (1 == vgNum) { + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + for (int32_t i = 0; i < tbNum; ++i) { + vgInfo = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == vgInfo) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + *vgInfo = *(SVgroupInfo*)pIter; + + ctgDebug("Got tb hash vgroup, vgId:%d, epNum %d, current %s port %d", vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + if (update) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + pRes->pRes = vgInfo; + } else { + res.pRes = vgInfo; + taosArrayPush(pCtx->pResList, &res); + } + } + + return TSDB_CODE_SUCCESS; + } + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + sprintf(tbFullName, "%s.", dbFName); + int32_t offset = strlen(tbFullName); + SName* pName = NULL; + int32_t tbNameLen = 0; + + for (int32_t i = 0; i < tbNum; ++i) { + pName = taosArrayGet(pCtx->pNames, i); + + tbNameLen = offset + strlen(pName->tname); + strcpy(tbFullName + offset, pName->tname); + + uint32_t hashValue = (*fp)(tbFullName, (uint32_t)tbNameLen); + + void *pIter = taosHashIterate(dbInfo->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + if (hashValue >= vgInfo->hashBegin && hashValue <= vgInfo->hashEnd) { + taosHashCancelIterate(dbInfo->vgHash, pIter); + break; + } + + pIter = taosHashIterate(dbInfo->vgHash, pIter); + vgInfo = NULL; + } + + if (NULL == vgInfo) { + ctgError("no hash range found for hash value [%u], db:%s, numOfVgId:%d", hashValue, dbFName, taosHashGetSize(dbInfo->vgHash)); + CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + } + + SVgroupInfo* pNewVg = taosMemoryMalloc(sizeof(SVgroupInfo)); + if (NULL == pNewVg) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + *pNewVg = *vgInfo; + + ctgDebug("Got tb %s hash vgroup, vgId:%d, epNum %d, current %s port %d", tbFullName, vgInfo->vgId, vgInfo->epSet.numOfEps, + vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); + + if (update) { + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); + pRes->pRes = pNewVg; + } else { + res.pRes = pNewVg; + taosArrayPush(pCtx->pResList, &res); + } + } + + CTG_RET(code); +} + + int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2) { if (*(uint64_t *)key1 < ((SSTableVersion*)key2)->suid) { return -1; From 8fc86a31d0011761ddc7e365bda36ab62e2d63a3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 4 Aug 2022 20:53:43 +0800 Subject: [PATCH 05/32] enh: asynchronous catalog interface optimization --- include/libs/catalog/catalog.h | 9 +- source/libs/parser/inc/parUtil.h | 7 +- source/libs/parser/src/parUtil.c | 102 +++++++++++++++--- source/libs/parser/src/parser.c | 8 +- .../libs/parser/test/mockCatalogService.cpp | 43 +++++--- source/libs/parser/test/mockCatalogService.h | 1 + source/libs/parser/test/parInsertTest.cpp | 13 ++- source/libs/parser/test/parTestUtil.cpp | 11 +- 8 files changed, 151 insertions(+), 43 deletions(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 1d1849f24a..ee566d759a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -58,12 +58,17 @@ typedef struct SDbInfo { int64_t dbId; } SDbInfo; +typedef struct STablesReq { + char dbFName[TSDB_DB_FNAME_LEN]; + SArray* pTables; +} STablesReq; + typedef struct SCatalogReq { SArray* pDbVgroup; // element is db full name SArray* pDbCfg; // element is db full name SArray* pDbInfo; // element is db full name - SArray* pTableMeta; // element is SNAME - SArray* pTableHash; // element is SNAME + SArray* pTableMeta; // element is STablesReq + SArray* pTableHash; // element is STablesReq SArray* pUdf; // element is udf name SArray* pIndex; // element is index name SArray* pUser; // element is SUserAuthInfo diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 896e2bc239..24f34637bf 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -38,6 +38,11 @@ typedef struct SMsgBuf { char* buf; } SMsgBuf; +typedef struct SParseTablesMetaReq { + char dbFName[TSDB_DB_FNAME_LEN]; + SHashObj* pTables; +} SParseTablesMetaReq; + typedef struct SParseMetaCache { SHashObj* pTableMeta; // key is tbFName, element is STableMeta* SHashObj* pDbVgroup; // key is dbFName, element is SArray* @@ -94,7 +99,7 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); -void destoryParseMetaCache(SParseMetaCache* pMetaCache); +void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index e51800aece..ae5a281aab 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -474,6 +474,24 @@ static int32_t buildDbReq(SHashObj* pDbsHash, SArray** pDbs) { return TSDB_CODE_SUCCESS; } +static int32_t buildTableReqFromDb(SHashObj* pDbsHash, SArray** pDbs) { + if (NULL != pDbsHash) { + *pDbs = taosArrayInit(taosHashGetSize(pDbsHash), sizeof(STablesReq)); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SParseTablesMetaReq* p = taosHashIterate(pDbsHash, NULL); + while (NULL != p) { + STablesReq req = {0}; + strcpy(req.dbFName, p->dbFName); + buildTableReq(p->pTables, &req.pTables); + taosArrayPush(*pDbs, &req); + p = taosHashIterate(pDbsHash, p); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) { if (NULL != pUserAuthHash) { *pUserAuth = taosArrayInit(taosHashGetSize(pUserAuthHash), sizeof(SUserAuthInfo)); @@ -513,12 +531,12 @@ static int32_t buildUdfReq(SHashObj* pUdfHash, SArray** pUdf) { } int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq) { - int32_t code = buildTableReq(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); + int32_t code = buildTableReqFromDb(pMetaCache->pTableMeta, &pCatalogReq->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbVgroup, &pCatalogReq->pDbVgroup); } if (TSDB_CODE_SUCCESS == code) { - code = buildTableReq(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash); + code = buildTableReqFromDb(pMetaCache->pTableVgroup, &pCatalogReq->pTableHash); } if (TSDB_CODE_SUCCESS == code) { code = buildDbReq(pMetaCache->pDbCfg, &pCatalogReq->pDbCfg); @@ -587,6 +605,24 @@ static int32_t putDbDataToCache(const SArray* pDbReq, const SArray* pDbData, SHa return TSDB_CODE_SUCCESS; } +static int32_t putDbTableDataToCache(const SArray* pDbReq, const SArray* pTableData, SHashObj** pTable) { + int32_t ndbs = taosArrayGetSize(pDbReq); + int32_t tableNo = 0; + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = taosArrayGet(pDbReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(taosArrayGet(pReq->pTables, j), fullName); + if (TSDB_CODE_SUCCESS != putMetaDataToHash(fullName, strlen(fullName), pTableData, tableNo, pTable)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + ++tableNo; + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUserAuthData, SHashObj** pUserAuth) { int32_t nvgs = taosArrayGetSize(pUserAuthReq); for (int32_t i = 0; i < nvgs; ++i) { @@ -612,12 +648,12 @@ static int32_t putUdfToCache(const SArray* pUdfReq, const SArray* pUdfData, SHas } int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache) { - int32_t code = putTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); + int32_t code = putDbTableDataToCache(pCatalogReq->pTableMeta, pMetaData->pTableMeta, &pMetaCache->pTableMeta); if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbVgroup, pMetaData->pDbVgroup, &pMetaCache->pDbVgroup); } if (TSDB_CODE_SUCCESS == code) { - code = putTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, &pMetaCache->pTableVgroup); + code = putDbTableDataToCache(pCatalogReq->pTableHash, pMetaData->pTableHash, &pMetaCache->pTableVgroup); } if (TSDB_CODE_SUCCESS == code) { code = putDbDataToCache(pCatalogReq->pDbCfg, pMetaData->pDbCfg, &pMetaCache->pDbCfg); @@ -657,14 +693,38 @@ static int32_t reserveTableReqInCache(int32_t acctId, const char* pDb, const cha return reserveTableReqInCacheImpl(fullName, len, pTables); } +static int32_t reserveTableReqInDbCacheImpl(int32_t acctId, const char* pDb, const char* pTable, SHashObj* pDbs) { + SParseTablesMetaReq req = {0}; + int32_t len = snprintf(req.dbFName, sizeof(req.dbFName), "%d.%s", acctId, pDb); + int32_t code = reserveTableReqInCache(acctId, pDb, pTable, &req.pTables); + if (TSDB_CODE_SUCCESS == code) { + code = taosHashPut(pDbs, req.dbFName, len, &req, sizeof(SParseTablesMetaReq)); + } + return code; +} + +static int32_t reserveTableReqInDbCache(int32_t acctId, const char* pDb, const char* pTable, SHashObj** pDbs) { + if (NULL == *pDbs) { + *pDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == *pDbs) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + char fullName[TSDB_DB_FNAME_LEN]; + int32_t len = snprintf(fullName, sizeof(fullName), "%d.%s", acctId, pDb); + SParseTablesMetaReq* pReq = taosHashGet(*pDbs, fullName, len); + if (NULL == pReq) { + return reserveTableReqInDbCacheImpl(acctId, pDb, pTable, *pDbs); + } + return reserveTableReqInCache(acctId, pDb, pTable, &pReq->pTables); +} + int32_t reserveTableMetaInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { - return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableMeta); + return reserveTableReqInDbCache(acctId, pDb, pTable, &pMetaCache->pTableMeta); } int32_t reserveTableMetaInCacheExt(const SName* pName, SParseMetaCache* pMetaCache) { - char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); - return reserveTableReqInCacheImpl(fullName, strlen(fullName), &pMetaCache->pTableMeta); + return reserveTableReqInDbCache(pName->acctId, pName->dbname, pName->tname, &pMetaCache->pTableMeta); } int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta) { @@ -711,13 +771,11 @@ int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, } int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { - return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableVgroup); + return reserveTableReqInDbCache(acctId, pDb, pTable, &pMetaCache->pTableVgroup); } int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache) { - char fullName[TSDB_TABLE_FNAME_LEN]; - tNameExtractFullName(pName, fullName); - return reserveTableReqInCacheImpl(fullName, strlen(fullName), &pMetaCache->pTableVgroup); + return reserveTableReqInDbCache(pName->acctId, pName->dbname, pName->tname, &pMetaCache->pTableVgroup); } int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup) { @@ -919,10 +977,24 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) { return TSDB_CODE_SUCCESS; } -void destoryParseMetaCache(SParseMetaCache* pMetaCache) { - taosHashCleanup(pMetaCache->pTableMeta); +void destoryParseTablesMetaReqHash(SHashObj* pHash) { + SParseTablesMetaReq* p = taosHashIterate(pHash, NULL); + while (NULL != p) { + taosHashCleanup(p->pTables); + p = taosHashIterate(pHash, p); + } + taosHashCleanup(pHash); +} + +void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + if (request) { + destoryParseTablesMetaReqHash(pMetaCache->pTableMeta); + destoryParseTablesMetaReqHash(pMetaCache->pTableVgroup); + } else { + taosHashCleanup(pMetaCache->pTableMeta); + taosHashCleanup(pMetaCache->pTableVgroup); + } taosHashCleanup(pMetaCache->pDbVgroup); - taosHashCleanup(pMetaCache->pTableVgroup); taosHashCleanup(pMetaCache->pDbCfg); taosHashCleanup(pMetaCache->pDbInfo); taosHashCleanup(pMetaCache->pUserAuth); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 53ee717af2..34cd783ace 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -85,13 +85,13 @@ static int32_t setValueByBindParam(SValueNode* pVal, TAOS_MULTI_BIND* pParam) { if (IS_VAR_DATA_TYPE(pVal->node.resType.type)) { taosMemoryFreeClear(pVal->datum.p); } - + if (pParam->is_null && 1 == *(pParam->is_null)) { pVal->node.resType.type = TSDB_DATA_TYPE_NULL; pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_NULL].bytes; return TSDB_CODE_SUCCESS; } - + int32_t inputSize = (NULL != pParam->length ? *(pParam->length) : tDataTypes[pParam->buffer_type].bytes); pVal->node.resType.type = pParam->buffer_type; pVal->node.resType.bytes = inputSize; @@ -187,7 +187,7 @@ int32_t qParseSqlSyntax(SParseContext* pCxt, SQuery** pQuery, struct SCatalogReq if (TSDB_CODE_SUCCESS == code) { code = buildCatalogReq(&metaCache, pCatalogReq); } - destoryParseMetaCache(&metaCache); + destoryParseMetaCache(&metaCache, true); terrno = code; return code; } @@ -203,7 +203,7 @@ int32_t qAnalyseSqlSemantic(SParseContext* pCxt, const struct SCatalogReq* pCata code = analyseSemantic(pCxt, pQuery, &metaCache); } } - destoryParseMetaCache(&metaCache); + destoryParseMetaCache(&metaCache, false); terrno = code; return code; } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 4158453110..6717a1fdf1 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -472,12 +472,16 @@ class MockCatalogServiceImpl { int32_t getAllTableMeta(SArray* pTableMetaReq, SArray** pTableMetaData) const { if (NULL != pTableMetaReq) { - int32_t ntables = taosArrayGetSize(pTableMetaReq); - *pTableMetaData = taosArrayInit(ntables, sizeof(SMetaRes)); - for (int32_t i = 0; i < ntables; ++i) { - SMetaRes res = {0}; - res.code = catalogGetTableMeta((const SName*)taosArrayGet(pTableMetaReq, i), (STableMeta**)&res.pRes); - taosArrayPush(*pTableMetaData, &res); + int32_t ndbs = taosArrayGetSize(pTableMetaReq); + *pTableMetaData = taosArrayInit(ndbs, sizeof(SMetaRes)); + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pTableMetaReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + SMetaRes res = {0}; + res.code = catalogGetTableMeta((const SName*)taosArrayGet(pReq->pTables, j), (STableMeta**)&res.pRes); + taosArrayPush(*pTableMetaData, &res); + } } } return TSDB_CODE_SUCCESS; @@ -485,13 +489,17 @@ class MockCatalogServiceImpl { int32_t getAllTableVgroup(SArray* pTableVgroupReq, SArray** pTableVgroupData) const { if (NULL != pTableVgroupReq) { - int32_t ntables = taosArrayGetSize(pTableVgroupReq); - *pTableVgroupData = taosArrayInit(ntables, sizeof(SMetaRes)); - for (int32_t i = 0; i < ntables; ++i) { - SMetaRes res = {0}; - res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo)); - res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pTableVgroupReq, i), (SVgroupInfo*)res.pRes); - taosArrayPush(*pTableVgroupData, &res); + int32_t ndbs = taosArrayGetSize(pTableVgroupReq); + *pTableVgroupData = taosArrayInit(ndbs, sizeof(SMetaRes)); + for (int32_t i = 0; i < ndbs; ++i) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pTableVgroupReq, i); + int32_t ntables = taosArrayGetSize(pReq->pTables); + for (int32_t j = 0; j < ntables; ++j) { + SMetaRes res = {0}; + res.pRes = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + res.code = catalogGetTableHashVgroup((const SName*)taosArrayGet(pReq->pTables, j), (SVgroupInfo*)res.pRes); + taosArrayPush(*pTableVgroupData, &res); + } } } return TSDB_CODE_SUCCESS; @@ -677,12 +685,17 @@ int32_t MockCatalogService::catalogGetAllMeta(const SCatalogReq* pCatalogReq, SM return impl_->catalogGetAllMeta(pCatalogReq, pMetaData); } +void MockCatalogService::destoryTablesReq(void* p) { + STablesReq* pRes = (STablesReq*)p; + taosArrayDestroy(pRes->pTables); +} + void MockCatalogService::destoryCatalogReq(SCatalogReq* pReq) { taosArrayDestroy(pReq->pDbVgroup); taosArrayDestroy(pReq->pDbCfg); taosArrayDestroy(pReq->pDbInfo); - taosArrayDestroy(pReq->pTableMeta); - taosArrayDestroy(pReq->pTableHash); + taosArrayDestroyEx(pReq->pTableMeta, destoryTablesReq); + taosArrayDestroyEx(pReq->pTableHash, destoryTablesReq); taosArrayDestroy(pReq->pUdf); taosArrayDestroy(pReq->pIndex); taosArrayDestroy(pReq->pUser); diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index d76a6abca8..c0330692ee 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -50,6 +50,7 @@ struct MockTableMeta { class MockCatalogServiceImpl; class MockCatalogService { public: + static void destoryTablesReq(void* p); static void destoryCatalogReq(SCatalogReq* pReq); static void destoryMetaRes(void* p); static void destoryMetaArrayRes(void* p); diff --git a/source/libs/parser/test/parInsertTest.cpp b/source/libs/parser/test/parInsertTest.cpp index 22a1be2579..6014a97efa 100644 --- a/source/libs/parser/test/parInsertTest.cpp +++ b/source/libs/parser/test/parInsertTest.cpp @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +#include + #include #include "mockCatalogService.h" @@ -20,6 +22,7 @@ #include "parInt.h" using namespace std; +using namespace std::placeholders; using namespace testing; namespace { @@ -63,7 +66,9 @@ class InsertTest : public Test { int32_t runAsync() { cxt_.async = true; - unique_ptr metaCache(new SParseMetaCache(), _destoryParseMetaCache); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), std::bind(_destoryParseMetaCache, _1, cref(request))); code_ = parseInsertSyntax(&cxt_, &res_, metaCache.get()); if (code_ != TSDB_CODE_SUCCESS) { cout << "parseInsertSyntax code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; @@ -81,6 +86,8 @@ class InsertTest : public Test { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); g_mockCatalogService->catalogGetAllMeta(catalogReq.get(), metaData.get()); + metaCache.reset(new SParseMetaCache()); + request = false; code_ = putMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); if (code_ != TSDB_CODE_SUCCESS) { cout << "putMetaDataToCache code:" << toString(code_) << ", msg:" << errMagBuf_ << endl; @@ -144,8 +151,8 @@ class InsertTest : public Test { static const int max_err_len = 1024; static const int max_sql_len = 1024 * 1024; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache) { - destoryParseMetaCache(pMetaCache); + static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } diff --git a/source/libs/parser/test/parTestUtil.cpp b/source/libs/parser/test/parTestUtil.cpp index 235cc487fb..3fe4b533e4 100644 --- a/source/libs/parser/test/parTestUtil.cpp +++ b/source/libs/parser/test/parTestUtil.cpp @@ -24,6 +24,7 @@ #include "parInt.h" using namespace std; +using namespace std::placeholders; using namespace testing; namespace ParserTest { @@ -118,8 +119,8 @@ class ParserTestBaseImpl { TEST_INTERFACE_ASYNC_API }; - static void _destoryParseMetaCache(SParseMetaCache* pMetaCache) { - destoryParseMetaCache(pMetaCache); + static void _destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { + destoryParseMetaCache(pMetaCache, request); delete pMetaCache; } @@ -340,7 +341,9 @@ class ParserTestBaseImpl { doParse(&cxt, query.get()); SQuery* pQuery = *(query.get()); - unique_ptr metaCache(new SParseMetaCache(), _destoryParseMetaCache); + bool request = true; + unique_ptr > metaCache( + new SParseMetaCache(), bind(_destoryParseMetaCache, _1, cref(request))); doCollectMetaKey(&cxt, pQuery, metaCache.get()); unique_ptr catalogReq(new SCatalogReq(), @@ -353,6 +356,8 @@ class ParserTestBaseImpl { unique_ptr metaData(new SMetaData(), MockCatalogService::destoryMetaData); doGetAllMeta(catalogReq.get(), metaData.get()); + metaCache.reset(new SParseMetaCache()); + request = false; doPutMetaDataToCache(catalogReq.get(), metaData.get(), metaCache.get()); doAuthenticate(&cxt, pQuery, metaCache.get()); From c91cc310c651d0e7b2f6e9f2fe51981eb5b64487 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 5 Aug 2022 14:30:56 +0800 Subject: [PATCH 06/32] fix: modify udf document --- docs/zh/07-develop/09-udf.md | 74 ++++++++++++----------------------- docs/zh/12-taos-sql/26-udf.md | 21 +++++++--- 2 files changed, 40 insertions(+), 55 deletions(-) diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 6071275b55..b8ae618105 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -124,52 +124,49 @@ gcc -g -O0 -fPIC -shared add_one.c -o add_one.so 用户可以通过 SQL 指令在系统中加载客户端所在主机上的 UDF 函数库（不能通过 RESTful 接口或 HTTP 管理界面来进行这一过程）。一旦创建成功，则当前 TDengine 集群的所有用户都可以在 SQL 指令中使用这些函数。UDF 存储在系统的 MNode 节点上，因此即使重启 TDengine 系统，已经创建的 UDF 也仍然可用。 -在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。此外， UDF 支持输入与输出类型不一致，用户需要保证输入数据类型与 UDF 程序匹配，UDF 输出数据类型与 OUTPUTTYPE 匹配。 +在创建 UDF 时，需要区分标量函数和聚合函数。如果创建时声明了错误的函数类别，则可能导致通过 SQL 指令调用函数时出错。此外，用户需要保证输入数据类型与 UDF 程序匹配，UDF 输出数据类型与 OUTPUTTYPE 匹配。 - 创建标量函数 ```sql -CREATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) [ BUFSIZE B ]; +CREATE FUNCTION function_name AS library_path OUTPUTTYPE output_type; ``` - - ids(X)：标量函数未来在 SQL 指令中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - - ids(Y)：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； - - typename(Z)：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - - B：中间计算结果的缓冲区大小，单位是字节，最小 0，最大 512，如果不使用可以不设置。 + - function_name：标量函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udf 的实际名称一致； + - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + - output_type：此函数计算结果的数据类型名称； - 例如，如下语句可以把 add_one.so 创建为系统中可用的 UDF： + 例如，如下语句可以把 libbitand.so 创建为系统中可用的 UDF： ```sql - CREATE FUNCTION add_one AS "/home/taos/udf_example/add_one.so" OUTPUTTYPE INT; + CREATE FUNCTION bit_and AS "/home/taos/udf_example/libbitand.so" OUTPUTTYPE INT; ``` - 创建聚合函数： ```sql -CREATE AGGREGATE FUNCTION ids(X) AS ids(Y) OUTPUTTYPE typename(Z) [ BUFSIZE B ]; +CREATE AGGREGATE FUNCTION function_name AS library_path OUTPUTTYPE output_type [ BUFSIZE buffer_size ]; ``` - - ids(X)：聚合函数未来在 SQL 指令中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； - - ids(Y)：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； - - typename(Z)：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； - - B：中间计算结果的缓冲区大小，单位是字节，最小 0，最大 512，如果不使用可以不设置。 + - function_name：聚合函数未来在 SQL 中被调用时的函数名，必须与函数实现中 udfNormalFunc 的实际名称一致； + - library_path：包含 UDF 函数实现的动态链接库的库文件绝对路径（指的是库文件在当前客户端所在主机上的保存路径，通常是指向一个 .so 文件），这个路径需要用英文单引号或英文双引号括起来； + - output_type：此函数计算结果的数据类型，与上文中 udfNormalFunc 的 itype 参数不同，这里不是使用数字表示法，而是直接写类型名称即可； + - buffer_size：中间计算结果的缓冲区大小，单位是字节。如果不使用可以不设置。 - 关于中间计算结果的使用，可以参考示例程序[demo.c](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/demo.c) - - 例如，如下语句可以把 demo.so 创建为系统中可用的 UDF： + 例如，如下语句可以把 libsqrsum.so 创建为系统中可用的 UDF： ```sql - CREATE AGGREGATE FUNCTION demo AS "/home/taos/udf_example/demo.so" OUTPUTTYPE DOUBLE bufsize 14; + CREATE AGGREGATE FUNCTION sqr_sum AS "/home/taos/udf_example/libsqrsum.so" OUTPUTTYPE DOUBLE bufsize 8; ``` ### 管理 UDF - 删除指定名称的用户定义函数： ``` -DROP FUNCTION ids(X); +DROP FUNCTION function_name; ``` -- ids(X)：此参数的含义与 CREATE 指令中的 ids(X) 参数一致，也即要删除的函数的名字，例如 +- function_name：此参数的含义与 CREATE 指令中的 function_name 参数一致，也即要删除的函数的名字，例如 ```sql -DROP FUNCTION add_one; +DROP FUNCTION bit_and; ``` - 显示系统中当前可用的所有 UDF： ```sql @@ -180,53 +177,32 @@ SHOW FUNCTIONS; 在 SQL 指令中，可以直接以在系统中创建 UDF 时赋予的函数名来调用用户定义函数。例如： ```sql -SELECT X(c) FROM table/stable; +SELECT X(c1,c2) FROM table/stable; ``` -表示对名为 c 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 +表示对名为 c1, c2 的数据列调用名为 X 的用户定义函数。SQL 指令中用户定义函数可以配合 WHERE 等查询特性来使用。 -## UDF 的一些使用限制 - -在当前版本下，使用 UDF 存在如下这些限制： - -1. 在创建和调用 UDF 时，服务端和客户端都只支持 Linux 操作系统； -2. UDF 不能与系统内建的 SQL 函数混合使用，暂不支持在一条 SQL 语句中使用多个不同名的 UDF ； -3. UDF 只支持以单个数据列作为输入； -4. UDF 只要创建成功，就会被持久化存储到 MNode 节点中； -5. 无法通过 RESTful 接口来创建 UDF； -6. UDF 在 SQL 中定义的函数名，必须与 .so 库文件实现中的接口函数名前缀保持一致，也即必须是 udfNormalFunc 的名称，而且不可与 TDengine 中已有的内建 SQL 函数重名。 ## 示例代码 -### 标量函数示例 [add_one](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/add_one.c) +### 标量函数示例 [bit_and](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/bit_and.c)
-add_one.c +bit_and.c ```c -{{#include tests/script/sh/add_one.c}} +{{#include tests/script/sh/bit_and.c}} ```
-### 向量函数示例 [abs_max](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/abs_max.c) +### 聚合函数示例 [sqr_sum](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/sqr_sum.c)
-abs_max.c +sqr_sum.c ```c -{{#include tests/script/sh/abs_max.c}} -``` - -
- -### 使用中间计算结果示例 [demo](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/demo.c) - -
-demo.c - -```c -{{#include tests/script/sh/demo.c}} +{{#include tests/script/sh/sqr_sum.c}} ```
diff --git a/docs/zh/12-taos-sql/26-udf.md b/docs/zh/12-taos-sql/26-udf.md index bd8d61a584..1292206311 100644 --- a/docs/zh/12-taos-sql/26-udf.md +++ b/docs/zh/12-taos-sql/26-udf.md @@ -8,21 +8,30 @@ title: 用户自定义函数 ## 创建函数 ```sql -CREATE [AGGREGATE] FUNCTION func_name AS library_path OUTPUTTYPE type_name [BUFSIZE value] +CREATE [AGGREGATE] FUNCTION func_name AS library_path OUTPUTTYPE type_name [BUFSIZE buffer_size] ``` 语法说明： AGGREGATE：标识此函数是标量函数还是聚集函数。 -func_name：函数名，必须与函数实现中udfNormalFunc的实际名称一致。 +func_name：函数名，必须与函数实现中 udf 的实际名称一致。 library_path：包含UDF函数实现的动态链接库的绝对路径，是在客户端侧主机上的绝对路径。 -OUTPUTTYPE：标识此函数的返回类型。 -BUFSIZE：中间结果的缓冲区大小，单位是字节。不设置则默认为0。最大不可超过512字节。 +type_name：标识此函数的返回类型。 +buffer_size：中间结果的缓冲区大小，单位是字节。不设置则默认为0。 关于如何开发自定义函数，请参考 [UDF使用说明](../../develop/udf)。 ## 删除自定义函数 +``` +DROP FUNCTION function_name; +``` + +- function_name：此参数的含义与 CREATE 指令中的 function_name 参数一致，也即要删除的函数的名字，例如 + + +## 显示 UDF + ```sql -DROP FUNCTION func_name -``` \ No newline at end of file +SHOW FUNCTION; +``` From 16ba0c658e9ca25847357ee4279fc219931fbeb9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 Aug 2022 15:36:05 +0800 Subject: [PATCH 07/32] enh: refactor getting table meta/hash --- source/client/src/clientImpl.c | 46 +++++++-- source/libs/catalog/inc/catalogInt.h | 23 +++-- source/libs/catalog/src/ctgAsync.c | 119 +++++++++++----------- source/libs/catalog/src/ctgCache.c | 144 +++++++-------------------- source/libs/catalog/src/ctgRemote.c | 8 +- source/libs/catalog/src/ctgUtil.c | 43 +++++++- 6 files changed, 189 insertions(+), 194 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c4dc1705c..20637b7e6e 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1922,7 +1922,7 @@ _OVER: return code; } -int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, +int32_t appendTbToReq(SHashObj* pHash, int32_t pos1, int32_t len1, int32_t pos2, int32_t len2, const char* str, int32_t acctId, char* db) { SName name; @@ -1957,20 +1957,33 @@ int32_t appendTbToReq(SArray* pList, int32_t pos1, int32_t len1, int32_t pos2, i return -1; } - taosArrayPush(pList, &name); + char dbFName[TSDB_DB_FNAME_LEN]; + sprintf(dbFName, "%d.%.*s", acctId, dbLen, dbName); + + STablesReq* pDb = taosHashGet(pHash, dbFName, strlen(dbFName)); + if (pDb) { + taosArrayPush(pDb->pTables, &name); + } else { + STablesReq db; + db.pTables = taosArrayInit(20, sizeof(SName)); + strcpy(db.dbFName, dbFName); + taosArrayPush(db.pTables, &name); + taosHashPut(pHash, dbFName, strlen(dbFName), &db, sizeof(db)); + } return TSDB_CODE_SUCCESS; } int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq) { - *pReq = taosArrayInit(10, sizeof(SName)); - if (NULL == *pReq) { + SHashObj* pHash = taosHashInit(3, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pHash) { terrno = TSDB_CODE_OUT_OF_MEMORY; return terrno; } bool inEscape = false; int32_t code = 0; + void *pIter = NULL; int32_t vIdx = 0; int32_t vPos[2]; @@ -1985,7 +1998,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, vLen[vIdx] = i - vPos[vIdx]; } - code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + code = appendTbToReq(pHash, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); if (code) { goto _return; } @@ -2035,7 +2048,7 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, vLen[vIdx] = i - vPos[vIdx]; } - code = appendTbToReq(*pReq, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); + code = appendTbToReq(pHash, vPos[0], vLen[0], vPos[1], vLen[1], tbList, acctId, dbName); if (code) { goto _return; } @@ -2067,14 +2080,31 @@ int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, goto _return; } + int32_t dbNum = taosHashGetSize(pHash); + *pReq = taosArrayInit(dbNum, sizeof(STablesReq)); + pIter = taosHashIterate(pHash, NULL); + while (pIter) { + STablesReq* pDb = (STablesReq*)pIter; + taosArrayPush(*pReq, pDb); + pIter = taosHashIterate(pHash, pIter); + } + + taosHashCleanup(pHash); + return TSDB_CODE_SUCCESS; _return: terrno = TSDB_CODE_TSC_INVALID_OPERATION; - taosArrayDestroy(*pReq); - *pReq = NULL; + pIter = taosHashIterate(pHash, NULL); + while (pIter) { + STablesReq* pDb = (STablesReq*)pIter; + taosArrayDestroy(pDb->pTables); + pIter = taosHashIterate(pHash, pIter); + } + + taosHashCleanup(pHash); return terrno; } diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index e12bcfd18f..d4b8a39197 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -32,6 +32,7 @@ extern "C" { #define CTG_DEFAULT_RENT_SLOT_SIZE 10 #define CTG_DEFAULT_MAX_RETRY_TIMES 3 #define CTG_DEFAULT_BATCH_NUM 64 +#define CTG_DEFAULT_FETCH_NUM 8 #define CTG_RENT_SLOT_SECOND 1.5 @@ -113,7 +114,8 @@ typedef struct SCtgTbMetaCtx { } SCtgTbMetaCtx; typedef struct SCtgFetch { - int32_t reqIdx; + int32_t dbIdx; + int32_t tbIdx; int32_t fetchIdx; int32_t resIdx; int32_t flag; @@ -121,12 +123,12 @@ typedef struct SCtgFetch { int32_t vgId; } SCtgFetch; -typedef struct SCtgTbMetaBCtx { +typedef struct SCtgTbMetasCtx { int32_t fetchNum; SArray* pNames; SArray* pResList; SArray* pFetchs; -} SCtgTbMetaBCtx; +} SCtgTbMetasCtx; typedef struct SCtgTbIndexCtx { SName* pName; @@ -155,12 +157,12 @@ typedef struct SCtgTbHashCtx { SName* pName; } SCtgTbHashCtx; -typedef struct SCtgTbHashBCtx { +typedef struct SCtgTbHashsCtx { int32_t fetchNum; SArray* pNames; SArray* pResList; SArray* pFetchs; -} SCtgTbHashBCtx; +} SCtgTbHashsCtx; typedef struct SCtgIndexCtx { @@ -617,7 +619,7 @@ int32_t ctgdShowCacheInfo(void); int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq); int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList); +int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList); int32_t ctgOpUpdateVgroup(SCtgCacheOperation *action); int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *action); @@ -696,7 +698,7 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); -int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update); +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); @@ -708,6 +710,8 @@ int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* targ int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target); char * ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, char* dbFName, int32_t vgId); +int32_t ctgGetTablesReqNum(SArray *pList); +int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); void ctgFreeSTableIndex(void *info); void ctgClearSubTaskRes(SCtgSubRes *pRes); @@ -717,6 +721,11 @@ void ctgFreeTbCacheImpl(SCtgTbCache *pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup); +FORCE_INLINE SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); + return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); +} + extern SCatalogMgmt gCtgMgmt; extern SCtgDebug gCTGDebug; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index eed4a956b5..616a9cafe7 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -50,7 +50,7 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { +int32_t ctgInitGetTbMetasTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SName *name = (SName*)param; SCtgTask task = {0}; @@ -58,18 +58,19 @@ int32_t ctgInitGetTbMetaBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { task.taskId = taskIdx; task.pJob = pJob; - task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetaBCtx)); + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbMetasCtx)); if (NULL == task.taskCtx) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - SCtgTbMetaBCtx* ctx = task.taskCtx; + SCtgTbMetasCtx* ctx = task.taskCtx; ctx->pNames = param; - ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + ctx->pResList = taosArrayInit(pJob->tbMetaNum, sizeof(SMetaRes)); taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", + pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbMetaNum); return TSDB_CODE_SUCCESS; } @@ -177,7 +178,7 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, void* param) { return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetTbHashBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { +int32_t ctgInitGetTbHashsTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SName *name = (SName*)param; SCtgTask task = {0}; @@ -185,18 +186,19 @@ int32_t ctgInitGetTbHashBTask(SCtgJob *pJob, int32_t taskIdx, void* param) { task.taskId = taskIdx; task.pJob = pJob; - task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashBCtx)); + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbHashsCtx)); if (NULL == task.taskCtx) { CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - SCtgTbHashBCtx* ctx = task.taskCtx; + SCtgTbHashsCtx* ctx = task.taskCtx; ctx->pNames = param; - ctx->pResList = taosArrayInit(taosArrayGetSize(ctx->pNames), sizeof(SMetaRes)); + ctx->pResList = taosArrayInit(pJob->tbHashNum, sizeof(SMetaRes)); taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbNum:%d", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames)); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbNum:%d, tbNum:%d", + pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), taosArrayGetSize(ctx->pNames), pJob->tbHashNum); return TSDB_CODE_SUCCESS; } @@ -477,9 +479,9 @@ int32_t ctgInitTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t *tas int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, const SCatalogReq* pReq, catalogCallback fp, void* param) { int32_t code = 0; - int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); + int32_t tbMetaNum = (int32_t)ctgGetTablesReqNum(pReq->pTableMeta); int32_t dbVgNum = (int32_t)taosArrayGetSize(pReq->pDbVgroup); - int32_t tbHashNum = (int32_t)taosArrayGetSize(pReq->pTableHash); + int32_t tbHashNum = (int32_t)ctgGetTablesReqNum(pReq->pTableHash); int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf); int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0; int32_t dnodeNum = pReq->dNodeRequired ? 1 : 0; @@ -647,7 +649,7 @@ int32_t ctgDumpTbMetaRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } -int32_t ctgDumpTbMetaBRes(SCtgTask* pTask) { +int32_t ctgDumpTbMetasRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableMeta = pTask->res; @@ -686,7 +688,7 @@ int32_t ctgDumpTbHashRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } -int32_t ctgDumpTbHashBRes(SCtgTask* pTask) { +int32_t ctgDumpTbHashsRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; pJob->jobRes.pTableHash = pTask->res; @@ -929,9 +931,9 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); #if CTG_BATCH_FETCH - SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); + SName* pName = ctgGetFetchName(ctx->pNames, pFetch); int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; #else @@ -1079,7 +1081,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * */ #if CTG_BATCH_FETCH - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); pRes->code = 0; pRes->pRes = pOut->tbMeta; pOut->tbMeta = NULL; @@ -1098,7 +1100,7 @@ _return: #if CTG_BATCH_FETCH if (code) { - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->reqIdx); + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); pRes->code = code; pRes->pRes = NULL; if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { @@ -1184,9 +1186,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashsRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; - SCtgTbHashBCtx* ctx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); @@ -1197,7 +1199,8 @@ int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf case TDMT_MND_USE_DB: { SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; - CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, true)); + STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1217,12 +1220,10 @@ int32_t ctgHandleGetTbHashBRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf _return: if (code) { - SName* pName = taosArrayGet(ctx->pNames, pFetch->reqIdx); // TODO - - SMetaRes res = {0}; - int32_t num = taosArrayGetSize(ctx->pNames); + STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); + int32_t num = taosArrayGetSize(pReq->pTables); for (int32_t i = 0; i < num; ++i) { - SMetaRes *pRes = taosArrayGet(ctx->pResList, i); + SMetaRes *pRes = taosArrayGet(ctx->pResList, pFetch->resIdx + i); pRes->code = code; pRes->pRes = NULL; } @@ -1468,12 +1469,21 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } -int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { +int32_t ctgLaunchGetTbMetasTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgTbMetaBCtx* pCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* pCtx = (SCtgTbMetasCtx*)pTask->taskCtx; - CTG_ERR_RET(ctgGetTbMetaBFromCache(pCtg, pConn, pCtx, pCtx->pNames)); + int32_t dbNum = taosArrayGetSize(pCtx->pNames); + int32_t fetchIdx = 0; + int32_t baseResIdx = 0; + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* pReq = taosArrayGet(pCtx->pNames, i); + ctgDebug("start to check tb metas in db %s, tbNum %d", pReq->dbFName, taosArrayGetSize(pReq->pTables)); + CTG_ERR_RET(ctgGetTbMetasFromCache(pCtg, pConn, pCtx, i, &fetchIdx, baseResIdx, pReq->pTables)); + baseResIdx += taosArrayGetSize(pReq->pTables); + } + pCtx->fetchNum = taosArrayGetSize(pCtx->pFetchs); if (pCtx->fetchNum <= 0) { TSWAP(pTask->res, pCtx->pResList); @@ -1487,7 +1497,7 @@ int32_t ctgLaunchGetTbMetaBTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); pTask->msgIdx = pFetch->fetchIdx; CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); @@ -1568,46 +1578,33 @@ _return: CTG_RET(code); } -int32_t ctgLaunchGetTbHashBTask(SCtgTask *pTask) { +int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgTbHashBCtx* pCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgTbHashsCtx* pCtx = (SCtgTbHashsCtx*)pTask->taskCtx; SCtgDBCache *dbCache = NULL; - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - int32_t dbNum = 1; - int32_t fIdx = 0; - int32_t tbNum = 0; + int32_t dbNum = taosArrayGetSize(pCtx->pNames); + int32_t fetchIdx = 0; + int32_t baseResIdx = 0; int32_t code = 0; - for (int32_t i = 0; i < dbNum; ++i) { // TODO - SName* pName = taosArrayGet(pCtx->pNames, 0); - if (IS_SYS_DBNAME(pName->dbname)) { - strcpy(dbFName, pName->dbname); - } else { - tNameGetFullDbName(pName, dbFName); - } + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* pReq = taosArrayGet(pCtx->pNames, i); - CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGetVgInfoBFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, dbFName, false)); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; - } else { - if (NULL == pCtx->pFetchs) { - pCtx->pFetchs = taosArrayInit(dbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.resIdx = tbNum; - tbNum += taosArrayGetSize(pCtx->pNames); // TODO - - taosArrayPush(pCtx->pFetchs, &fetch); - taosArraySetSize(pCtx->pResList, tbNum); + baseResIdx += taosArrayGetSize(pReq->pTables); + } else { + ctgAddFetch(&pCtx->pFetchs, i, -1, &fetchIdx, baseResIdx, 0); + + baseResIdx += taosArrayGetSize(pReq->pTables); + taosArraySetSize(pCtx->pResList, baseResIdx); } } @@ -1624,7 +1621,7 @@ int32_t ctgLaunchGetTbHashBTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = taosArrayGet(pCtx->pNames, pFetch->reqIdx); + SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); pTask->msgIdx = pFetch->fetchIdx; @@ -1903,8 +1900,8 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, - {ctgInitGetTbMetaBTask, ctgLaunchGetTbMetaBTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaBRes, NULL, NULL}, - {ctgInitGetTbHashBTask, ctgLaunchGetTbHashBTask, ctgHandleGetTbHashBRsp, ctgDumpTbHashBRes, NULL, NULL}, + {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetasRes, NULL, NULL}, + {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1a28ff3e25..930361419e 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -2151,7 +2151,7 @@ int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMet } #if 0 -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray** pResList) { +int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, SArray** pResList) { int32_t tbNum = taosArrayGetSize(ctx->pNames); SName* fName = taosArrayGet(ctx->pNames, 0); int32_t fIdx = 0; @@ -2189,7 +2189,7 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } SCtgFetch fetch = {0}; - fetch.reqIdx = i; + fetch.tbIdx = i; fetch.fetchIdx = fIdx++; fetch.flag = nctx.flag; @@ -2207,9 +2207,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } #endif -int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaBCtx* ctx, SArray* pList) { +int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList) { int32_t tbNum = taosArrayGetSize(pList); - int32_t fIdx = 0; SName* pName = taosArrayGet(pList, 0); char dbFName[TSDB_DB_FNAME_LEN] = {0}; int32_t flag = CTG_FLAG_UNKNOWN_STB; @@ -2230,18 +2229,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { - SMetaRes res = {0}; - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); } return TSDB_CODE_SUCCESS; @@ -2249,22 +2238,12 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe for (int32_t i = 0; i < tbNum; ++i) { SName* pName = taosArrayGet(pList, i); - SMetaRes res = {0}; pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); continue; } @@ -2272,17 +2251,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); continue; } @@ -2296,21 +2266,20 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe nctx.tbInfo.suid = tbMeta->suid; nctx.tbInfo.tbType = tbMeta->tableType; + SMetaRes res = {0}; STableMeta* pTableMeta = NULL; if (tbMeta->tableType != TSDB_CHILD_TABLE) { int32_t metaSize = CTG_META_SIZE(tbMeta); pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { - //ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pTableMeta, tbMeta, metaSize); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); @@ -2326,10 +2295,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe cloneTableMeta(lastTableMeta, &pTableMeta); memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); @@ -2342,15 +2309,14 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe int32_t metaSize = sizeof(SCTableMeta); pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(pTableMeta, tbMeta, metaSize); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", pName->tname, nctx.tbInfo.tbType, dbFName); @@ -2358,17 +2324,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe char* stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); continue; @@ -2379,17 +2336,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); taosHashRelease(dbCache->stbCache, stName); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); continue; @@ -2400,22 +2348,11 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe CTG_LOCK(CTG_READ, &pCache->metaLock); if (NULL == pCache->pMeta) { ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", pTableMeta->suid, dbFName); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); @@ -2424,24 +2361,13 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe STableMeta* stbMeta = pCache->pMeta; if (stbMeta->suid != nctx.tbInfo.suid) { - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, nctx.tbInfo.suid); - if (NULL == ctx->pFetchs) { - ctx->pFetchs = taosArrayInit(tbNum, sizeof(SCtgFetch)); - } - - SCtgFetch fetch = {0}; - fetch.reqIdx = i; - fetch.fetchIdx = fIdx++; - fetch.flag = flag; - - taosArrayPush(ctx->pFetchs, &fetch); - taosArrayPush(ctx->pResList, &res); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); + taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); taosMemoryFreeClear(pTableMeta); @@ -2451,16 +2377,14 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe metaSize = CTG_META_SIZE(stbMeta); pTableMeta = taosMemoryRealloc(pTableMeta, metaSize); if (NULL == pTableMeta) { - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); - if (pCache) { - CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - } + CTG_UNLOCK(CTG_READ, &pCache->metaLock); + taosHashRelease(dbCache->tbCache, pCache); res.pRes = pTableMeta; taosArrayPush(ctx->pResList, &res); @@ -2469,6 +2393,8 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe lastTableMeta = pTableMeta; } + ctgReleaseDBCache(pCtg, dbCache); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index a8d5a7ae95..e464f0f36c 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -466,9 +466,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { - SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; @@ -512,9 +512,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo *pConn, SCtgT pName = ctx->pName; } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { - SCtgTbMetaBCtx* ctx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - pName = taosArrayGet(ctx->pNames, fetch->reqIdx); + pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; pName = ctx->pName; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index c841255b42..d21eacf756 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -635,7 +635,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { break; } case CTG_TASK_GET_TB_META_BATCH: { - SCtgTbMetaBCtx* taskCtx = (SCtgTbMetaBCtx*)pTask->taskCtx; + SCtgTbMetasCtx* taskCtx = (SCtgTbMetasCtx*)pTask->taskCtx; taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchMeta); taosArrayDestroy(taskCtx->pFetchs); // NO NEED TO FREE pNames @@ -656,7 +656,7 @@ void ctgFreeTaskCtx(SCtgTask* pTask) { break; } case CTG_TASK_GET_TB_HASH_BATCH: { - SCtgTbHashBCtx* taskCtx = (SCtgTbHashBCtx*)pTask->taskCtx; + SCtgTbHashsCtx* taskCtx = (SCtgTbHashsCtx*)pTask->taskCtx; taosArrayDestroyEx(taskCtx->pResList, ctgFreeBatchHash); taosArrayDestroy(taskCtx->pFetchs); // NO NEED TO FREE pNames @@ -874,7 +874,7 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashBCtx *pCtx, char* dbFName, bool update) { +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; SMetaRes res = {0}; int32_t vgNum = taosHashGetSize(dbInfo->vgHash); @@ -888,7 +888,7 @@ int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); - int32_t tbNum = taosArrayGetSize(pCtx->pNames); + int32_t tbNum = taosArrayGetSize(pNames); if (1 == vgNum) { void *pIter = taosHashIterate(dbInfo->vgHash, NULL); @@ -923,7 +923,7 @@ int32_t ctgGetVgInfoBFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d int32_t tbNameLen = 0; for (int32_t i = 0; i < tbNum; ++i) { - pName = taosArrayGet(pCtx->pNames, i); + pName = taosArrayGet(pNames, i); tbNameLen = offset + strlen(pName->tname); strcpy(tbFullName + offset, pName->tname); @@ -1112,4 +1112,37 @@ int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, cha return TSDB_CODE_SUCCESS; } +int32_t ctgGetTablesReqNum(SArray *pList) { + if (NULL == pList) { + return 0; + } + + int32_t total = 0; + int32_t n = taosArrayGetSize(pList); + for (int32_t i = 0; i < n; ++i) { + STablesReq *pReq = taosArrayGet(pList, i); + total += taosArrayGetSize(pReq->pTables); + } + + return total; +} + +int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fetchIdx, int32_t resIdx, int32_t flag) { + if (NULL == (*pFetchs)) { + *pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgFetch)); + } + + SCtgFetch fetch = {0}; + fetch.dbIdx = dbIdx; + fetch.tbIdx = tbIdx; + fetch.fetchIdx = (*fetchIdx)++; + fetch.resIdx = resIdx; + fetch.flag = flag; + + taosArrayPush(*pFetchs, &fetch); + + return TSDB_CODE_SUCCESS; +} + + From f03055c373772126709f7dafbfa0db38042b5ed2 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Fri, 5 Aug 2022 15:49:03 +0800 Subject: [PATCH 08/32] shell: change shell record source file command --- tools/shell/src/shellEngine.c | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 4526ff2230..f0bda82172 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -22,7 +22,8 @@ static bool shellIsEmptyCommand(const char *cmd); static int32_t shellRunSingleCommand(char *command); -static int32_t shellRunCommand(char *command); +static void shellRecordCommandToHistory(char *command); +static int32_t shellRunCommand(char *command, bool recordHistory); static void shellRunSingleCommandImp(char *command); static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision); static int32_t shellDumpResultToFile(const char *fname, TAOS_RES *tres); @@ -101,11 +102,7 @@ int32_t shellRunSingleCommand(char *command) { return 0; } -int32_t shellRunCommand(char *command) { - if (shellIsEmptyCommand(command)) { - return 0; - } - +void shellRecordCommandToHistory(char *command) { SShellHistory *pHistory = &shell.history; if (pHistory->hstart == pHistory->hend || pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE] == NULL || @@ -120,6 +117,14 @@ int32_t shellRunCommand(char *command) { pHistory->hstart = (pHistory->hstart + 1) % SHELL_MAX_HISTORY_SIZE; } } +} + +int32_t shellRunCommand(char *command, bool recordHistory) { + if (shellIsEmptyCommand(command)) { + return 0; + } + + if (recordHistory) shellRecordCommandToHistory(command); char quote = 0, *cmd = command; for (char c = *command++; c != 0; c = *command++) { @@ -826,11 +831,15 @@ void shellSourceFile(const char *file) { size_t cmd_len = 0; char *line = NULL; char fullname[PATH_MAX] = {0}; + char sourceFileCommand[PATH_MAX + 8] = {0}; if (taosExpandDir(file, fullname, PATH_MAX) != 0) { tstrncpy(fullname, file, PATH_MAX); } + sprintf(sourceFileCommand, "source %s;",fullname); + shellRecordCommandToHistory(sourceFileCommand); + TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM); if (pFile == NULL) { fprintf(stderr, "failed to open file %s\r\n", fullname); @@ -853,9 +862,13 @@ void shellSourceFile(const char *file) { continue; } + if (line[read_len - 1] == '\r') { + line[read_len - 1] = ' '; + } + memcpy(cmd + cmd_len, line, read_len); printf("%s%s\r\n", shell.info.promptHeader, cmd); - shellRunCommand(cmd); + shellRunCommand(cmd, false); memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } @@ -977,7 +990,7 @@ void *shellThreadLoop(void *arg) { } taosResetTerminalMode(); - } while (shellRunCommand(command) == 0); + } while (shellRunCommand(command, true) == 0); taosMemoryFreeClear(command); shellWriteHistory(); @@ -1019,7 +1032,7 @@ int32_t shellExecute() { if (pArgs->commands != NULL) { printf("%s%s\r\n", shell.info.promptHeader, pArgs->commands); char *cmd = strdup(pArgs->commands); - shellRunCommand(cmd); + shellRunCommand(cmd, true); taosMemoryFree(cmd); } From 992912246f734edcab444ff7815599da5e6f479d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 Aug 2022 15:58:41 +0800 Subject: [PATCH 09/32] fix: fix get db name issue --- source/libs/catalog/src/ctgAsync.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 616a9cafe7..37994aa97f 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1621,16 +1621,12 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { for (int32_t i = 0; i < pCtx->fetchNum; ++i) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); - SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); + STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); pTask->msgIdx = pFetch->fetchIdx; SBuildUseDBInput input = {0}; - if (IS_SYS_DBNAME(pName->dbname)) { - strcpy(input.db, pName->dbname); - } else { - tNameGetFullDbName(pName, input.db); - } + strcpy(input.db, pReq->dbFName); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; From ac63ee40c95c95e1f2a458e0bd71ff8196eb219f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 5 Aug 2022 17:44:11 +0800 Subject: [PATCH 10/32] fix: fix get tb meta for tb cfg case --- source/libs/catalog/inc/catalogInt.h | 6 +- source/libs/catalog/src/ctgAsync.c | 194 ++++++++++++++++++++++++--- source/libs/catalog/src/ctgUtil.c | 4 + 3 files changed, 178 insertions(+), 26 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index d4b8a39197..c452f61b75 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -720,11 +720,7 @@ void ctgClearHandle(SCatalog* pCtg); void ctgFreeTbCacheImpl(SCtgTbCache *pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); int32_t ctgGetTbHashVgroup(SCatalog *pCtg, SRequestConnInfo *pConn, const SName *pTableName, SVgroupInfo *pVgroup); - -FORCE_INLINE SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { - STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); - return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); -} +SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 37994aa97f..bbca85ed14 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -930,18 +930,10 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); -#if CTG_BATCH_FETCH - SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); - SName* pName = ctgGetFetchName(ctx->pNames, pFetch); - int32_t flag = pFetch->flag; - int32_t* vgId = &pFetch->vgId; -#else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SName* pName = ctx->pName; int32_t flag = ctx->flag; int32_t* vgId = &ctx->vgId; -#endif CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1080,17 +1072,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * } */ -#if CTG_BATCH_FETCH - SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); - pRes->code = 0; - pRes->pRes = pOut->tbMeta; - pOut->tbMeta = NULL; - if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { - TSWAP(pTask->res, ctx->pResList); - } -#else TSWAP(pTask->res, pOut->tbMeta); -#endif _return: @@ -1098,7 +1080,177 @@ _return: ctgReleaseVgInfoToCache(pCtg, dbCache); } -#if CTG_BATCH_FETCH + if (pTask->res) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + + +int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + SCtgDBCache *dbCache = NULL; + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SName* pName = ctgGetFetchName(ctx->pNames, pFetch); + int32_t flag = pFetch->flag; + int32_t* vgId = &pFetch->vgId; + + CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); + + switch (reqType) { + case TDMT_MND_USE_DB: { + SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; + + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, pOut->dbVgroup, pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + + return TSDB_CODE_SUCCESS; + } + case TDMT_MND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + if (CTG_FLAG_IS_STB(flag)) { + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + if (NULL != dbCache) { + SVgroupInfo vgInfo = {0}; + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pName, &vgInfo)); + + ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); + + *vgId = vgInfo.vgId; + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + + ctgReleaseVgInfoToCache(pCtg, dbCache); + } else { + SBuildUseDBInput input = {0}; + + tstrncpy(input.db, dbFName, tListLen(input.db)); + input.vgVersion = CTG_DEFAULT_INVALID_VERSION; + + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + } + + return TSDB_CODE_SUCCESS; + } + + ctgError("no tbmeta got, tbName:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); + + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (pMsgCtx->lastOut) { + TSWAP(pMsgCtx->out, pMsgCtx->lastOut); + STableMetaOutput* pLastOut = (STableMetaOutput*)pMsgCtx->out; + TSWAP(pLastOut->tbMeta, pOut->tbMeta); + } + + break; + } + case TDMT_VND_TABLE_META: { + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + if (CTG_IS_META_NULL(pOut->metaType)) { + ctgError("no tbmeta got, tbNmae:%s", tNameGetTableName(pName)); + ctgRemoveTbMetaFromCache(pCtg, pName, false); + CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST); + } + + if (CTG_FLAG_IS_STB(flag)) { + break; + } + + if (CTG_IS_META_TABLE(pOut->metaType) && TSDB_SUPER_TABLE == pOut->tbMeta->tableType) { + ctgDebug("will continue to refresh tbmeta since got stb, tbName:%s", tNameGetTableName(pName)); + + taosMemoryFreeClear(pOut->tbMeta); + + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + } else if (CTG_IS_META_BOTH(pOut->metaType)) { + int32_t exist = 0; + if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + taosMemoryFreeClear(pOut->tbMeta); + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (pOut->tbMeta) { + exist = 1; + } + } + + if (0 == exist) { + TSWAP(pMsgCtx->lastOut, pMsgCtx->out); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); + } + } + break; + } + default: + ctgError("invalid reqType %d", reqType); + CTG_ERR_JRET(TSDB_CODE_INVALID_MSG); + break; + } + + STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out; + + ctgUpdateTbMetaToCache(pCtg, pOut, false); + + if (CTG_IS_META_BOTH(pOut->metaType)) { + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } + +/* + else if (CTG_IS_META_CTABLE(pOut->metaType)) { + SName stbName = *pName; + strcpy(stbName.tname, pOut->tbName); + SCtgTbMetaCtx stbCtx = {0}; + stbCtx.flag = flag; + stbCtx.pName = &stbName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); + if (NULL == pOut->tbMeta) { + ctgDebug("stb no longer exist, stbName:%s", stbName.tname); + CTG_ERR_JRET(ctgRelaunchGetTbMetaTask(pTask)); + + return TSDB_CODE_SUCCESS; + } + + memcpy(pOut->tbMeta, &pOut->ctbMeta, sizeof(pOut->ctbMeta)); + } +*/ + + SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); + pRes->code = 0; + pRes->pRes = pOut->tbMeta; + pOut->tbMeta = NULL; + if (0 == atomic_sub_fetch_32(&ctx->fetchNum, 1)) { + TSWAP(pTask->res, ctx->pResList); + } + +_return: + + if (dbCache) { + ctgReleaseVgInfoToCache(pCtg, dbCache); + } + if (code) { SMetaRes* pRes = taosArrayGet(ctx->pResList, pFetch->resIdx); pRes->code = code; @@ -1107,7 +1259,6 @@ _return: TSWAP(pTask->res, ctx->pResList); } } -#endif if (pTask->res) { ctgHandleTaskEnd(pTask, code); @@ -1116,6 +1267,7 @@ _return: CTG_RET(code); } + int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; @@ -1896,7 +2048,7 @@ SCtgAsyncFps gCtgAsyncFps[] = { {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, {ctgInitGetSvrVerTask, ctgLaunchGetSvrVerTask, ctgHandleGetSvrVerRsp, ctgDumpSvrVer, NULL, NULL}, - {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetasRes, NULL, NULL}, + {ctgInitGetTbMetasTask, ctgLaunchGetTbMetasTask, ctgHandleGetTbMetasRsp, ctgDumpTbMetasRes, NULL, NULL}, {ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL}, }; diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index d21eacf756..de9ea7fb5a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -1144,5 +1144,9 @@ int32_t ctgAddFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t *fet return TSDB_CODE_SUCCESS; } +SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch) { + STablesReq* pReq = (STablesReq*)taosArrayGet(pNames, pFetch->dbIdx); + return (SName*)taosArrayGet(pReq->pTables, pFetch->tbIdx); +} From 3ec64ad253deff2875027e155a37616b16b42fda Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 09:26:19 +0800 Subject: [PATCH 11/32] fix: fix task timeout error --- source/common/src/systable.c | 6 +++--- source/libs/scheduler/src/schStatus.c | 2 +- source/libs/scheduler/src/schTask.c | 1 + source/libs/scheduler/src/scheduler.c | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 29d7b557c8..675d3bbfdd 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -308,9 +308,9 @@ static const SSysDbTableSchema offsetSchema[] = { }; static const SSysDbTableSchema querySchema[] = { - {.name = "query_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "req_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, - {.name = "connId", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, + {.name = "kill_id", .bytes = TSDB_QUERY_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "query_id", .bytes = 8, .type = TSDB_DATA_TYPE_UBIGINT}, + {.name = "conn_id", .bytes = 4, .type = TSDB_DATA_TYPE_UINT}, {.name = "app", .bytes = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "pid", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "user", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index a4fa4f2839..64cda573f1 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -64,7 +64,7 @@ _return: int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SSchedulerReq* pReq) { SSchJob *pJob = schAcquireJob(jobId); if (NULL == pJob) { - qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index ecd0253e1c..520ddee25e 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -536,6 +536,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { if (TSDB_CODE_SCH_TIMEOUT_ERROR == errCode) { pTask->maxExecTimes++; + pTask->maxRetryTimes++; if (pTask->timeoutUsec < SCH_MAX_TASK_TIMEOUT_USEC) { pTask->timeoutUsec *= 2; if (pTask->timeoutUsec > SCH_MAX_TASK_TIMEOUT_USEC) { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3a15523040..a37cd4fd9e 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -150,7 +150,7 @@ void schedulerFreeJob(int64_t* jobId, int32_t errCode) { SSchJob *pJob = schAcquireJob(*jobId); if (NULL == pJob) { - qError("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); + qWarn("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, *jobId); return; } From b3e53c651e37d16c2355f6b5acf67bd953e6629a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 10:04:56 +0800 Subject: [PATCH 12/32] fix(query): fix interp + fill interpolation behavior TD-18173 TD-18174 --- source/libs/executor/src/timewindowoperator.c | 40 ++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0fccc40889..8a0564c129 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2234,7 +2234,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { blockDataCleanup(pResBlock); - int32_t numOfRows = 0; + //int32_t numOfRows = 0; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -2263,7 +2263,8 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, numOfRows, v, false); + //colDataAppend(pDst, numOfRows, v, false); + colDataAppend(pDst, pResBlock->info.rows, v, false); } pResBlock->info.rows += 1; @@ -2312,12 +2313,47 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } + // add current row if timestamp match + if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) { + for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { + SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); + + char* v = colDataGetData(pSrc, i); + colDataAppend(pDst, pResBlock->info.rows, v, false); + } + + pResBlock->info.rows += 1; + doKeepPrevRows(pSliceInfo, pBlock, i); + + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + if (pSliceInfo->current > pSliceInfo->win.ekey) { doSetOperatorCompleted(pOperator); break; } } } + + //check if need to interpolate after ts range + while (pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } } // restore the value From 77d8657e6912d288acb9342a984d00acf759c746 Mon Sep 17 00:00:00 2001 From: gccgdb1234 Date: Sat, 6 Aug 2022 10:48:50 +0800 Subject: [PATCH 13/32] fix: correct wrong parameter thread_count_create_tbl --- source/libs/sync/test/sh/insert.tpl.json | 2 +- tests/pytest/cluster/TD-3693/insert1Data.json | 2 +- tests/pytest/cluster/TD-3693/insert2Data.json | 2 +- tests/pytest/dockerCluster/insert.json | 2 +- tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json | 2 +- tests/pytest/perfbenchmark/bug3433.py | 2 +- tests/pytest/perfbenchmark/joinPerformance.py | 2 +- tests/pytest/perfbenchmark/taosdemoInsert.py | 2 +- tests/pytest/query/nestedQuery/insertData.json | 2 +- tests/pytest/query/query1970YearsAf.py | 2 +- tests/pytest/tools/insert-interlace.json | 2 +- tests/pytest/tools/insert-tblimit-tboffset-createdb.json | 2 +- tests/pytest/tools/insert-tblimit-tboffset-insertrec.json | 2 +- tests/pytest/tools/insert-tblimit-tboffset.json | 2 +- tests/pytest/tools/insert-tblimit-tboffset0.json | 2 +- tests/pytest/tools/insert-tblimit1-tboffset.json | 2 +- tests/pytest/tools/insert.json | 2 +- .../tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json | 2 +- .../taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json | 2 +- .../tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json | 2 +- .../taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json | 2 +- .../NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json | 2 +- .../NanoTestCase/taosdemoTestNanoDatabaseNow.json | 2 +- .../NanoTestCase/taosdemoTestNanoDatabasecsv.json | 2 +- tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json | 2 +- .../tools/taosdemoAllTest/TD-4985/query-limit-offset.json | 2 +- .../tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-disorder.json | 2 +- .../tools/taosdemoAllTest/insert-drop-exist-auto-N00.json | 2 +- .../tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-illegal.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-newdb.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-newtable.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-offset.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-renewdb.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-sample.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert-timestep.json | 2 +- .../insertBinaryLenLarge16374AllcolLar49151.json | 2 +- tests/pytest/tools/taosdemoAllTest/insertChildTab0.json | 2 +- tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json | 2 +- .../tools/taosdemoAllTest/insertColumnsAndTagNum4096.json | 2 +- .../tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json | 2 +- tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json | 2 +- .../tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json | 2 +- tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json | 2 +- .../pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json | 2 +- .../tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json | 2 +- tests/pytest/tools/taosdemoAllTest/insertRestful.json | 2 +- tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json | 2 +- tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json | 2 +- .../tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json | 2 +- tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json | 2 +- tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json | 2 +- tests/pytest/tools/taosdemoAllTest/manual_block2.json | 2 +- .../pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json | 2 +- .../pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json | 2 +- tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json | 2 +- tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json | 2 +- tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json | 2 +- tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json | 2 +- tests/pytest/tools/taosdemoAllTest/query-interrupt.json | 2 +- tests/pytest/tools/taosdemoAllTest/queryInsertdata.json | 2 +- tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json | 2 +- tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json | 2 +- tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json | 2 +- .../tools/taosdemoAllTest/stmt/1174-small-stmt-random.json | 2 +- tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json | 2 +- tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json | 2 +- .../pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json | 2 +- .../pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json | 2 +- .../pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json | 2 +- .../taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json | 2 +- .../taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json | 2 +- .../tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json | 2 +- .../tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json | 2 +- tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json | 2 +- .../pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json | 2 +- .../tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json | 2 +- tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json | 2 +- .../pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json | 2 +- tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json | 2 +- .../pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json | 2 +- .../stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json | 2 +- .../pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json | 2 +- .../tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json | 2 +- .../taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json | 2 +- .../tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json | 2 +- .../taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json | 2 +- .../tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json | 2 +- .../taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json | 2 +- .../taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json | 2 +- .../taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json | 2 +- .../tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json | 2 +- .../stmt/insertTimestepMulRowsLargeint16-stmt.json | 2 +- .../stmt/nsertColumnsAndTagNumLarge4096-stmt.json | 2 +- tests/pytest/tools/taosdemoAllTest/subInsertdata.json | 2 +- tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json | 2 +- tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json | 2 +- tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json | 2 +- tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json | 2 +- .../pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json | 2 +- .../taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json | 2 +- .../tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json | 2 +- .../tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json | 2 +- tests/pytest/tools/taosdemoPerformance.py | 2 +- tests/pytest/tsdb/insertDataDb1.json | 2 +- tests/pytest/tsdb/insertDataDb1Replica2.json | 2 +- tests/pytest/tsdb/insertDataDb2.json | 2 +- tests/pytest/tsdb/insertDataDb2Newstab.json | 2 +- tests/pytest/tsdb/insertDataDb2NewstabReplica2.json | 2 +- tests/pytest/tsdb/insertDataDb2Replica2.json | 2 +- tests/pytest/util/taosdemoCfg.py | 2 +- tests/pytest/wal/insertDataDb1.json | 2 +- tests/pytest/wal/insertDataDb1Replica2.json | 2 +- tests/pytest/wal/insertDataDb2.json | 2 +- tests/pytest/wal/insertDataDb2Newstab.json | 2 +- tests/pytest/wal/insertDataDb2NewstabReplica2.json | 2 +- tests/pytest/wal/insertDataDb2Replica2.json | 2 +- tests/system-test/1-insert/manyVgroups.json | 2 +- tests/system-test/1-insert/performanceInsert.json | 2 +- 125 files changed, 125 insertions(+), 125 deletions(-) diff --git a/source/libs/sync/test/sh/insert.tpl.json b/source/libs/sync/test/sh/insert.tpl.json index 1d952b98e8..631e490a2a 100644 --- a/source/libs/sync/test/sh/insert.tpl.json +++ b/source/libs/sync/test/sh/insert.tpl.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 8, - "thread_count_create_tbl": 8, + "create_table_thread_count": 8, "result_file": "./tpl_insert_result_tpl", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/cluster/TD-3693/insert1Data.json b/tests/pytest/cluster/TD-3693/insert1Data.json index 6900ce0366..ad83a35160 100644 --- a/tests/pytest/cluster/TD-3693/insert1Data.json +++ b/tests/pytest/cluster/TD-3693/insert1Data.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/cluster/TD-3693/insert2Data.json b/tests/pytest/cluster/TD-3693/insert2Data.json index e55fa996fb..86495f0ce9 100644 --- a/tests/pytest/cluster/TD-3693/insert2Data.json +++ b/tests/pytest/cluster/TD-3693/insert2Data.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/dockerCluster/insert.json b/tests/pytest/dockerCluster/insert.json index 32e1043c4e..ce8d7978fa 100644 --- a/tests/pytest/dockerCluster/insert.json +++ b/tests/pytest/dockerCluster/insert.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "databases": [{ diff --git a/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json b/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json index dc9de1626a..4b622c3f28 100644 --- a/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json +++ b/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/perfbenchmark/bug3433.py b/tests/pytest/perfbenchmark/bug3433.py index 7f2dfad403..3e7de39bed 100644 --- a/tests/pytest/perfbenchmark/bug3433.py +++ b/tests/pytest/perfbenchmark/bug3433.py @@ -185,7 +185,7 @@ class TDTestCase: "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/perfbenchmark/joinPerformance.py b/tests/pytest/perfbenchmark/joinPerformance.py index b85c09926a..d30bec6664 100644 --- a/tests/pytest/perfbenchmark/joinPerformance.py +++ b/tests/pytest/perfbenchmark/joinPerformance.py @@ -168,7 +168,7 @@ class JoinPerf: "user": self.user, "password": self.password, "thread_count": cpu_count(), - "thread_count_create_tbl": cpu_count(), + "create_table_thread_count": cpu_count(), "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/perfbenchmark/taosdemoInsert.py b/tests/pytest/perfbenchmark/taosdemoInsert.py index 774103aa85..a23797a62b 100644 --- a/tests/pytest/perfbenchmark/taosdemoInsert.py +++ b/tests/pytest/perfbenchmark/taosdemoInsert.py @@ -172,7 +172,7 @@ class Taosdemo: "user": self.user, "password": self.password, "thread_count": cpu_count(), - "thread_count_create_tbl": cpu_count(), + "create_table_thread_count": cpu_count(), "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/query/nestedQuery/insertData.json b/tests/pytest/query/nestedQuery/insertData.json index 1aad170bb0..18a843015c 100644 --- a/tests/pytest/query/nestedQuery/insertData.json +++ b/tests/pytest/query/nestedQuery/insertData.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/query/query1970YearsAf.py b/tests/pytest/query/query1970YearsAf.py index 6a5c0796ed..e7e9fa5329 100644 --- a/tests/pytest/query/query1970YearsAf.py +++ b/tests/pytest/query/query1970YearsAf.py @@ -133,7 +133,7 @@ class TDTestCase: "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "/tmp/insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-interlace.json b/tests/pytest/tools/insert-interlace.json index 0e17edf8fd..8d96c20fe7 100644 --- a/tests/pytest/tools/insert-interlace.json +++ b/tests/pytest/tools/insert-interlace.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 5000, diff --git a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json index bbac60872e..e50e67943e 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset-createdb.json +++ b/tests/pytest/tools/insert-tblimit-tboffset-createdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json index 8f795338d2..fe4945483c 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json +++ b/tests/pytest/tools/insert-tblimit-tboffset-insertrec.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit-tboffset.json b/tests/pytest/tools/insert-tblimit-tboffset.json index 2c2d86c481..92b28241a6 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset.json +++ b/tests/pytest/tools/insert-tblimit-tboffset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit-tboffset0.json b/tests/pytest/tools/insert-tblimit-tboffset0.json index ce83ea3e60..0c1e00976b 100644 --- a/tests/pytest/tools/insert-tblimit-tboffset0.json +++ b/tests/pytest/tools/insert-tblimit-tboffset0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert-tblimit1-tboffset.json b/tests/pytest/tools/insert-tblimit1-tboffset.json index b15aaf4eed..ff002e9528 100644 --- a/tests/pytest/tools/insert-tblimit1-tboffset.json +++ b/tests/pytest/tools/insert-tblimit1-tboffset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/insert.json b/tests/pytest/tools/insert.json index 523561dc6d..4489730722 100644 --- a/tests/pytest/tools/insert.json +++ b/tests/pytest/tools/insert.json @@ -7,7 +7,7 @@ "password": "taosdata", "thread_count": 2, "num_of_records_per_req": 10, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "databases": [{ "dbinfo": { "name": "db01", diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json index a11261681a..3c876c61c7 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertMSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json index 080231551e..b9162242d4 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertNanoDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json index fe0ecbe2de..3fbaeceeba 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoInsertUSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json index 1af2952a69..6b0631da39 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabase.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json index 39c5e49909..bf9b015154 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseInsertForSub.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json index f4dbf1ee41..346fe31be9 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabaseNow.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json index 84b511a446..65a2836a49 100644 --- a/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json +++ b/tests/pytest/tools/taosdemoAllTest/NanoTestCase/taosdemoTestNanoDatabasecsv.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json index 75dbcb4432..b7b6c186e6 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-3453/query-interrupt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json index 0c2e9cf34a..edb9ed7cb8 100644 --- a/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-4985/query-limit-offset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json index e90474e872..b1d7dc4935 100755 --- a/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json +++ b/tests/pytest/tools/taosdemoAllTest/TD-5213/insertSigcolumnsNum4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json b/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json index 21603b1902..c1c27cf6d7 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-1s1tnt1r.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json b/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json index c944c26915..360ec07370 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-1s1tntmr.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-disorder.json b/tests/pytest/tools/taosdemoAllTest/insert-disorder.json index 4908d3999c..930496a877 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-disorder.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-disorder.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json index 03f531f52b..12dadf8006 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-N00.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json index ce2a34627b..759a3f074d 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-drop-exist-auto-Y00.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-illegal.json b/tests/pytest/tools/taosdemoAllTest/insert-illegal.json index 6e438b33df..321495782d 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-illegal.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-illegal.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json b/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json index 54e646a5a0..5dd37ee8b0 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-interlace-row.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json b/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json index 9a47a873dd..7fbee6fee0 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-interval-speed.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 100, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-newdb.json b/tests/pytest/tools/taosdemoAllTest/insert-newdb.json index 2eb17b1aab..16e1f94481 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-newdb.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-newdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-newtable.json b/tests/pytest/tools/taosdemoAllTest/insert-newtable.json index abe277bf5b..86c9359ffb 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-newtable.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-newtable.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json b/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json index 2dae7eb1d7..7eee9ce55b 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-nodbnodrop.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-offset.json b/tests/pytest/tools/taosdemoAllTest/insert-offset.json index 642d01db3e..d3946cee3c 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-offset.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-offset.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json b/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json index 3ef4360aef..c812b4971e 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-renewdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-sample.json b/tests/pytest/tools/taosdemoAllTest/insert-sample.json index 5b25281e78..e24e20067c 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-sample.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-sample.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert-timestep.json b/tests/pytest/tools/taosdemoAllTest/insert-timestep.json index 6432fde4ba..ceadfc677a 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert-timestep.json +++ b/tests/pytest/tools/taosdemoAllTest/insert-timestep.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json b/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json index 4e59d86679..69ebe45e50 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json +++ b/tests/pytest/tools/taosdemoAllTest/insertBinaryLenLarge16374AllcolLar49151.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json b/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json index 80d6817b5d..8b7086530e 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertChildTab0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json b/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json index a35c28f0ac..1e052ff2a4 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertChildTabLess0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json index 05d47c3611..c67b1dba14 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json +++ b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNum4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json index e63b3613ba..25e43aefa7 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json +++ b/tests/pytest/tools/taosdemoAllTest/insertColumnsAndTagNumLarge4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json b/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json index 137e608386..af04d9c1a3 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertColumnsNum0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json b/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json index 63a4a2ab58..84a5fe9452 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json +++ b/tests/pytest/tools/taosdemoAllTest/insertInterlaceRowsLarge1M.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json b/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json index f3212bc30d..d092a41483 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json +++ b/tests/pytest/tools/taosdemoAllTest/insertMaxNumPerReq.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json index 9711ead80e..45523618f0 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReq0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json index 24c61cfa8c..a95c40f9eb 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json +++ b/tests/pytest/tools/taosdemoAllTest/insertNumOfrecordPerReqless0.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertRestful.json b/tests/pytest/tools/taosdemoAllTest/insertRestful.json index ab7ee9a73b..26770c3d09 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertRestful.json +++ b/tests/pytest/tools/taosdemoAllTest/insertRestful.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json b/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json index d835822e8f..74737b4dec 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json +++ b/tests/pytest/tools/taosdemoAllTest/insertSigcolumnsNum4096.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json b/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json index 4c7cdfe39d..e0e9f72a56 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json +++ b/tests/pytest/tools/taosdemoAllTest/insertTagsNumLarge128.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json b/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json index b563dcc94b..fdc1994782 100644 --- a/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json +++ b/tests/pytest/tools/taosdemoAllTest/insertTimestepMulRowsLargeint16.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json b/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json index 0f1a874cc3..91d6c1a837 100644 --- a/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json +++ b/tests/pytest/tools/taosdemoAllTest/insert_5M_rows.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json index bdab459987..45a718705a 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_block1_comp.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_block2.json b/tests/pytest/tools/taosdemoAllTest/manual_block2.json index 763421c7f3..f01e55fb53 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_block2.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_block2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json index 0579aedf69..f097f15ee1 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_A.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json index d541cb6567..2df1fc42aa 100644 --- a/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json +++ b/tests/pytest/tools/taosdemoAllTest/manual_change_time_1_1_B.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json index c134391a5f..be1df2030f 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit1.json @@ -7,7 +7,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json index e9f759f8f7..a8552404d5 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit5.json @@ -7,7 +7,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json index 9b46ff105b..316fbba4a0 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-limit94.json @@ -7,7 +7,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json index fdcaa131e6..d03b29d90f 100644 --- a/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json +++ b/tests/pytest/tools/taosdemoAllTest/moredemo-offset-newdb.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/query-interrupt.json b/tests/pytest/tools/taosdemoAllTest/query-interrupt.json index 01028f68ad..1b276cb2b0 100644 --- a/tests/pytest/tools/taosdemoAllTest/query-interrupt.json +++ b/tests/pytest/tools/taosdemoAllTest/query-interrupt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json b/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json index 0fc789c7e3..8565e4a711 100644 --- a/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json +++ b/tests/pytest/tools/taosdemoAllTest/queryInsertdata.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json b/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json index 940adfb61c..0f9be9bdc3 100644 --- a/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json +++ b/tests/pytest/tools/taosdemoAllTest/queryInsertrestdata.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json index a4baf73689..443da39fa1 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json index a7a514e9dc..bd5709ca5e 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-large-taosc.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json index 3c38f92680..209f414c1b 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt-random.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json index 2ee489c7a3..903c8a9c93 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json index 44da22aa3f..dcbec40034 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/1174-small-taosc.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 1, + "create_table_thread_count": 1, "result_file": "1174.out", "confirm_parameter_prompt": "no", "num_of_records_per_req": 51, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json index b2805a38e5..1ea4de5cfe 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tnt1r-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json index ac540befb6..86f2fa6c4d 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-1s1tntmr-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json index 9a7ad93636..d634ab8369 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-disorder-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json index 919b918395..4b69118ef5 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-N00-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json index dcf52931ad..32043996b6 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-drop-exist-auto-Y00-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json index d2304ed537..a1a0b89e48 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interlace-row-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json index d297240613..f5cea2ccc3 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-interval-speed-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 100, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json index d117c5b345..c3bdea61c6 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newdb-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json index 1b36b3cbe9..e92644d33e 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-newtable-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json index ea95736a00..0618c04b30 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-nodbnodrop-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json index 8318de6672..356ac38d14 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-offset-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json index b6cb47f2c5..2f8f693166 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-renewdb-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json index 348e93ff8b..c1da95ba8c 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-sample-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json index edbaae60a1..9522f0e7b5 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insert-timestep-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file":"./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json index 1c72b4f402..bcbda0a301 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertBinaryLenLarge16374AllcolLar49151-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json index 4626babd95..2b30aa3e9e 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTab0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json index f140883de1..f3c577b30c 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertChildTabLess0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json index d1d2db2df3..a0ff887250 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsAndTagNum4096-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json index d79d4cace5..5ff9ec63a2 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertColumnsNum0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json index eb0ab0f04a..79ce66097b 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertInterlaceRowsLarge1M-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json index 489632c645..4b21f0a184 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertMaxNumPerReq-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json index 19eb92bf4c..9fb85aef23 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReq0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json index dbda4f74a1..80944de3f5 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertNumOfrecordPerReqless0-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json index 966c285d2f..834ffb56d3 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertSigcolumnsNum4096-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json index c1fc02553f..f39aa94830 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertTagsNumLarge128-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json index ed3eb280f6..6345227788 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/insertTimestepMulRowsLargeint16-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json b/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json index 1d7ad8a90e..75a365bbff 100644 --- a/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json +++ b/tests/pytest/tools/taosdemoAllTest/stmt/nsertColumnsAndTagNumLarge4096-stmt.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/subInsertdata.json b/tests/pytest/tools/taosdemoAllTest/subInsertdata.json index 1ca302a320..f5e7ac3018 100644 --- a/tests/pytest/tools/taosdemoAllTest/subInsertdata.json +++ b/tests/pytest/tools/taosdemoAllTest/subInsertdata.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json b/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json index ef63546278..896a72598d 100644 --- a/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json +++ b/tests/pytest/tools/taosdemoAllTest/subInsertdataMaxsql100.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json index b6e5847b54..8211a92a2d 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertMSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json index ed97fea33e..304ff99c26 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertNanoDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json index db34bfc6b8..444e6564be 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoInsertUSDB.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json index d029ddea21..67003a1fb5 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabase.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json index f8a181d352..7454af6521 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseInsertForSub.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json index b06ec55ef6..602a39ca24 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabaseNow.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json index 6a6a6da297..79d3bc5ed8 100644 --- a/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json +++ b/tests/pytest/tools/taosdemoAllTest/taosdemoTestNanoDatabasecsv.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 10, + "create_table_thread_count": 10, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tools/taosdemoPerformance.py b/tests/pytest/tools/taosdemoPerformance.py index 82c57a656d..9a4b564319 100644 --- a/tests/pytest/tools/taosdemoPerformance.py +++ b/tests/pytest/tools/taosdemoPerformance.py @@ -94,7 +94,7 @@ class taosdemoPerformace: "user": "root", "password": "taosdata", "thread_count": 10, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "databases": [db] } diff --git a/tests/pytest/tsdb/insertDataDb1.json b/tests/pytest/tsdb/insertDataDb1.json index 92735dad69..f771551b26 100644 --- a/tests/pytest/tsdb/insertDataDb1.json +++ b/tests/pytest/tsdb/insertDataDb1.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb1Replica2.json b/tests/pytest/tsdb/insertDataDb1Replica2.json index a5fc525157..ec84d71d88 100644 --- a/tests/pytest/tsdb/insertDataDb1Replica2.json +++ b/tests/pytest/tsdb/insertDataDb1Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2.json b/tests/pytest/tsdb/insertDataDb2.json index 02301e0242..494465d23c 100644 --- a/tests/pytest/tsdb/insertDataDb2.json +++ b/tests/pytest/tsdb/insertDataDb2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2Newstab.json b/tests/pytest/tsdb/insertDataDb2Newstab.json index 2f5f2367b4..647a587cad 100644 --- a/tests/pytest/tsdb/insertDataDb2Newstab.json +++ b/tests/pytest/tsdb/insertDataDb2Newstab.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json b/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json index 67f3b2cd4f..13cf2e561c 100644 --- a/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json +++ b/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/tsdb/insertDataDb2Replica2.json b/tests/pytest/tsdb/insertDataDb2Replica2.json index 3d033f13cc..c651657a6d 100644 --- a/tests/pytest/tsdb/insertDataDb2Replica2.json +++ b/tests/pytest/tsdb/insertDataDb2Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/util/taosdemoCfg.py b/tests/pytest/util/taosdemoCfg.py index 7523a80898..f708d303de 100644 --- a/tests/pytest/util/taosdemoCfg.py +++ b/tests/pytest/util/taosdemoCfg.py @@ -50,7 +50,7 @@ class TDTaosdemoCfg: "user": "root", "password": "taosdata", "thread_count": cpu_count(), - "thread_count_create_tbl": cpu_count(), + "create_table_thread_count": cpu_count(), "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb1.json b/tests/pytest/wal/insertDataDb1.json index a14fe58141..2dc0cf2b7f 100644 --- a/tests/pytest/wal/insertDataDb1.json +++ b/tests/pytest/wal/insertDataDb1.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb1Replica2.json b/tests/pytest/wal/insertDataDb1Replica2.json index a5fc525157..ec84d71d88 100644 --- a/tests/pytest/wal/insertDataDb1Replica2.json +++ b/tests/pytest/wal/insertDataDb1Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2.json b/tests/pytest/wal/insertDataDb2.json index 891a21f73e..35232a6333 100644 --- a/tests/pytest/wal/insertDataDb2.json +++ b/tests/pytest/wal/insertDataDb2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2Newstab.json b/tests/pytest/wal/insertDataDb2Newstab.json index 2f5f2367b4..647a587cad 100644 --- a/tests/pytest/wal/insertDataDb2Newstab.json +++ b/tests/pytest/wal/insertDataDb2Newstab.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2NewstabReplica2.json b/tests/pytest/wal/insertDataDb2NewstabReplica2.json index 67f3b2cd4f..13cf2e561c 100644 --- a/tests/pytest/wal/insertDataDb2NewstabReplica2.json +++ b/tests/pytest/wal/insertDataDb2NewstabReplica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/pytest/wal/insertDataDb2Replica2.json b/tests/pytest/wal/insertDataDb2Replica2.json index 3d033f13cc..c651657a6d 100644 --- a/tests/pytest/wal/insertDataDb2Replica2.json +++ b/tests/pytest/wal/insertDataDb2Replica2.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 4, - "thread_count_create_tbl": 4, + "create_table_thread_count": 4, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json index 20ac320552..3b0fa96b08 100644 --- a/tests/system-test/1-insert/manyVgroups.json +++ b/tests/system-test/1-insert/manyVgroups.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 8, - "thread_count_create_tbl": 8, + "create_table_thread_count": 8, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, diff --git a/tests/system-test/1-insert/performanceInsert.json b/tests/system-test/1-insert/performanceInsert.json index de410c30f2..7278a6f735 100644 --- a/tests/system-test/1-insert/performanceInsert.json +++ b/tests/system-test/1-insert/performanceInsert.json @@ -6,7 +6,7 @@ "user": "root", "password": "taosdata", "thread_count": 8, - "thread_count_create_tbl": 8, + "create_table_thread_count": 8, "result_file": "./insert_res.txt", "confirm_parameter_prompt": "no", "insert_interval": 0, From fdf977f9355c658e2a5670ac20e46ffd68c1590c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 6 Aug 2022 11:30:06 +0800 Subject: [PATCH 14/32] fix: add checks for stream query --- source/libs/parser/src/parTranslater.c | 56 ++++++++++++++++++-------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index db14a4f1c3..280e84990c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1254,19 +1254,20 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p return TSDB_CODE_SUCCESS; } if (isSelectStmt(pCxt->pCurrStmt)) { - // select percentile() without from clause is also valid - if (NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { - return TSDB_CODE_SUCCESS; - } - SNode* pTable = ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable; - if (QUERY_NODE_REAL_TABLE == nodeType(pTable) && - (TSDB_CHILD_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType || - TSDB_NORMAL_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType)) { - return TSDB_CODE_SUCCESS; - } + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, + "%s is only supported in single table query", pFunc->functionName); } - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, - "%s is only supported in single table query", pFunc->functionName); + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + SNode* pTable = pSelect->pFromTable; + // select percentile() without from clause is also valid + if ((NULL != pTable && (QUERY_NODE_REAL_TABLE != nodeType(pTable) || + (TSDB_CHILD_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType && + TSDB_NORMAL_TABLE != ((SRealTableNode*)pTable)->pMeta->tableType))) || + NULL != pSelect->pPartitionByList) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, + "%s is only supported in single table query", pFunc->functionName); + } + return TSDB_CODE_SUCCESS; } static bool isStar(SNode* pNode) { @@ -2509,9 +2510,31 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static int32_t translateStateWindow(STranslateContext* pCxt, SStateWindowNode* pState) { +static bool isPartitionByTbname(SNodeList* pPartitionByList) { + if (1 != LIST_LENGTH(pPartitionByList)) { + return false; + } + SNode* pPartKey = nodesListGetNode(pPartitionByList, 0); + return QUERY_NODE_FUNCTION != nodeType(pPartKey) || FUNCTION_TYPE_TBNAME != ((SFunctionNode*)pPartKey)->funcType; +} + +static int32_t checkStateWindowForStream(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (!pCxt->createStream) { + return TSDB_CODE_SUCCESS; + } + if (TSDB_SUPER_TABLE == ((SRealTableNode*)pSelect->pFromTable)->pMeta->tableType && + !isPartitionByTbname(pSelect->pPartitionByList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); + } + return TSDB_CODE_SUCCESS; +} + +static int32_t translateStateWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { + SStateWindowNode* pState = (SStateWindowNode*)pSelect->pWindow; nodesWalkExprPostOrder(pState->pExpr, checkStateExpr, pCxt); - // todo check for "function not support for state_window" + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + pCxt->errCode = checkStateWindowForStream(pCxt, pSelect); + } return pCxt->errCode; } @@ -2522,14 +2545,13 @@ static int32_t translateSessionWindow(STranslateContext* pCxt, SSessionWindowNod if (PRIMARYKEY_TIMESTAMP_COL_ID != pSession->pCol->colId) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SESSION_COL); } - // todo check for "function not support for session" return TSDB_CODE_SUCCESS; } static int32_t translateSpecificWindow(STranslateContext* pCxt, SSelectStmt* pSelect) { switch (nodeType(pSelect->pWindow)) { case QUERY_NODE_STATE_WINDOW: - return translateStateWindow(pCxt, (SStateWindowNode*)pSelect->pWindow); + return translateStateWindow(pCxt, pSelect); case QUERY_NODE_SESSION_WINDOW: return translateSessionWindow(pCxt, (SSessionWindowNode*)pSelect->pWindow); case QUERY_NODE_INTERVAL_WINDOW: @@ -4708,7 +4730,7 @@ static int32_t checkCreateStream(STranslateContext* pCxt, SCreateStreamStmt* pSt } } - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY); + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } static void getSourceDatabase(SNode* pStmt, int32_t acctId, char* pDbFName) { From d5d873c7f89c7744eef055b1bb290fdfe7c92959 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 11:32:29 +0800 Subject: [PATCH 15/32] fix(query): fix interp fill(prev) issue --- source/libs/executor/src/timewindowoperator.c | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 8a0564c129..4cbe9e467f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2087,7 +2087,7 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, - int32_t rowIndex, SSDataBlock* pResBlock) { + int32_t rowIndex, SSDataBlock* pResBlock, bool isPrevRowSet) { int32_t rows = pResBlock->info.rows; // todo set the correct primary timestamp column @@ -2163,6 +2163,10 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } case TSDB_FILL_PREV: { + if (!isPrevRowSet) { + break; + } + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; @@ -2170,6 +2174,10 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } case TSDB_FILL_NEXT: { + if (!isPrevRowSet) { + break; + } + char* p = colDataGetData(pSrc, rowIndex); colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); pResBlock->info.rows += 1; @@ -2235,6 +2243,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { blockDataCleanup(pResBlock); //int32_t numOfRows = 0; + bool isPrevRowSet = false; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -2269,6 +2278,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2285,7 +2295,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2302,10 +2312,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } else { // it is the last row of current block doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; } } else { // ts > pSliceInfo->current while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2329,6 +2340,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2347,7 +2359,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { //check if need to interpolate after ts range while (pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock, isPrevRowSet); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { From 457cae2561754579cd01f42fe576773961c27105 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 6 Aug 2022 11:46:05 +0800 Subject: [PATCH 16/32] os: fix win sync compile error --- source/libs/sync/inc/syncInt.h | 20 ++++++++++---------- source/libs/sync/src/syncMain.c | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 82399f52b9..0548dc6c63 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -262,20 +262,20 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); // trace log -void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); -void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); +FORCE_INLINE void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); +FORCE_INLINE void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); -void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); -void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); +FORCE_INLINE void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); +FORCE_INLINE void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); -void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); -void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); +FORCE_INLINE void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); +FORCE_INLINE void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); -void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); -void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); +FORCE_INLINE void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); +FORCE_INLINE void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); -void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); -void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); +FORCE_INLINE void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); +FORCE_INLINE void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 48ebf824e8..71b0afa0e0 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2965,7 +2965,7 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) { return true; } -inline void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { +FORCE_INLINE void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); @@ -2976,7 +2976,7 @@ inline void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { +FORCE_INLINE void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { char logBuf[256]; char host[64]; uint16_t port; @@ -2987,7 +2987,7 @@ inline void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { +FORCE_INLINE void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); @@ -2997,7 +2997,7 @@ inline void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestV syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { +FORCE_INLINE void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); @@ -3007,7 +3007,7 @@ inline void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestV syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { +FORCE_INLINE void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); @@ -3022,7 +3022,7 @@ inline void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntri syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { +FORCE_INLINE void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); @@ -3037,7 +3037,7 @@ inline void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntri syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { +FORCE_INLINE void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); @@ -3050,7 +3050,7 @@ inline void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppend syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { +FORCE_INLINE void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); @@ -3063,7 +3063,7 @@ inline void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppend syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { +FORCE_INLINE void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); @@ -3075,7 +3075,7 @@ inline void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppend syncNodeEventLog(pSyncNode, logBuf); } -inline void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { +FORCE_INLINE void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { char host[64]; uint16_t port; syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); From 240a4a197c58e297476fa35456f90fde521658ce Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 6 Aug 2022 13:10:01 +0800 Subject: [PATCH 17/32] os: fix win sync compile error --- source/libs/sync/inc/syncInt.h | 128 +++++++++++++++++++++++++++++--- source/libs/sync/src/syncMain.c | 122 ------------------------------ 2 files changed, 118 insertions(+), 132 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 0548dc6c63..d75fcf10c0 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -262,20 +262,128 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); // trace log -FORCE_INLINE void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); -FORCE_INLINE void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); -FORCE_INLINE void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); -FORCE_INLINE void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); +static FORCE_INLINE void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-request-vote to %s:%d {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, port, + pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + syncNodeEventLog(pSyncNode, logBuf); +} -FORCE_INLINE void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); -FORCE_INLINE void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); +static FORCE_INLINE void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, + port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + syncNodeEventLog(pSyncNode, logBuf); +} -FORCE_INLINE void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); -FORCE_INLINE void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); +static FORCE_INLINE void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "send sync-request-vote-reply to %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, + pMsg->term, pMsg->voteGranted, s); + syncNodeEventLog(pSyncNode, logBuf); +} -FORCE_INLINE void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); -FORCE_INLINE void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); +static FORCE_INLINE void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "recv sync-request-vote-reply from %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, + port, pMsg->term, pMsg->voteGranted, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +static FORCE_INLINE void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", pterm:%" PRIu64 ", cmt:%" PRId64 + ", " + "datalen:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, + pMsg->dataLen, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +static FORCE_INLINE void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries from %s:%d {term:%" PRIu64 ", pre-index:%" PRIu64 ", pre-term:%" PRIu64 + ", cmt:%" PRIu64 ", pterm:%" PRIu64 + ", " + "datalen:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, + pMsg->dataLen, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +static FORCE_INLINE void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, + pMsg->dataLen, pMsg->dataCount, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +static FORCE_INLINE void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries-batch from %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, + pMsg->dataLen, pMsg->dataCount, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +static FORCE_INLINE void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "}, %s", + host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +static FORCE_INLINE void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "}, %s", + host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); + syncNodeEventLog(pSyncNode, logBuf); +} // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 71b0afa0e0..36afa3f278 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2964,125 +2964,3 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) { return true; } - -FORCE_INLINE void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-request-vote to %s:%d {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, port, - pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { - char logBuf[256]; - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - snprintf(logBuf, sizeof(logBuf), - "recv sync-request-vote from %s:%d, {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "send sync-request-vote-reply to %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, - pMsg->term, pMsg->voteGranted, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "recv sync-request-vote-reply from %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, - port, pMsg->term, pMsg->voteGranted, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 - ", " - "datalen:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, - pMsg->dataLen, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries from %s:%d {term:%" PRIu64 ", pre-index:%" PRIu64 ", pre-term:%" PRIu64 - ", cmt:%" PRIu64 ", pterm:%" PRIu64 - ", " - "datalen:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, - pMsg->dataLen, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, - pMsg->dataLen, pMsg->dataCount, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch from %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, - pMsg->dataLen, pMsg->dataCount, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 - "}, %s", - host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -FORCE_INLINE void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 - "}, %s", - host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); - syncNodeEventLog(pSyncNode, logBuf); -} From e628b70726cbf566e60f2676264b2109f636690f Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 6 Aug 2022 13:17:08 +0800 Subject: [PATCH 18/32] refactor(sync): make leader life longer --- source/dnode/vnode/src/vnd/vnodeSync.c | 4 ++-- source/libs/sync/src/syncMain.c | 5 ++++- source/libs/sync/src/syncRequestVote.c | 6 ++++++ source/libs/sync/test/sh/a.sh | 2 +- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 1586143319..a269f81ddd 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -708,8 +708,8 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { } setPingTimerMS(pVnode->sync, 5000); - setElectTimerMS(pVnode->sync, 2800); - setHeartbeatTimerMS(pVnode->sync, 900); + setElectTimerMS(pVnode->sync, 4000); + setHeartbeatTimerMS(pVnode->sync, 700); return 0; } diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 48ebf824e8..daa65add3c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1270,6 +1270,8 @@ int32_t syncNodeStopElectTimer(SSyncNode* pSyncNode) { atomic_add_fetch_64(&pSyncNode->electTimerLogicClockUser, 1); taosTmrStop(pSyncNode->pElectTimer); pSyncNode->pElectTimer = NULL; + + sTrace("vgId:%d, sync %s stop elect timer", pSyncNode->vgId, syncUtilState2String(pSyncNode->state)); return ret; } @@ -1343,7 +1345,8 @@ int32_t syncNodeStopHeartbeatTimer(SSyncNode* pSyncNode) { atomic_add_fetch_64(&pSyncNode->heartbeatTimerLogicClockUser, 1); taosTmrStop(pSyncNode->pHeartbeatTimer); pSyncNode->pHeartbeatTimer = NULL; - sTrace("vgId:%d, stop heartbeat timer", pSyncNode->vgId); + + sTrace("vgId:%d, sync %s stop heartbeat timer", pSyncNode->vgId, syncUtilState2String(pSyncNode->state)); return ret; } diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index b47294a8e1..122a81930b 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -57,11 +57,14 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); +#if 0 if (logOK) { syncNodeUpdateTerm(ths, pMsg->term); } else { syncNodeUpdateTermWithoutStepDown(ths, pMsg->term); } +#endif } ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); @@ -167,11 +170,14 @@ int32_t syncNodeOnRequestVoteSnapshotCb(SSyncNode* ths, SyncRequestVote* pMsg) { // maybe update term if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); +#if 0 if (logOK) { syncNodeUpdateTerm(ths, pMsg->term); } else { syncNodeUpdateTermWithoutStepDown(ths, pMsg->term); } +#endif } ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); diff --git a/source/libs/sync/test/sh/a.sh b/source/libs/sync/test/sh/a.sh index 3983d30b7c..f4ffaa5062 100644 --- a/source/libs/sync/test/sh/a.sh +++ b/source/libs/sync/test/sh/a.sh @@ -89,7 +89,7 @@ for file in `ls ${logpath}/log.dnode*.vgId*.commit`;do line=`cat ${file} | tail -n1` echo $line | awk '{print $5, $0}' >> ${tmpfile} done -cat ${tmpfile} | sort -k1 > ${logpath}/log.commits +cat ${tmpfile} | sort -k1 | awk 'BEGIN{vgid=$1}{if($1==vgid){print $0}else{print ""; print $0; vgid=$1;}}END{}' > ${logpath}/log.commits exit 0 From d42c9b0eaeb581d2009e7f2dec132e3f00e5620d Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 6 Aug 2022 13:44:01 +0800 Subject: [PATCH 19/32] os: fix win sync compile error --- source/libs/sync/inc/syncInt.h | 128 +++----------------------------- source/libs/sync/src/syncMain.c | 122 ++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 118 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index d75fcf10c0..82399f52b9 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -262,128 +262,20 @@ int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); // trace log +void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); +void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); -static FORCE_INLINE void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-request-vote to %s:%d {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, port, - pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); - syncNodeEventLog(pSyncNode, logBuf); -} +void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); +void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s); -static FORCE_INLINE void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { - char logBuf[256]; - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - snprintf(logBuf, sizeof(logBuf), - "recv sync-request-vote from %s:%d, {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, - port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); - syncNodeEventLog(pSyncNode, logBuf); -} +void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); +void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s); -static FORCE_INLINE void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "send sync-request-vote-reply to %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, - pMsg->term, pMsg->voteGranted, s); - syncNodeEventLog(pSyncNode, logBuf); -} +void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); +void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s); -static FORCE_INLINE void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), "recv sync-request-vote-reply from %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, - port, pMsg->term, pMsg->voteGranted, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -static FORCE_INLINE void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 - ", " - "datalen:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, - pMsg->dataLen, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -static FORCE_INLINE void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries from %s:%d {term:%" PRIu64 ", pre-index:%" PRIu64 ", pre-term:%" PRIu64 - ", cmt:%" PRIu64 ", pterm:%" PRIu64 - ", " - "datalen:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, - pMsg->dataLen, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -static FORCE_INLINE void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, - pMsg->dataLen, pMsg->dataCount, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -static FORCE_INLINE void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-batch from %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 - ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", - host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, - pMsg->dataLen, pMsg->dataCount, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -static FORCE_INLINE void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 - "}, %s", - host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); - syncNodeEventLog(pSyncNode, logBuf); -} - -static FORCE_INLINE void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { - char host[64]; - uint16_t port; - syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); - char logBuf[256]; - snprintf(logBuf, sizeof(logBuf), - "recv sync-append-entries-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 - "}, %s", - host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); - syncNodeEventLog(pSyncNode, logBuf); -} +void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); +void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s); // for debug -------------- void syncNodePrint(SSyncNode* pObj); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 36afa3f278..9320e42ec5 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2964,3 +2964,125 @@ bool syncNodeCanChange(SSyncNode* pSyncNode) { return true; } + +void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-request-vote to %s:%d {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, port, + pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s) { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, {term:%" PRIu64 ", lindex:%" PRId64 ", lterm:%" PRIu64 "}, %s", host, + port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogSendRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "send sync-request-vote-reply to %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, port, + pMsg->term, pMsg->voteGranted, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogRecvRequestVoteReply(SSyncNode* pSyncNode, const SyncRequestVoteReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), "recv sync-request-vote-reply from %s:%d {term:%" PRIu64 ", grant:%d}, %s", host, + port, pMsg->term, pMsg->voteGranted, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogSendAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-append-entries to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", pterm:%" PRIu64 ", cmt:%" PRId64 + ", " + "datalen:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, + pMsg->dataLen, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogRecvAppendEntries(SSyncNode* pSyncNode, const SyncAppendEntries* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries from %s:%d {term:%" PRIu64 ", pre-index:%" PRIu64 ", pre-term:%" PRIu64 + ", cmt:%" PRIu64 ", pterm:%" PRIu64 + ", " + "datalen:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->commitIndex, pMsg->privateTerm, + pMsg->dataLen, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogSendAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-append-entries-batch to %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, + pMsg->dataLen, pMsg->dataCount, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogRecvAppendEntriesBatch(SSyncNode* pSyncNode, const SyncAppendEntriesBatch* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries-batch from %s:%d, {term:%" PRIu64 ", pre-index:%" PRId64 ", pre-term:%" PRIu64 + ", pterm:%" PRIu64 ", cmt:%" PRId64 ", datalen:%d, count:%d}, %s", + host, port, pMsg->term, pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->privateTerm, pMsg->commitIndex, + pMsg->dataLen, pMsg->dataCount, s); + syncNodeEventLog(pSyncNode, logBuf); +} + + void syncLogSendAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->destId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "send sync-append-entries-reply to %s:%d, {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "}, %s", + host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); + syncNodeEventLog(pSyncNode, logBuf); +} + +void syncLogRecvAppendEntriesReply(SSyncNode* pSyncNode, const SyncAppendEntriesReply* pMsg, const char* s) { + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + char logBuf[256]; + snprintf(logBuf, sizeof(logBuf), + "recv sync-append-entries-reply from %s:%d {term:%" PRIu64 ", pterm:%" PRIu64 ", success:%d, match:%" PRId64 + "}, %s", + host, port, pMsg->term, pMsg->privateTerm, pMsg->success, pMsg->matchIndex, s); + syncNodeEventLog(pSyncNode, logBuf); +} From 4f074c929371a4a1200cf8f81dadfc8d53d78906 Mon Sep 17 00:00:00 2001 From: Yang Zhao Date: Sat, 6 Aug 2022 14:37:42 +0800 Subject: [PATCH 20/32] docs: update taosBenchmark doc (#15796) * docs: update taosbenchmark doc in 3.0 * docs: update taos-tools doc * docs: update taosbenchmark * Update 05-taosbenchmark.md --- docs/zh/14-reference/05-taosbenchmark.md | 35 +++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/zh/14-reference/05-taosbenchmark.md b/docs/zh/14-reference/05-taosbenchmark.md index f4aff0169d..f84ec65b4c 100644 --- a/docs/zh/14-reference/05-taosbenchmark.md +++ b/docs/zh/14-reference/05-taosbenchmark.md @@ -233,11 +233,28 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **drop** : 插入前是否删除数据库，默认为 true。 +#### 流式计算相关配置参数 + +创建流式计算的相关参数在 json 配置文件中的 `stream` 中配置，具体参数如下。 + +- **stream_name** : 流式计算的名称，必填项。 + +- **stream_stb** : 流式计算对应的超级表名称，必填项。 + +- **stream_sql** : 流式计算的sql语句，必填项。 + +- **trigger_mode** : 流式计算的触发模式，可选项。 + +- **watermark** : 流式计算的水印，可选项。 + +- **drop** : 是否创建流式计算，可选项为 "yes" 或者 "no", 为 "no" 时不创建。 + #### 超级表相关配置参数 -创建超级表时的相关参数在 json 配置文件中的 `super_tables` 中配置，具体参数如下表。 +创建超级表时的相关参数在 json 配置文件中的 `super_tables` 中配置，具体参数如下。 - **name**: 超级表名，必须配置，没有默认值。 + - **child_table_exists** : 子表是否已经存在，默认值为 "no"，可选值为 "yes" 或 "no"。 - **child_table_count** : 子表的数量，默认值为 10。 @@ -288,6 +305,22 @@ taosBenchmark -A INT,DOUBLE,NCHAR,BINARY\(16\) - **tags_file** : 仅当 insert_mode 为 taosc, rest 的模式下生效。 最终的 tag 的数值与 childtable_count 有关，如果 csv 文件内的 tag 数据行小于给定的子表数量，那么会循环读取 csv 文件数据直到生成 childtable_count 指定的子表数量；否则则只会读取 childtable_count 行 tag 数据。也即最终生成的子表数量为二者取小。 +#### tsma配置参数 + +指定tsma的配置参数在 `super_tables` 中的 `tsmas` 中，具体参数如下。 + +- **name** : 指定 tsma 的名字，必选项。 + +- **function** : 指定 tsma 的函数，必选项。 + +- **interval** : 指定 tsma 的时间间隔，必选项。 + +- **sliding** : 指定 tsma 的窗口时间位移，必选项。 + +- **custom** : 指定 tsma 的创建语句结尾追加的自定义配置，可选项。 + +- **start_when_inserted** : 指定当插入多少行时创建 tsma，可选项，默认为 0。 + #### 标签列与数据列配置参数 指定超级表标签列与数据列的配置参数分别在 `super_tables` 中的 `columns` 和 `tag` 中。 From 70f396799397ee754316b12c605924444fb4ec70 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 14:42:48 +0800 Subject: [PATCH 21/32] fix(query): fix interp + fill(prev) result error --- source/libs/executor/src/timewindowoperator.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 4cbe9e467f..1eee2c197e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2291,6 +2291,10 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } else if (ts < pSliceInfo->current) { + //in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate + doKeepPrevRows(pSliceInfo, pBlock, i); + isPrevRowSet = true; + if (i < pBlock->info.rows - 1) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { From 214d3609b2971ee41e9a9d4277a18bc6d4117b9f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 6 Aug 2022 10:04:12 +0800 Subject: [PATCH 22/32] fix(stream):set num of stream session child --- include/common/tcommon.h | 1 + include/common/tmsg.h | 1 + source/common/src/tdatablock.c | 3 +- source/libs/executor/src/executil.c | 1 + source/libs/executor/src/executorimpl.c | 2 +- source/libs/executor/src/timewindowoperator.c | 42 +++++-------------- source/libs/stream/src/streamData.c | 1 + source/libs/stream/src/streamDispatch.c | 1 + 8 files changed, 17 insertions(+), 35 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 4eea744be1..7d78c2dc2f 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -107,6 +107,7 @@ typedef struct SDataBlockInfo { int32_t childId; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize STimeWindow calWin; // used for stream, do not serialize + TSKEY watermark;// used for stream } SDataBlockInfo; typedef struct SSDataBlock { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index bfb80ec8f8..ed4bcd0584 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1369,6 +1369,7 @@ typedef struct { int64_t skey; int64_t ekey; int64_t version; // for stream + TSKEY watermark;// for stream char data[]; } SRetrieveTableRsp; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 51c21eafa9..302874962e 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1272,8 +1272,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { colDataAssign(pDst, pSrc, src->info.rows, &src->info); } - dst->info.rows = src->info.rows; - dst->info.capacity = src->info.rows; + dst->info = src->info; return 0; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 6bbfca804f..34247d3b47 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -191,6 +191,7 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { pBlock->info.blockId = pNode->dataBlockId; pBlock->info.type = STREAM_INVALID; pBlock->info.calWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pBlock->info.watermark = INT64_MIN; for (int32_t i = 0; i < numOfCols; ++i) { SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c1a3d19b6d..eaffbc2579 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4180,7 +4180,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t children = 0; pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION == type) { - int32_t children = 1; + int32_t children = pHandle->numOfVgroups; pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 8a0564c129..b7868358f9 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1752,30 +1752,11 @@ void increaseTs(SqlFunctionCtx* pCtx) { } } -SSDataBlock* createDeleteBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.hasVarCol = false; - pBlock->info.groupId = 0; - pBlock->info.rows = 0; - pBlock->info.type = STREAM_DELETE_RESULT; - pBlock->info.rowSize = sizeof(TSKEY) + sizeof(uint64_t); - - pBlock->pDataBlock = taosArrayInit(2, sizeof(SColumnInfoData)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; - infoData.info.bytes = sizeof(TSKEY); - // window start ts - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_UBIGINT; - infoData.info.bytes = sizeof(uint64_t); - taosArrayPush(pBlock->pDataBlock, &infoData); - - return pBlock; -} - void initIntervalDownStream(SOperatorInfo* downstream, uint8_t type, SAggSupporter* pSup) { - ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + // Todo(liuyao) support partition by column + return; + } SStreamScanInfo* pScanInfo = downstream->info; pScanInfo->sessionSup.parentType = type; pScanInfo->sessionSup.pIntervalAggSup = pSup; @@ -2872,13 +2853,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { // process the rest of the data return pInfo->pUpdateRes; } - // doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); - // if (pInfo->pPullDataRes->info.rows != 0) { - // // process the rest of the data - // ASSERT(IS_FINAL_OP(pInfo)); - // printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); - // return pInfo->pPullDataRes; - // } doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); if (pInfo->pDelRes->info.rows != 0) { // process the rest of the data @@ -2898,6 +2872,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv"); maxTs = TMAX(maxTs, pBlock->info.window.ekey); + maxTs = TMAX(maxTs, pBlock->info.watermark); if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA || pBlock->info.type == STREAM_INVALID) { @@ -2986,6 +2961,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, pUpdated, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); + } else { + pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; } finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); @@ -3020,7 +2997,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->pDelRes; } - // ASSERT(false); return NULL; } @@ -3032,6 +3008,7 @@ SSDataBlock* createSpecialDataBlock(EStreamType type) { pBlock->info.type = type; pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); + pBlock->info.watermark = INT64_MIN; pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); SColumnInfoData infoData = {0}; @@ -3221,7 +3198,6 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { SStreamSessionAggOperatorInfo* pChInfo = pChild->info; destroyStreamSessionAggOperatorInfo(pChInfo, numOfOutput); taosMemoryFreeClear(pChild); - taosMemoryFreeClear(pChInfo); } } colDataDestroy(&pInfo->twAggSup.timeWindowData); @@ -3986,6 +3962,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true); } maxTs = TMAX(maxTs, pBlock->info.window.ekey); + maxTs = TMAX(maxTs, pBlock->info.watermark); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); @@ -4109,6 +4086,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; // restore the value pOperator->status = OP_RES_TO_RETURN; // semi operator diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 0bf6d4c921..54014d7df9 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -35,6 +35,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); pDataBlock->info.version = be64toh(pRetrieve->version); + pDataBlock->info.watermark = be64toh(pRetrieve->watermark); pDataBlock->info.type = pRetrieve->streamBlockType; pDataBlock->info.childId = pReq->upstreamChildId; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 66e689dd3e..8d6d31e37f 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -184,6 +184,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->skey = htobe64(pBlock->info.window.skey); pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); + pRetrieve->watermark = htobe64(pBlock->info.watermark); int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); From 5750525f9a66e5f5441a465abd094529615e987d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 6 Aug 2022 15:25:53 +0800 Subject: [PATCH 23/32] fix: condition race error --- source/libs/parser/src/parTranslater.c | 2 +- source/libs/planner/src/planner.c | 7 +++++-- source/libs/scheduler/src/schTask.c | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 280e84990c..9971f20d3d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1253,7 +1253,7 @@ static int32_t translateRepeatScanFunc(STranslateContext* pCxt, SFunctionNode* p if (!fmIsRepeatScanFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } - if (isSelectStmt(pCxt->pCurrStmt)) { + if (!isSelectStmt(pCxt->pCurrStmt)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_SUPPORT_SINGLE_TABLE, "%s is only supported in single table query", pFunc->functionName); } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 1f531b0708..c1296982e0 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -85,7 +85,7 @@ static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDown } int32_t qSetSubplanExecutionNode(SSubplan* subplan, int32_t groupId, SDownstreamSourceNode* pSource) { - planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.groupId, groupId); + planDebug("QID:0x%" PRIx64 " set subplan execution node, groupId:%d", subplan->id.queryId, groupId); return setSubplanExecutionNode(subplan->pNode, groupId, pSource); } @@ -104,7 +104,10 @@ static void clearSubplanExecutionNode(SPhysiNode* pNode) { FOREACH(pChild, pNode->pChildren) { clearSubplanExecutionNode((SPhysiNode*)pChild); } } -void qClearSubplanExecutionNode(SSubplan* pSubplan) { clearSubplanExecutionNode(pSubplan->pNode); } +void qClearSubplanExecutionNode(SSubplan* pSubplan) { + planDebug("QID:0x%" PRIx64 " clear subplan execution node, groupId:%d", pSubplan->id.queryId, pSubplan->id.groupId); + clearSubplanExecutionNode(pSubplan->pNode); +} int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType && NULL == pSubplan->pNode) { diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index ecd0253e1c..978a72cf34 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -284,7 +284,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { for (int32_t i = 0; i < parentNum; ++i) { SSchTask *parent = *(SSchTask **)taosArrayGet(pTask->parents, i); - int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); SCH_LOCK(SCH_WRITE, &parent->planLock); SDownstreamSourceNode source = { @@ -298,6 +297,8 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { qSetSubplanExecutionNode(parent->plan, pTask->plan->id.groupId, &source); SCH_UNLOCK(SCH_WRITE, &parent->planLock); + int32_t readyNum = atomic_add_fetch_32(&parent->childReady, 1); + if (SCH_TASK_READY_FOR_LAUNCH(readyNum, parent)) { SCH_TASK_DLOG("all %d children task done, start to launch parent task 0x%" PRIx64, readyNum, parent->taskId); SCH_ERR_RET(schLaunchTask(pJob, parent)); From f09e48ca0b9928ba3bd9a0b90ee3fdf22ce3434c Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 6 Aug 2022 15:28:43 +0800 Subject: [PATCH 24/32] ci: map taosrest into container for arm --- tests/parallel_test/run_case.sh | 2 ++ tests/parallel_test/run_container.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 52a5627e34..eda66a884a 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -49,11 +49,13 @@ if [ $ent -eq 0 ]; then export PATH=$PATH:/home/TDengine/debug/build/bin export LD_LIBRARY_PATH=/home/TDengine/debug/build/lib ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + ln -s /home/TDengine/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null CONTAINER_TESTDIR=/home/TDengine else export PATH=$PATH:/home/TDinternal/debug/build/bin export LD_LIBRARY_PATH=/home/TDinternal/debug/build/lib ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so 2>/dev/null + ln -s /home/TDinternal/debug/build/lib/libtaos.so /usr/lib/libtaos.so.1 2>/dev/null CONTAINER_TESTDIR=/home/TDinternal/community fi mkdir -p /var/lib/taos/subscribe diff --git a/tests/parallel_test/run_container.sh b/tests/parallel_test/run_container.sh index affd9128a4..f0ee9be46f 100755 --- a/tests/parallel_test/run_container.sh +++ b/tests/parallel_test/run_container.sh @@ -100,6 +100,7 @@ docker run \ -v "$TMP_DIR/thread_volume/$thread_no/sim:${SIM_DIR}" \ -v ${TMP_DIR}/thread_volume/$thread_no/coredump:$coredump_dir \ -v $WORKDIR/taos-connector-python/taos:/usr/local/lib/python3.8/site-packages/taos:ro \ + -v $WORKDIR/taos-connector-python/taosrest:/usr/local/lib/python3.8/site-packages/taosrest:ro \ --rm --ulimit core=-1 taos_test:v1.0 $CONTAINER_TESTDIR/tests/parallel_test/run_case.sh -d "$exec_dir" -c "$cmd" $extra_param ret=$? exit $ret From a17f46a59c7aa331ae099a90a49cf4b0df202482 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 15:43:19 +0800 Subject: [PATCH 25/32] fix: fix msgIdx mismatch issue --- source/libs/catalog/inc/catalogInt.h | 22 ++-- source/libs/catalog/src/ctgAsync.c | 174 +++++++++++++++----------- source/libs/catalog/src/ctgRemote.c | 179 +++++++++++++++++---------- source/libs/catalog/src/ctgUtil.c | 8 +- 4 files changed, 235 insertions(+), 148 deletions(-) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c452f61b75..4ef23860cd 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -239,6 +239,7 @@ typedef struct SCtgBatch { SRequestConnInfo conn; char dbFName[TSDB_DB_FNAME_LEN]; SArray* pTaskIds; + SArray* pMsgIdxs; } SCtgBatch; typedef struct SCtgJob { @@ -287,6 +288,7 @@ typedef struct SCtgTaskCallbackParam { SArray* taskId; int32_t reqType; int32_t batchId; + SArray* msgIdx; } SCtgTaskCallbackParam; @@ -314,12 +316,16 @@ typedef struct SCtgTask { SArray* pParents; SCtgSubRes subRes; SHashObj* pBatchs; - int32_t msgIdx; } SCtgTask; +typedef struct SCtgTaskReq { + SCtgTask* pTask; + int32_t msgIdx; +} SCtgTaskReq; + typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*); typedef int32_t (*ctgLanchTaskFp)(SCtgTask*); -typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTask*, int32_t, const SDataBuf *, int32_t); +typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTaskReq*, int32_t, const SDataBuf *, int32_t); typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*); typedef int32_t (*ctgCloneTaskResFp)(SCtgTask*, void**); typedef int32_t (*ctgCompTaskFp)(SCtgTask*, void*, bool*); @@ -518,7 +524,7 @@ typedef struct SCtgOperation { #define CTG_FLAG_MAKE_STB(_isStb) (((_isStb) == 1) ? CTG_FLAG_STB : ((_isStb) == 0 ? CTG_FLAG_NOT_STB : CTG_FLAG_UNKNOWN_STB)) #define CTG_FLAG_MATCH_STB(_flag, tbType) (CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || (CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE)) -#define CTG_GET_TASK_MSGCTX(_task) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_task)->msgIdx) : &(_task)->msgCtx) +#define CTG_GET_TASK_MSGCTX(_task, _id) (((CTG_TASK_GET_TB_META_BATCH == (_task)->type) || (CTG_TASK_GET_TB_HASH_BATCH == (_task)->type)) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx) #define CTG_META_SIZE(pMeta) (sizeof(STableMeta) + ((pMeta)->tableInfo.numOfTags + (pMeta)->tableInfo.numOfColumns) * sizeof(SSchema)) @@ -665,7 +671,7 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); -int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask); +int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTaskReq* tReq); int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray **out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); @@ -673,9 +679,9 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *funcName, SFuncInfo *out, SCtgTask* pTask); int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *user, SGetUserAuthRsp *out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask); -int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTaskReq* tReq); +int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTaskReq* tReq); +int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTaskReq* tReq); int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask); int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask); int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, char **out, SCtgTask* pTask); @@ -698,7 +704,7 @@ void ctgFreeJob(void* job); void ctgFreeHandleImpl(SCatalog* pCtg); void ctgFreeVgInfo(SDBVgInfo *vgInfo); int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName *pTableName, SVgroupInfo *pVgroup); -int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update); +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache *dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index bbca85ed14..63d99cc58b 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -381,7 +381,10 @@ int32_t ctgInitGetTbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - if (NULL == pDb) { + SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + if (NULL == pDb || NULL == pTb) { + taosHashCleanup(pDb); + taosHashCleanup(pTb); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } @@ -400,18 +403,26 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); } - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + int32_t dbNum = taosArrayGetSize(pReq->pTableMeta); + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pTableMeta, i); + taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + int32_t tbNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < tbNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(name, dbFName); - taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + + dbNum = taosArrayGetSize(pReq->pTableHash); + for (int32_t i = 0; i < dbNum; ++i) { + STablesReq* p = taosArrayGet(pReq->pTableHash, i); + taosHashPut(pDb, p->dbFName, strlen(p->dbFName), p->dbFName, TSDB_DB_FNAME_LEN); + int32_t tbNum = taosArrayGetSize(p->pTables); + for (int32_t m = 0; m < tbNum; ++m) { + SName* name = taosArrayGet(p->pTables, m); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { @@ -430,16 +441,6 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashCleanup(pDb); // REFRESH TABLE META - SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { SName* name = taosArrayGet(pReq->pTableCfg, i); @@ -924,12 +925,13 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbMetaRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDBCache *dbCache = NULL; + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; SName* pName = ctx->pName; int32_t flag = ctx->flag; @@ -947,7 +949,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); return TSDB_CODE_SUCCESS; } @@ -967,7 +969,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); ctgReleaseVgInfoToCache(pCtg, dbCache); } else { @@ -976,7 +978,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } return TSDB_CODE_SUCCESS; @@ -1014,7 +1016,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * taosMemoryFreeClear(pOut->tbMeta); - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } else if (CTG_IS_META_BOTH(pOut->metaType)) { int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { @@ -1033,7 +1035,7 @@ int32_t ctgHandleGetTbMetaRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf * if (0 == exist) { TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, tReq)); } } break; @@ -1088,14 +1090,15 @@ _return: } -int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbMetasRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; SCtgDBCache *dbCache = NULL; + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); SName* pName = ctgGetFetchName(ctx->pNames, pFetch); int32_t flag = pFetch->flag; int32_t* vgId = &pFetch->vgId; @@ -1112,7 +1115,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); return TSDB_CODE_SUCCESS; } @@ -1132,7 +1135,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); ctgReleaseVgInfoToCache(pCtg, dbCache); } else { @@ -1141,7 +1144,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } return TSDB_CODE_SUCCESS; @@ -1179,7 +1182,7 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf taosMemoryFreeClear(pOut->tbMeta); - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } else if (CTG_IS_META_BOTH(pOut->metaType)) { int32_t exist = 0; if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) { @@ -1192,13 +1195,14 @@ int32_t ctgHandleGetTbMetasRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf taosMemoryFreeClear(pOut->tbMeta); CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &stbCtx, &pOut->tbMeta)); if (pOut->tbMeta) { + ctgDebug("use cached stb meta, tbName:%s", tNameGetTableName(pName)); exist = 1; } } if (0 == exist) { TSWAP(pMsgCtx->lastOut, pMsgCtx->out); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pOut->dbFName, pOut->tbName, NULL, tReq)); } } break; @@ -1268,8 +1272,9 @@ _return: } -int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbVgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgDbVgCtx* ctx = (SCtgDbVgCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1301,8 +1306,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbHashRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgTbHashCtx* ctx = (SCtgTbHashCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; @@ -1338,12 +1344,13 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbHashsRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbHashsRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgTbHashsCtx* ctx = (SCtgTbHashsCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; - SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask); - SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx); + SCtgFetch* pFetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target)); @@ -1352,7 +1359,7 @@ int32_t ctgHandleGetTbHashsRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out; STablesReq* pReq = taosArrayGet(ctx->pNames, pFetch->dbIdx); - CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, tReq, pOut->dbVgroup, ctx, pMsgCtx->target, pReq->pTables, true)); CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, pMsgCtx->target, pOut->dbId, pOut->dbVgroup, false)); pOut->dbVgroup = NULL; @@ -1393,8 +1400,9 @@ _return: } -int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); STableIndex* pOut = (STableIndex*)pTask->msgCtx.out; @@ -1415,8 +1423,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetTbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetTbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1428,8 +1437,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbCfgRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1441,13 +1451,14 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDbInfoRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDbInfoRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { CTG_RET(TSDB_CODE_APP_ERROR); } -int32_t ctgHandleGetQnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetQnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1459,8 +1470,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetDnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetDnodeRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1472,8 +1484,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetIndexRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1485,8 +1498,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetUdfRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetUdfRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); TSWAP(pTask->res, pTask->msgCtx.out); @@ -1498,8 +1512,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetUserRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; bool pass = false; @@ -1542,8 +1557,9 @@ _return: CTG_RET(code); } -int32_t ctgHandleGetSvrVerRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { +int32_t ctgHandleGetSvrVerRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1556,7 +1572,8 @@ _return: CTG_RET(code); } -int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32_t* vgId) { +int32_t ctgAsyncRefreshTbMeta(SCtgTaskReq *tReq, int32_t flag, SName* pName, int32_t* vgId) { + SCtgTask* pTask = tReq->pTask; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; int32_t code = 0; @@ -1564,14 +1581,14 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32 if (CTG_FLAG_IS_SYS_DB(flag)) { ctgDebug("will refresh sys db tbmeta, tbName:%s", tNameGetTableName(pName)); - CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)pName->dbname, (char *)pName->tname, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, (char *)pName->dbname, (char *)pName->tname, NULL, tReq)); } if (CTG_FLAG_IS_STB(flag)) { ctgDebug("will refresh tbmeta, supposed to be stb, tbName:%s", tNameGetTableName(pName)); // if get from mnode failed, will not try vnode - CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, pTask)); + CTG_RET(ctgGetTbMetaFromMnode(pCtg, pConn, pName, NULL, tReq)); } SCtgDBCache *dbCache = NULL; @@ -1586,14 +1603,14 @@ int32_t ctgAsyncRefreshTbMeta(SCtgTask *pTask, int32_t flag, SName* pName, int32 ctgDebug("will refresh tbmeta, not supposed to be stb, tbName:%s, flag:%d", tNameGetTableName(pName), flag); *vgId = vgInfo.vgId; - CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, pTask)); + CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pName, &vgInfo, NULL, tReq)); } else { SBuildUseDBInput input = {0}; tstrncpy(input.db, dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq)); } _return: @@ -1616,7 +1633,10 @@ int32_t ctgLaunchGetTbMetaTask(SCtgTask *pTask) { } SCtgTbMetaCtx* pCtx = (SCtgTbMetaCtx*)pTask->taskCtx; - CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pCtx->flag, pCtx->pName, &pCtx->vgId)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pCtx->flag, pCtx->pName, &pCtx->vgId)); return TSDB_CODE_SUCCESS; } @@ -1651,11 +1671,11 @@ int32_t ctgLaunchGetTbMetasTask(SCtgTask *pTask) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); SName* pName = ctgGetFetchName(pCtx->pNames, pFetch); - pTask->msgIdx = pFetch->fetchIdx; - CTG_ERR_RET(ctgAsyncRefreshTbMeta(pTask, pFetch->flag, pName, &pFetch->vgId)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId)); } - - pTask->msgIdx = 0; return TSDB_CODE_SUCCESS; } @@ -1680,8 +1700,11 @@ int32_t ctgLaunchGetDbVgTask(SCtgTask *pTask) { tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } _return: @@ -1717,8 +1740,11 @@ int32_t ctgLaunchGetTbHashTask(SCtgTask *pTask) { tstrncpy(input.db, pCtx->dbFName, tListLen(input.db)); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } _return: @@ -1746,7 +1772,10 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, pReq->dbFName, &dbCache)); if (NULL != dbCache) { - CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, pTask, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_JRET(ctgGetVgInfosFromHashValue(pCtg, &tReq, dbCache->vgCache.vgInfo, pCtx, pReq->dbFName, pReq->pTables, false)); ctgReleaseVgInfoToCache(pCtg, dbCache); dbCache = NULL; @@ -1775,18 +1804,17 @@ int32_t ctgLaunchGetTbHashsTask(SCtgTask *pTask) { SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, i); STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx); - pTask->msgIdx = pFetch->fetchIdx; - SBuildUseDBInput input = {0}; strcpy(input.db, pReq->dbFName); input.vgVersion = CTG_DEFAULT_INVALID_VERSION; - CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pTask)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = pFetch->fetchIdx; + CTG_ERR_RET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, &tReq)); } - pTask->msgIdx = 0; - _return: if (dbCache) { diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index 660d91ea8a..a9f2d426bc 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -22,9 +22,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; - SArray* pTaskId = cbParam->taskId; SCatalog* pCtg = pJob->pCtg; - int32_t taskNum = taosArrayGetSize(pTaskId); + int32_t taskNum = taosArrayGetSize(cbParam->taskId); SDataBuf taskMsg = *pMsg; int32_t offset = 0; int32_t msgNum = (TSDB_CODE_SUCCESS == rspCode && pMsg->pData && (pMsg->len > 0)) ? ntohl(*(int32_t*)pMsg->pData) : 0; @@ -42,7 +41,8 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } for (int32_t i = 0; i < taskNum; ++i) { - int32_t* taskId = taosArrayGet(pTaskId, i); + int32_t* taskId = taosArrayGet(cbParam->taskId, i); + int32_t* msgIdx = taosArrayGet(cbParam->msgIdx, i); SCtgTask* pTask = taosArrayGet(pJob->pTasks, *taskId); if (msgNum > 0) { rsp.reqType = ntohl(*(int32_t*)((char*)pMsg->pData + offset)); @@ -59,8 +59,10 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu taskMsg.msgType = rsp.reqType; taskMsg.pData = rsp.msg; taskMsg.len = rsp.msgLen; + + ASSERT(rsp.msgIdx == *msgIdx); } else { - rsp.msgIdx = pTask->msgIdx++; + rsp.msgIdx = *msgIdx; rsp.reqType = -1; taskMsg.msgType = -1; taskMsg.pData = NULL; @@ -68,11 +70,14 @@ int32_t ctgHandleBatchRsp(SCtgJob* pJob, SCtgTaskCallbackParam* cbParam, SDataBu } pTask->pBatchs = pBatchs; - pTask->msgIdx = rsp.msgIdx; - ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, pTask->msgIdx, TMSG_INFO(taskMsg.msgType + 1)); - - (*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = rsp.msgIdx; + + ctgDebug("QID:0x%" PRIx64 " ctg task %d idx %d start to handle rsp %s", pJob->queryId, pTask->taskId, rsp.msgIdx, TMSG_INFO(taskMsg.msgType + 1)); + + (*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, rsp.reqType, &taskMsg, (rsp.rspCode ? rsp.rspCode : rspCode)); } CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -341,7 +346,10 @@ int32_t ctgHandleMsgCallback(void* param, SDataBuf* pMsg, int32_t rspCode) { pTask->pBatchs = pBatchs; #endif - CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(pTask, cbParam->reqType, pMsg, rspCode)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].handleRspFp)(&tReq, cbParam->reqType, pMsg, rspCode)); #if CTG_BATCH_FETCH CTG_ERR_JRET(ctgLaunchBatchs(pJob->pCtg, pJob, pBatchs)); @@ -359,7 +367,7 @@ _return: CTG_API_LEAVE(code); } -int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int32_t msgType, +int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, SArray* pMsgIdx, int32_t msgType, SMsgSendInfo** pMsgSendInfo) { int32_t code = 0; SMsgSendInfo* msgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); @@ -379,6 +387,7 @@ int32_t ctgMakeMsgSendInfo(SCtgJob* pJob, SArray* pTaskId, int32_t batchId, int3 param->refId = pJob->refId; param->taskId = pTaskId; param->batchId = batchId; + param->msgIdx = pMsgIdx; msgSendInfo->param = param; msgSendInfo->paramFreeFp = ctgFreeMsgSendParam; @@ -397,10 +406,10 @@ _return: } int32_t ctgAsyncSendMsg(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob* pJob, SArray* pTaskId, int32_t batchId, - char* dbFName, int32_t vgId, int32_t msgType, void* msg, uint32_t msgSize) { + SArray* pMsgIdx, char* dbFName, int32_t vgId, int32_t msgType, void* msg, uint32_t msgSize) { int32_t code = 0; SMsgSendInfo* pMsgSendInfo = NULL; - CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, msgType, &pMsgSendInfo)); + CTG_ERR_JRET(ctgMakeMsgSendInfo(pJob, pTaskId, batchId, pMsgIdx, msgType, &pMsgSendInfo)); ctgUpdateSendTargetInfo(pMsgSendInfo, msgType, dbFName, vgId); @@ -431,9 +440,10 @@ _return: CTG_RET(code); } -int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTask* pTask, int32_t msgType, void* msg, +int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgTaskReq* tReq, int32_t msgType, void* msg, uint32_t msgSize) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SHashObj* pBatchs = pTask->pBatchs; SCtgJob* pJob = pTask->pJob; SCtgBatch* pBatch = taosHashGet(pBatchs, &vgId, sizeof(vgId)); @@ -443,13 +453,14 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == pBatch) { newBatch.pMsgs = taosArrayInit(pJob->subTaskNum, sizeof(SBatchMsg)); newBatch.pTaskIds = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); - if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds) { + newBatch.pMsgIdxs = taosArrayInit(pJob->subTaskNum, sizeof(int32_t)); + if (NULL == newBatch.pMsgs || NULL == newBatch.pTaskIds || NULL == newBatch.pMsgIdxs) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } newBatch.conn = *pConn; - req.msgIdx = pTask->msgIdx; + req.msgIdx = tReq->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -459,6 +470,9 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == taosArrayPush(newBatch.pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + if (NULL == taosArrayPush(newBatch.pMsgIdxs, &req.msgIdx)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } newBatch.msgSize = sizeof(SBatchReq) + sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { @@ -469,7 +483,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; @@ -496,7 +510,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT return TSDB_CODE_SUCCESS; } - req.msgIdx = pTask->msgIdx; + req.msgIdx = tReq->msgIdx; req.msgType = msgType; req.msgLen = msgSize; req.msg = msg; @@ -506,6 +520,10 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT if (NULL == taosArrayPush(pBatch->pTaskIds, &pTask->taskId)) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } + if (NULL == taosArrayPush(pBatch->pMsgIdxs, &req.msgIdx)) { + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + pBatch->msgSize += sizeof(req) + msgSize - POINTER_BYTES; if (vgId > 0) { @@ -516,7 +534,7 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT } else if (TDMT_VND_TABLE_META == msgType) { if (CTG_TASK_GET_TB_META_BATCH == pTask->type) { SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx; - SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, pTask->msgIdx); + SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx); pName = ctgGetFetchName(ctx->pNames, fetch); } else { SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; @@ -588,8 +606,8 @@ int32_t ctgLaunchBatchs(SCatalog* pCtg, SCtgJob* pJob, SHashObj* pBatchs) { ctgDebug("QID:0x%" PRIx64 " ctg start to launch batch %d", pJob->queryId, pBatch->batchId); CTG_ERR_JRET(ctgBuildBatchReqMsg(pBatch, *vgId, &msg)); - code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->dbFName, *vgId, - pBatch->msgType, msg, pBatch->msgSize); + code = ctgAsyncSendMsg(pCtg, &pBatch->conn, pJob, pBatch->pTaskIds, pBatch->batchId, pBatch->pMsgIdxs, + pBatch->dbFName, *vgId, pBatch->msgType, msg, pBatch->msgSize); pBatch->pTaskIds = NULL; CTG_ERR_JRET(code); @@ -628,10 +646,13 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -639,7 +660,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -674,10 +695,13 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -685,7 +709,7 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -706,10 +730,11 @@ int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SArray } int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildUseDBInput* input, SUseDbOutput* out, - SCtgTask* pTask) { + SCtgTaskReq* tReq) { char* msg = NULL; int32_t msgLen = 0; int32_t reqType = TDMT_MND_USE_DB; + SCtgTask* pTask = tReq ? tReq->pTask : NULL; void* (*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; ctgDebug("try to get db vgInfo from mnode, dbFName:%s", input->db); @@ -726,10 +751,10 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, input->db)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, input->db)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -737,7 +762,7 @@ int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SBuildU } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -778,10 +803,13 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)dbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)dbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -789,7 +817,7 @@ int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -830,10 +858,13 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)indexName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)indexName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -841,7 +872,7 @@ int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -884,10 +915,13 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -895,7 +929,7 @@ int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName* n } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -936,10 +970,13 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)funcName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)funcName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -947,7 +984,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -988,10 +1025,13 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, (char*)user)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, pOut, (char*)user)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -999,7 +1039,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1020,7 +1060,8 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const } int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName, - STableMetaOutput* out, SCtgTask* pTask) { + STableMetaOutput* out, SCtgTaskReq* tReq) { + SCtgTask *pTask = tReq ? tReq->pTask : NULL; SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char* msg = NULL; SEpSet* pVnodeEpSet = NULL; @@ -1044,10 +1085,10 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1055,7 +1096,7 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1076,15 +1117,16 @@ int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* } int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out, - SCtgTask* pTask) { + SCtgTaskReq* tReq) { char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); - return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, pTask); + return ctgGetTbMetaFromMnodeImpl(pCtg, pConn, dbFName, (char*)pTableName->tname, out, tReq); } int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* vgroupInfo, - STableMetaOutput* out, SCtgTask* pTask) { + STableMetaOutput* out, SCtgTaskReq* tReq) { + SCtgTask *pTask = tReq ? tReq->pTask : NULL; char dbFName[TSDB_DB_FNAME_LEN]; tNameGetFullDbName(pTableName, dbFName); int32_t reqType = TDMT_VND_TABLE_META; @@ -1118,10 +1160,10 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, pOut, tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen)); #else SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; @@ -1132,7 +1174,7 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SNa } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->vgId, reqType, msg, msgLen)); #endif } @@ -1175,14 +1217,17 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, (char*)tbFName)); SRequestConnInfo vConn = {.pTrans = pConn->pTrans, .requestId = pConn->requestId, .requestObjRefId = pConn->requestObjRefId, .mgmtEps = vgroupInfo->epSet}; #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, &tReq, reqType, msg, msgLen)); #else SCtgTbCfgCtx* ctx = (SCtgTbCfgCtx*)pTask->taskCtx; char dbFName[TSDB_DB_FNAME_LEN]; @@ -1193,7 +1238,7 @@ int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, ctx->pVgInfo->vgId, reqType, msg, msgLen)); #endif } @@ -1234,10 +1279,13 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, (char*)tbFName)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, (char*)tbFName)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1245,7 +1293,7 @@ int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const S } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } @@ -1280,10 +1328,13 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou } if (pTask) { - CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask), reqType, NULL, NULL)); + CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, -1), reqType, NULL, NULL)); #if CTG_BATCH_FETCH - CTG_RET(ctgAddBatch(pCtg, 0, pConn, pTask, reqType, msg, msgLen)); + SCtgTaskReq tReq; + tReq.pTask = pTask; + tReq.msgIdx = -1; + CTG_RET(ctgAddBatch(pCtg, 0, pConn, &tReq, reqType, msg, msgLen)); #else SArray* pTaskId = taosArrayInit(1, sizeof(int32_t)); if (NULL == pTaskId) { @@ -1291,7 +1342,7 @@ int32_t ctgGetSvrVerFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, char** ou } taosArrayPush(pTaskId, &pTask->taskId); - CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, 0, reqType, msg, msgLen)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen)); #endif } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index de9ea7fb5a..8e5fb90f1a 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -26,6 +26,7 @@ void ctgFreeMsgSendParam(void* param) { SCtgTaskCallbackParam* pParam = (SCtgTaskCallbackParam*)param; taosArrayDestroy(pParam->taskId); + taosArrayDestroy(pParam->msgIdx); taosMemoryFree(param); } @@ -874,8 +875,9 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_RET(code); } -int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { +int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo *dbInfo, SCtgTbHashsCtx *pCtx, char* dbFName, SArray* pNames, bool update) { int32_t code = 0; + SCtgTask* pTask = tReq->pTask; SMetaRes res = {0}; int32_t vgNum = taosHashGetSize(dbInfo->vgHash); if (vgNum <= 0) { @@ -904,7 +906,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); pRes->pRes = vgInfo; } else { @@ -958,7 +960,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTask* pTask, SDBVgInfo *d vgInfo->epSet.eps[vgInfo->epSet.inUse].fqdn, vgInfo->epSet.eps[vgInfo->epSet.inUse].port); if (update) { - SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, pTask->msgIdx); + SCtgFetch* pFetch = taosArrayGet(pCtx->pFetchs, tReq->msgIdx); SMetaRes *pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx + i); pRes->pRes = pNewVg; } else { From 16a24f5a8452ae240a6390ff8b5219a7e3eb6331 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 6 Aug 2022 16:12:49 +0800 Subject: [PATCH 26/32] fix: add rsp msg size --- source/client/src/clientImpl.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 0c4dc1705c..47f6fef6a8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1308,8 +1308,8 @@ int32_t doProcessMsgFromServer(void* param) { char tbuf[40] = {0}; TRACE_TO_STR(trace, tbuf); - tscDebug("processMsgFromServer handle %p, message: %s, code: %s, gtid: %s", pMsg->info.handle, - TMSG_INFO(pMsg->msgType), tstrerror(pMsg->code), tbuf); + tscDebug("processMsgFromServer handle %p, message: %s, size:%d, code: %s, gtid: %s", pMsg->info.handle, + TMSG_INFO(pMsg->msgType), pMsg->contLen, tstrerror(pMsg->code), tbuf); if (pSendInfo->requestObjRefId != 0) { SRequestObj* pRequest = (SRequestObj*)taosAcquireRef(clientReqRefPool, pSendInfo->requestObjRefId); From 784752010afc5fac6aa8a7898530c929d6abe267 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 16:18:11 +0800 Subject: [PATCH 27/32] fix(query): fix interp fill(next) result errors --- source/libs/executor/inc/executorimpl.h | 3 + source/libs/executor/src/timewindowoperator.c | 85 +++++++++++++++---- 2 files changed, 72 insertions(+), 16 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index b62ff2bef1..f8a7ab330f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -737,6 +737,9 @@ typedef struct STimeSliceOperatorInfo { SInterval interval; int64_t current; SArray* pPrevRow; // SArray + SArray* pNextRow; // SArray + bool isPrevRowSet; + bool isNextRowSet; int32_t fillType; // fill type SColumn tsCol; // primary timestamp column SExprSupp scalarSup; // scalar calculation diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 1eee2c197e..9fcb255e9e 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2084,10 +2084,30 @@ static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock memcpy(pkey->pData, val, pkey->bytes); } } + + pSliceInfo->isPrevRowSet = true; +} + +static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + + // null data should not be kept since it can not be used to perform interpolation + if (!colDataIsNull_s(pColInfoData, i)) { + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i); + + pkey->isNull = false; + char* val = colDataGetData(pColInfoData, rowIndex); + memcpy(pkey->pData, val, pkey->bytes); + } + } + + pSliceInfo->isNextRowSet = true; } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, - int32_t rowIndex, SSDataBlock* pResBlock, bool isPrevRowSet) { + int32_t rowIndex, SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; // todo set the correct primary timestamp column @@ -2163,7 +2183,7 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } case TSDB_FILL_PREV: { - if (!isPrevRowSet) { + if (!pSliceInfo->isPrevRowSet) { break; } @@ -2174,12 +2194,14 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp } case TSDB_FILL_NEXT: { - if (!isPrevRowSet) { + if (!pSliceInfo->isNextRowSet) { break; } - char* p = colDataGetData(pSrc, rowIndex); - colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); + //char* p = colDataGetData(pSrc, rowIndex); + //colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); + colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; break; } @@ -2213,6 +2235,35 @@ static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pB taosArrayPush(pInfo->pPrevRow, &key); } + pInfo->isPrevRowSet = false; + + return TSDB_CODE_SUCCESS; +} + +static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { + if (pInfo->pNextRow != NULL) { + return TSDB_CODE_SUCCESS; + } + + pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys)); + if (pInfo->pNextRow == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SGroupKeys key = {0}; + key.bytes = pColInfo->info.bytes; + key.type = pColInfo->info.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pColInfo->info.bytes); + taosArrayPush(pInfo->pNextRow, &key); + } + + pInfo->isNextRowSet = false; + return TSDB_CODE_SUCCESS; } @@ -2242,15 +2293,19 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { blockDataCleanup(pResBlock); - //int32_t numOfRows = 0; - bool isPrevRowSet = false; while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { break; } - int32_t code = initPrevRowsKeeper(pSliceInfo, pBlock); + int32_t code; + code = initPrevRowsKeeper(pSliceInfo, pBlock); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + code = initNextRowsKeeper(pSliceInfo, pBlock); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -2272,13 +2327,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); char* v = colDataGetData(pSrc, i); - //colDataAppend(pDst, numOfRows, v, false); colDataAppend(pDst, pResBlock->info.rows, v, false); } pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2293,13 +2346,12 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } else if (ts < pSliceInfo->current) { //in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; if (i < pBlock->info.rows - 1) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2316,11 +2368,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } else { // it is the last row of current block doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; } } else { // ts > pSliceInfo->current + //in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i); + while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock, isPrevRowSet); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2344,7 +2398,6 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { pResBlock->info.rows += 1; doKeepPrevRows(pSliceInfo, pBlock, i); - isPrevRowSet = true; pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); @@ -2363,7 +2416,7 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { //check if need to interpolate after ts range while (pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock, isPrevRowSet); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { From 8c3cc0161adaaeff9267808dbe79186383e5441f Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 16:18:11 +0800 Subject: [PATCH 28/32] fix(query): fix interp fill(next) result errors --- source/libs/executor/src/timewindowoperator.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9fcb255e9e..f078bf4b56 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2344,8 +2344,9 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } else if (ts < pSliceInfo->current) { - //in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate + //in case interpolation window starts and ends between two datapoints, fill(prev), fill(next) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); + doKeepNextRows(pSliceInfo, pBlock, i + 1); if (i < pBlock->info.rows - 1) { int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); From 97794bd9d81b7bfe5a16e0a01fb034e1aa0b2c0d Mon Sep 17 00:00:00 2001 From: dingbo Date: Sat, 6 Aug 2022 16:28:51 +0800 Subject: [PATCH 29/32] docs: add releases.md --- .gitignore | 1 + docs/zh/28-releases.md | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 docs/zh/28-releases.md diff --git a/.gitignore b/.gitignore index c25f60075c..1c609d3ba3 100644 --- a/.gitignore +++ b/.gitignore @@ -118,3 +118,4 @@ contrib/* !contrib/test sql debug*/ +.env \ No newline at end of file diff --git a/docs/zh/28-releases.md b/docs/zh/28-releases.md new file mode 100644 index 0000000000..5f30325829 --- /dev/null +++ b/docs/zh/28-releases.md @@ -0,0 +1,9 @@ +--- +sidebar_label: 发布历史 +title: 发布历史 +--- + +import Release from "/components/Release"; + + + From d62f075a49722d735130ef01ae5e8971278f886e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 6 Aug 2022 16:42:17 +0800 Subject: [PATCH 30/32] chore: update libtaos ws 29424d5 for3.0 (#15804) * chore: add libtaos-ws for 3.0 * chore: update taosws-rs * chore: add libtaosws to install/remove script * chore: update taosws-rs * chore: update taosws-rs * chore: update taos-tools, taosws-rs for 3.0 * fix: packaging/tools/make_install.sh for 3.0 * chore: update taos-tools * chore: fix release script for 3.0 * chore: update taosws-rs for 3.0 * chore: add taows-rs submodule for 3.0 * chore: update taosws-rs for 3.0 * fix: install script support taosws for 3.0 * fix: script error handle for 3.0 * chore: update taosws-rs for 3.0 fix segfault * chore: change container_build for websocket build * fix: install script for taosws * fix: . * chore: update taosws-rs for 3.0 * chore: update taosws-rs for 3.0 * chore: update tools/CMakeLists.txt to allow compile taosws-rw on any platform * chore: taosws 648cc62 for 3.0 * chore: update taosws 29424d5 for 3.0 --- cmake/taosws_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index a2b45a5bfa..de6409a8c6 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taosws-rs.git - GIT_TAG 648cc62 + GIT_TAG 29424d5 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 78fb4f51b0824a4a3e1397fa3158d6f9ce52aa64 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 6 Aug 2022 16:18:11 +0800 Subject: [PATCH 31/32] fix(query): fix interp fill(next) result errors --- source/libs/executor/src/timewindowoperator.c | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index f078bf4b56..d12410869a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2107,7 +2107,7 @@ static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock } static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, - int32_t rowIndex, SSDataBlock* pResBlock) { + SSDataBlock* pResBlock) { int32_t rows = pResBlock->info.rows; // todo set the correct primary timestamp column @@ -2198,8 +2198,6 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp break; } - //char* p = colDataGetData(pSrc, rowIndex); - //colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot); colDataAppend(pDst, rows, pkey->pData, false); pResBlock->info.rows += 1; @@ -2344,15 +2342,16 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { break; } } else if (ts < pSliceInfo->current) { - //in case interpolation window starts and ends between two datapoints, fill(prev), fill(next) need to interpolate + // in case interpolation window starts and ends between two datapoints, fill(prev) need to interpolate doKeepPrevRows(pSliceInfo, pBlock, i); - doKeepNextRows(pSliceInfo, pBlock, i + 1); if (i < pBlock->info.rows - 1) { + // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + doKeepNextRows(pSliceInfo, pBlock, i + 1); int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); if (nextTs > pSliceInfo->current) { while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2371,11 +2370,11 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { doKeepPrevRows(pSliceInfo, pBlock, i); } } else { // ts > pSliceInfo->current - //in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate + // in case interpolation window starts and ends between two datapoints, fill(next) need to interpolate doKeepNextRows(pSliceInfo, pBlock, i); while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { @@ -2415,9 +2414,10 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } - //check if need to interpolate after ts range - while (pSliceInfo->current <= pSliceInfo->win.ekey) { - genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pBlock->info.rows - 1, pResBlock); + // check if need to interpolate after ts range + // except for fill(next) + while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, pResBlock); pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); if (pResBlock->info.rows >= pResBlock->info.capacity) { From d44bce1c6f64936106d2550f16550088e6b1008c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Sat, 6 Aug 2022 17:20:15 +0800 Subject: [PATCH 32/32] fix: update vnode's commitTerm --- source/dnode/vnode/src/vnd/vnodeCommit.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 650c98eb88..460d0b6fbb 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -223,6 +223,8 @@ int vnodeCommit(SVnode *pVnode) { vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; + pVnode->state.commitTerm = pVnode->state.applyTerm; + // save info info.config = pVnode->config; info.state.committed = pVnode->state.applied;