From 3e81daf578e09fd864ee1f35864820e25b2df00d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Jul 2022 17:28:37 +0800 Subject: [PATCH 01/13] fix(query): fix memory leak. --- include/libs/function/function.h | 6 +++ source/dnode/vnode/src/tsdb/tsdbRead.c | 15 +++++-- source/libs/executor/src/executil.c | 13 +++--- source/libs/scalar/inc/sclInt.h | 2 +- source/libs/scalar/src/filter.c | 11 ++++- source/libs/scalar/src/scalar.c | 59 +++++++++++++++----------- source/libs/scalar/src/sclvector.c | 14 +++--- 7 files changed, 76 insertions(+), 44 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index a569c8de54..7722ed7ba1 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -172,7 +172,13 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); +typedef enum { + CREATED_COLDATA = 0x1, // the newly created column data needs to be destroyed. + INPUT_COLDATA = 0x2, // input column data should not be released. +} SCOLDATA_TYPE_E; + struct SScalarParam { + SCOLDATA_TYPE_E type; SColumnInfoData *columnData; SHashObj *pHashFilter; int32_t hashValueType; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 53579474df..6116bc1a49 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2181,12 +2181,21 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret return VND_TSDB(pVnode); } -static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { +SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level) { + int64_t startVer = (pCond->startVersion == -1)? 0:pCond->startVersion; + if (VND_IS_RSMA(pVnode)) { - return (SVersionRange){.minVer = pCond->startVersion, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)}; + return (SVersionRange){.minVer = startVer, .maxVer = tdRSmaGetMaxSubmitVer(pVnode->pSma, level)}; } - return (SVersionRange){.minVer = pCond->startVersion, .maxVer = pVnode->state.applied}; + int64_t endVer = 0; + if (pCond->endVersion == -1) { // user not specified end version, set current maximum version of vnode as the endVersion + endVer = pVnode->state.applied; + } else { + endVer = (pCond->endVersion > pVnode->state.applied)? pVnode->state.applied:pCond->endVersion; + } + + return (SVersionRange){.minVer = startVer, .maxVer = endVer}; } // // todo not unref yet, since it is not support multi-group interpolation query diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 4e5458e620..a9cbb89eec 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -748,11 +748,12 @@ SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode) { SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.scale = pColNode->node.resType.scale; + + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; c.precision = pColNode->node.resType.precision; return c; } @@ -774,6 +775,8 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->suid = pTableScanNode->scan.suid; pCond->type = BLOCK_LOAD_OFFSET_ORDER; + pCond->startVersion = -1; + pCond->endVersion = -1; // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index c2d984c7dd..1794d8f5ed 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -57,7 +57,7 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); -SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows); +int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a7f66ebb7d..57819db0b8 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3827,13 +3827,20 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData SScalarParam output = {0}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows); + int32_t code = sclCreateColumnInfoData(&type, pSrc->info.rows, &output); + if (code != TSDB_CODE_SUCCESS) { + return code; + } SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); FLT_ERR_RET(scalarCalculate(info->sclCtx.node, pList, &output)); - *p = (int8_t *)output.columnData->pData; + *p = taosMemoryMalloc(output.numOfRows * sizeof(bool)); + + memcpy(*p, output.columnData->pData, output.numOfRows); + colDataDestroy(output.columnData); + taosMemoryFree(output.columnData); taosArrayDestroy(pList); return false; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index cbb1089d61..5c8e148e2d 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -35,12 +35,11 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { return TSDB_CODE_SUCCESS; } - -SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { +int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam) { SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pColumnData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return terrno; } pColumnData->info.type = pType->type; @@ -52,19 +51,25 @@ SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); - return NULL; - } else { - return pColumnData; + return terrno; } + + pParam->columnData = pColumnData; + pParam->type = CREATED_COLDATA; + return TSDB_CODE_SUCCESS; } int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { SScalarParam in = {.numOfRows = 1}; - in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1); + int32_t code = sclCreateColumnInfoData(&pValueNode->node.resType, 1, &in); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); colInfoDataEnsureCapacity(out->columnData, 1); - int32_t code = vectorConvertImpl(&in, out); + code = vectorConvertImpl(&in, out); sclFreeParam(&in); return code; @@ -190,8 +195,9 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t case QUERY_NODE_VALUE: { SValueNode *valueNode = (SValueNode *)node; + ASSERT(param->columnData == NULL); param->numOfRows = 1; - param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1); + /*int32_t code = */sclCreateColumnInfoData(&valueNode->node.resType, 1, param); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type || valueNode->isNull) { colDataAppendNULL(param->columnData, 0); } else { @@ -429,10 +435,9 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(code); } - output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); - if (output->columnData == NULL) { - sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + code = sclCreateColumnInfoData(&node->node.resType, rowNum, output); + if (code != TSDB_CODE_SUCCESS) { + SCL_ERR_JRET(code); } code = (*ffpSet.process)(params, paramNum, output); @@ -482,10 +487,9 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->numOfRows = rowNum; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - output->columnData = sclCreateColumnInfoData(&t, rowNum); - if (output->columnData == NULL) { - sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + code = sclCreateColumnInfoData(&t, rowNum, output); + if (code != TSDB_CODE_SUCCESS) { + SCL_ERR_JRET(code); } bool value = false; @@ -537,18 +541,19 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t code = 0; // json not support in in operator - if(nodeType(node->pLeft) == QUERY_NODE_VALUE){ + if (nodeType(node->pLeft) == QUERY_NODE_VALUE) { SValueNode *valueNode = (SValueNode *)node->pLeft; - if(valueNode->node.resType.type == TSDB_DATA_TYPE_JSON && (node->opType == OP_TYPE_IN || node->opType == OP_TYPE_NOT_IN)){ + if (valueNode->node.resType.type == TSDB_DATA_TYPE_JSON && (node->opType == OP_TYPE_IN || node->opType == OP_TYPE_NOT_IN)) { SCL_RET(TSDB_CODE_QRY_JSON_IN_ERROR); } } SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); - output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { - sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + code = sclCreateColumnInfoData(&node->node.resType, rowNum, output); + if (code != TSDB_CODE_SUCCESS) { + SCL_ERR_JRET(code); + } } _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(node->opType); @@ -563,7 +568,10 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp _return: for (int32_t i = 0; i < paramNum; ++i) { -// sclFreeParam(¶ms[i]); + if (params[i].type == CREATED_COLDATA) { + colDataDestroy(params[i].columnData); + taosMemoryFree(params[i].columnData); + } } taosMemoryFreeClear(params); @@ -766,7 +774,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { return sclRewriteNonConstOperator(pNode, ctx); } - SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; + SScalarParam output = {0}; ctx->code = sclExecOperator(node, ctx, &output); if (ctx->code) { return DEAL_RES_ERROR; @@ -1026,7 +1034,8 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; } - + + sclFreeParam(res); taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 18bbacd5b7..98cb923b49 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -865,12 +865,11 @@ int32_t vectorGetConvertType(int32_t type1, int32_t type2) { } int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int32_t type) { - int32_t code = 0; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; - output->columnData = sclCreateColumnInfoData(&t, input->numOfRows); - if (output->columnData == NULL) { + int32_t code = sclCreateColumnInfoData(&t, input->numOfRows, output); + if (code != TSDB_CODE_SUCCESS) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -940,13 +939,12 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows); - if (pDest->columnData == NULL) { - sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double))); - return TSDB_CODE_OUT_OF_MEMORY; + int32_t code = sclCreateColumnInfoData(&t, pParam->numOfRows, pDest); + if (code != TSDB_CODE_SUCCESS) { + return code; } - int32_t code = vectorConvertImpl(pParam, pDest); + code = vectorConvertImpl(pParam, pDest); if (code != TSDB_CODE_SUCCESS) { return code; } From 3cd5a35054c5de940e06710efa16dddab8546e05 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 8 Jul 2022 17:48:34 +0800 Subject: [PATCH 02/13] refactor(stream): internal refactor --- include/common/tcommon.h | 6 +- include/common/tdatablock.h | 2 +- include/libs/executor/executor.h | 8 +- include/libs/wal/wal.h | 1 + source/common/src/tdatablock.c | 2 +- source/dnode/mnode/impl/src/mndSubscribe.c | 6 +- source/dnode/vnode/inc/vnode.h | 3 + source/dnode/vnode/src/inc/tq.h | 1 + source/dnode/vnode/src/tq/tq.c | 17 +- source/dnode/vnode/src/tq/tqExec.c | 52 +- source/dnode/vnode/src/tq/tqRead.c | 24 +- source/libs/executor/inc/executorimpl.h | 748 +++++++++++---------- source/libs/executor/src/executor.c | 6 +- source/libs/executor/src/executorMain.c | 39 +- source/libs/executor/src/executorimpl.c | 5 +- source/libs/executor/src/scanoperator.c | 117 +++- source/libs/wal/src/walRead.c | 18 +- tests/script/jenkins/basic.txt | 2 +- tests/system-test/failed.txt | 1 + 19 files changed, 650 insertions(+), 408 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 63167bd834..f7097ef0fb 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -55,7 +55,8 @@ enum { enum { STREAM_INPUT__DATA_SUBMIT = 1, STREAM_INPUT__DATA_BLOCK, - STREAM_INPUT__DATA_SCAN, + STREAM_INPUT__TABLE_SCAN, + STREAM_INPUT__TQ_SCAN, STREAM_INPUT__DATA_RETRIEVE, STREAM_INPUT__TRIGGER, STREAM_INPUT__CHECKPOINT, @@ -124,7 +125,8 @@ enum { }; typedef struct { - int8_t fetchType; + int8_t fetchType; + STqOffsetVal offset; union { SSDataBlock data; void* meta; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 8b64287033..af333f72aa 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -231,7 +231,7 @@ SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); -SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); +SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 0094afef12..9b1fe1d9f5 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -174,7 +174,13 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le */ int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); -int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); +int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); + +int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset); + +int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); + +void* qStreamExtractMetaMsg(qTaskInfo_t tinfo); void* qExtractReaderFromStreamScanner(void* scanner); int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 43792b5415..e4189afb44 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -194,6 +194,7 @@ int32_t walRestoreFromSnapshot(SWal *, int64_t ver); SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); void walCloseReader(SWalReader *pRead); int32_t walReadVer(SWalReader *pRead, int64_t ver); +int32_t walReadSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); // only for tq usage diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 08275182af..923eb8bab7 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1356,7 +1356,7 @@ SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) return col; } -SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) { +SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index) { ASSERT(pBlock != NULL); if (index >= taosArrayGetSize(pBlock->pDataBlock)) { return NULL; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 8dde3e92d8..5f2d657b57 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -546,7 +546,11 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { char cgroup[TSDB_CGROUP_LEN]; mndSplitSubscribeKey(pRebInfo->key, topic, cgroup, true); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); - ASSERT(pTopic); + /*ASSERT(pTopic);*/ + if (pTopic == NULL) { + mError("rebalance %s failed since topic %s was dropped, abort", pRebInfo->key, topic); + continue; + } taosRLockLatch(&pTopic->lock); rebOutput.pSub = mndCreateSub(pMnode, pTopic, pRebInfo->key); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b381046ecf..c605fcc8a0 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -173,6 +173,9 @@ int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqSeekVer(STqReader *pReader, int64_t ver); +int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); + int32_t tqReaderSetDataMsg(STqReader *pReader, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReader *pReader); bool tqNextDataBlockFilterOut(STqReader *pReader, SHashObj *filterOutUids); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 9c3bd85c71..b2ea08d50d 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -129,6 +129,7 @@ typedef struct { static STqMgmt tqMgmt = {0}; // tqRead +int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* offset); int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** pHeadWithCkSum); // tqExec diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e21b0fe9e8..c4dd335aa4 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -307,7 +307,22 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); - if (fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (!pHandle->fetchMeta && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (tqScanLog(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { + ASSERT(0); + code = -1; + goto OVER; + } + if (dataRsp.blockNum == 0) { + // add to async task + } + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + goto OVER; + } + + if (pHandle->fetchMeta && fetchOffsetNew.type == TMQ_OFFSET__LOG) { int64_t fetchVer = fetchOffsetNew.version + 1; SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 16822e6003..a6e76010e6 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -46,7 +46,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); if (metaGetTableEntryByUid(&mr, uid) < 0) { @@ -59,6 +59,46 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, i return 0; } +int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal* pOffset) { + qTaskInfo_t task = pExec->execCol.task[0]; + + if (qStreamPrepareScan1(task, pOffset) < 0) { + ASSERT(0); + } + + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(0); + } + + if (pDataBlock != NULL) { + tqAddBlockDataToRsp(pDataBlock, pRsp); + if (pRsp->withTbName) { + int64_t uid = pExec->pExecReader[0]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp); + } + pRsp->blockNum++; + continue; + } + + void* meta = qStreamExtractMetaMsg(task); + if (meta != NULL) { + // tq add meta to rsp + } + + if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { + ASSERT(0); + } + ASSERT(pRsp->rspOffset.type != 0); + + break; + } + + return 0; +} + int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset, int32_t workerId) { ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); qTaskInfo_t task = pExec->execCol.task[workerId]; @@ -67,7 +107,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S /*ASSERT(0);*/ /*}*/ - if (qStreamPrepareScan(task, offset.uid, offset.ts) < 0) { + if (qStreamPrepareTsdbScan(task, offset.uid, offset.ts) < 0) { ASSERT(0); } @@ -93,7 +133,7 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, S if (qGetStreamScanStatus(task, &uid, &ts) < 0) { ASSERT(0); } - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); #endif } pRsp->blockNum++; @@ -129,7 +169,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); } pRsp->blockNum++; } @@ -146,7 +186,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; @@ -164,7 +204,7 @@ int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataR tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; - tqAddTbNameToRsp(pTq, uid, pRsp, workerId); + tqAddTbNameToRsp(pTq, uid, pRsp); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 53ef17a6ba..504b040ed9 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -15,11 +15,6 @@ #include "tq.h" -int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOffsetVal offset) { - /*if ()*/ - return 0; -} - int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHead** ppCkHead) { int32_t code = 0; taosThreadMutexLock(&pHandle->pWalReader->mutex); @@ -84,8 +79,10 @@ STqReader* tqOpenReader(SVnode* pVnode) { return NULL; } - // TODO open - /*pReader->pWalReader = walOpenReader(pVnode->pWal, NULL);*/ + pReader->pWalReader = walOpenReader(pVnode->pWal, NULL); + if (pReader->pWalReader == NULL) { + return NULL; + } pReader->pVnodeMeta = pVnode->pMeta; pReader->pMsg = NULL; @@ -106,12 +103,19 @@ void tqCloseReader(STqReader* pReader) { taosMemoryFree(pReader); } +int32_t tqSeekVer(STqReader* pReader, int64_t ver) { + // + return walReadSeekVer(pReader->pWalReader, ver); +} + int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { bool fromProcessedMsg = pReader->pMsg != NULL; while (1) { if (!fromProcessedMsg) { if (walNextValidMsg(pReader->pWalReader) < 0) { + ret->offset.type = TMQ_OFFSET__LOG; + ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; return -1; } @@ -130,19 +134,23 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { memset(&ret->data, 0, sizeof(SSDataBlock)); int32_t code = tqRetrieveDataBlock(&ret->data, pReader); if (code != 0 || ret->data.info.rows == 0) { + ASSERT(0); +#if 0 if (fromProcessedMsg) { ret->fetchType = FETCH_TYPE__NONE; return 0; } else { break; } +#endif } - ret->fetchType = FETCH_TYPE__DATA; return 0; } if (fromProcessedMsg) { + ret->offset.type = TMQ_OFFSET__LOG; + ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; return 0; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 85ddfc4b2b..ef41bd457a 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -40,8 +40,8 @@ extern "C" { #include "tpagedbuf.h" #include "tstreamUpdate.h" -#include "vnode.h" #include "executorInt.h" +#include "vnode.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -51,13 +51,12 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) -#define START_TS_COLUMN_INDEX 0 -#define END_TS_COLUMN_INDEX 1 -#define UID_COLUMN_INDEX 2 -#define GROUPID_COLUMN_INDEX UID_COLUMN_INDEX +#define START_TS_COLUMN_INDEX 0 +#define END_TS_COLUMN_INDEX 1 +#define UID_COLUMN_INDEX 2 +#define GROUPID_COLUMN_INDEX UID_COLUMN_INDEX #define DELETE_GROUPID_COLUMN_INDEX 2 - enum { // when this task starts to execute, this status will set TASK_NOT_COMPLETED = 0x1u, @@ -81,8 +80,8 @@ typedef struct SResultInfo { // TODO refactor } SResultInfo; typedef struct STableQueryInfo { - TSKEY lastKey; // last check ts, todo remove it later - SResultRowPosition pos; // current active time window + TSKEY lastKey; // last check ts, todo remove it later + SResultRowPosition pos; // current active time window } STableQueryInfo; typedef struct SLimit { @@ -105,7 +104,7 @@ typedef struct STaskCostInfo { uint64_t loadDataTime; SFileBlockLoadRecorder* pRecoder; - uint64_t elapsedTime; + uint64_t elapsedTime; uint64_t firstStageMergeTime; uint64_t winInfoSize; @@ -118,8 +117,8 @@ typedef struct STaskCostInfo { } STaskCostInfo; typedef struct SOperatorCostInfo { - double openCost; - double totalCost; + double openCost; + double totalCost; } SOperatorCostInfo; struct SOperatorInfo; @@ -139,70 +138,81 @@ typedef struct STaskIdInfo { char* str; } STaskIdInfo; +typedef struct { + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + void* metaBlk; // for tmq fetching meta + SSDataBlock* pullOverBlk; // for streaming + SWalFilterCond cond; +} SStreamTaskInfo; + typedef struct SExecTaskInfo { - STaskIdInfo id; - uint32_t status; - STimeWindow window; - STaskCostInfo cost; - int64_t owner; // if it is in execution - int32_t code; + STaskIdInfo id; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + int32_t code; + + SStreamTaskInfo streamInfo; + struct { - char *tablename; - char *dbname; - int32_t tversion; - SSchemaWrapper*sw; + char* tablename; + char* dbname; + int32_t tversion; + SSchemaWrapper* sw; } schemaVer; - STableListInfo tableqinfoList; // this is a table list - const char* sql; // query sql string - jmp_buf env; // jump to this position when error happens. - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + STableListInfo tableqinfoList; // this is a table list + const char* sql; // query sql string + jmp_buf env; // jump to this position when error happens. + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] struct SOperatorInfo* pRoot; } SExecTaskInfo; enum { - OP_NOT_OPENED = 0x0, - OP_OPENED = 0x1, + OP_NOT_OPENED = 0x0, + OP_OPENED = 0x1, OP_RES_TO_RETURN = 0x5, - OP_EXEC_DONE = 0x9, + OP_EXEC_DONE = 0x9, }; typedef struct SOperatorFpSet { - __optr_open_fn_t _openFn; // DO NOT invoke this function directly - __optr_fn_t getNextFn; - __optr_fn_t getStreamResFn; // execute the aggregate in the stream model, todo remove it - __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP - __optr_close_fn_t closeFn; - __optr_encode_fn_t encodeResultRow; - __optr_decode_fn_t decodeResultRow; - __optr_explain_fn_t getExplainFn; + __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_fn_t getNextFn; + __optr_fn_t getStreamResFn; // execute the aggregate in the stream model, todo remove it + __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP + __optr_close_fn_t closeFn; + __optr_encode_fn_t encodeResultRow; + __optr_decode_fn_t decodeResultRow; + __optr_explain_fn_t getExplainFn; } SOperatorFpSet; typedef struct SExprSupp { SExprInfo* pExprInfo; - int32_t numOfExprs; // the number of scalar expression in group operator + int32_t numOfExprs; // the number of scalar expression in group operator SqlFunctionCtx* pCtx; int32_t* rowEntryInfoOffset; // offset value for each row result cell info } SExprSupp; typedef struct SOperatorInfo { - uint8_t operatorType; - bool blocking; // block operator or not - uint8_t status; // denote if current operator is completed - char* name; // name, for debug purpose - void* info; // extension attribution - SExprSupp exprSupp; - SExecTaskInfo* pTaskInfo; - SOperatorCostInfo cost; - SResultInfo resultInfo; - struct SOperatorInfo** pDownstream; // downstram pointer list - int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - SOperatorFpSet fpSet; + uint8_t operatorType; + bool blocking; // block operator or not + uint8_t status; // denote if current operator is completed + char* name; // name, for debug purpose + void* info; // extension attribution + SExprSupp exprSupp; + SExecTaskInfo* pTaskInfo; + SOperatorCostInfo cost; + SResultInfo resultInfo; + struct SOperatorInfo** pDownstream; // downstram pointer list + int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + SOperatorFpSet fpSet; } SOperatorInfo; typedef enum { EX_SOURCE_DATA_NOT_READY = 0x1, - EX_SOURCE_DATA_READY = 0x2, + EX_SOURCE_DATA_READY = 0x2, EX_SOURCE_DATA_EXHAUSTED = 0x3, } EX_SOURCE_STATUS; @@ -210,12 +220,12 @@ typedef enum { #define COL_MATCH_FROM_SLOT_ID 0x2 typedef struct SSourceDataInfo { - int32_t index; - SRetrieveTableRsp* pRsp; - uint64_t totalRows; - int32_t code; - EX_SOURCE_STATUS status; - const char* taskId; + int32_t index; + SRetrieveTableRsp* pRsp; + uint64_t totalRows; + int32_t code; + EX_SOURCE_STATUS status; + const char* taskId; } SSourceDataInfo; typedef struct SLoadRemoteDataInfo { @@ -237,22 +247,22 @@ typedef struct SExchangeInfo { } SExchangeInfo; typedef struct SColMatchInfo { - int32_t srcSlotId; // source slot id + int32_t srcSlotId; // source slot id int32_t colId; int32_t targetSlotId; - bool output; // todo remove this? + bool output; // todo remove this? bool reserved; - int32_t matchType; // determinate the source according to col id or slot id + int32_t matchType; // determinate the source according to col id or slot id } SColMatchInfo; typedef struct SScanInfo { - int32_t numOfAsc; - int32_t numOfDesc; + int32_t numOfAsc; + int32_t numOfDesc; } SScanInfo; typedef struct SSampleExecInfo { - double sampleRatio; // data block sample ratio, 1 by default - uint32_t seed; // random seed value + double sampleRatio; // data block sample ratio, 1 by default + uint32_t seed; // random seed value } SSampleExecInfo; enum { @@ -261,36 +271,38 @@ enum { }; typedef struct STableScanInfo { - STsdbReader* dataReader; - SReadHandle readHandle; + STsdbReader* dataReader; + SReadHandle readHandle; SFileBlockLoadRecorder readRecorder; - SScanInfo scanInfo; - int32_t scanTimes; - SNode* pFilterNode; // filter info, which is push down by optimizer - SqlFunctionCtx* pCtx; // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup - int32_t* rowEntryInfoOffset; // todo: remove this by using SExprSup - SExprInfo* pExpr;// todo: remove this by using SExprSup + SScanInfo scanInfo; + int32_t scanTimes; + SNode* pFilterNode; // filter info, which is push down by optimizer + SqlFunctionCtx* + pCtx; // which belongs to the direct upstream operator operator query context,todo: remove this by using SExprSup + int32_t* rowEntryInfoOffset; // todo: remove this by using SExprSup + SExprInfo* pExpr; // todo: remove this by using SExprSup - SSDataBlock* pResBlock; - SArray* pColMatchInfo; - SExprSupp pseudoSup; + SSDataBlock* pResBlock; + SArray* pColMatchInfo; + SExprSupp pseudoSup; SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time + // window to check if current data block needs to be loaded. - SSampleExecInfo sample; // sample execution info + SSampleExecInfo sample; // sample execution info int32_t curTWinIdx; - int32_t currentGroupId; - int32_t currentTable; - uint64_t queryId; // todo remove it - uint64_t taskId; // todo remove it + int32_t currentGroupId; + int32_t currentTable; + uint64_t queryId; // todo remove it + uint64_t taskId; // todo remove it struct { uint64_t uid; - int64_t ts; + int64_t ts; } lastStatus; int8_t scanMode; @@ -298,37 +310,37 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo *pCols; - SSDataBlock *pRes; - SArray *pColMatchInfo; - int32_t curPos; - SReadHandle readHandle; - STableListInfo *pTableList; + SColumnInfo* pCols; + SSDataBlock* pRes; + SArray* pColMatchInfo; + int32_t curPos; + SReadHandle readHandle; + STableListInfo* pTableList; } STagScanInfo; typedef struct SLastrowScanInfo { - SSDataBlock *pRes; - SArray *pTableList; - SReadHandle readHandle; - void *pLastrowReader; - SArray *pColMatchInfo; - int32_t *pSlotIds; + SSDataBlock* pRes; + SArray* pTableList; + SReadHandle readHandle; + void* pLastrowReader; + SArray* pColMatchInfo; + int32_t* pSlotIds; } SLastrowScanInfo; typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, - STREAM_SCAN_FROM_DATAREADER, // todo(liuyao) delete it + STREAM_SCAN_FROM_DATAREADER, // todo(liuyao) delete it STREAM_SCAN_FROM_DATAREADER_RETRIEVE, STREAM_SCAN_FROM_DATAREADER_RANGE, } EStreamScanMode; typedef struct SCatchSupporter { - SHashObj* pWindowHashTable; // quick locate the window object for each window - SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file - int32_t keySize; - int64_t* pKeyBuf; + SHashObj* pWindowHashTable; // quick locate the window object for each window + SDiskbasedBuf* pDataBuf; // buffer based on blocked-wised disk file + int32_t keySize; + int64_t* pKeyBuf; } SCatchSupporter; typedef struct SStreamAggSupporter { @@ -336,56 +348,56 @@ typedef struct SStreamAggSupporter { SArray* pCurWins; int32_t valueSize; int32_t keySize; - char* pKeyBuf; // window key buffer - SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + char* pKeyBuf; // window key buffer + SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row SArray* pScanWindow; } SStreamAggSupporter; typedef struct SessionWindowSupporter { SStreamAggSupporter* pStreamAggSup; - int64_t gap; - uint8_t parentType; + int64_t gap; + uint8_t parentType; } SessionWindowSupporter; typedef struct SStreamScanInfo { - uint64_t tableUid; // queried super table uid - SExprInfo* pPseudoExpr; - int32_t numOfPseudoExpr; - int32_t primaryTsIndex; // primary time stamp slot id - SReadHandle readHandle; - SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. - SArray* pColMatchInfo; // - SNode* pCondition; + uint64_t tableUid; // queried super table uid + SExprInfo* pPseudoExpr; + int32_t numOfPseudoExpr; + int32_t primaryTsIndex; // primary time stamp slot id + SReadHandle readHandle; + SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. + SArray* pColMatchInfo; // + SNode* pCondition; - SArray* pBlockLists; // multiple SSDatablock. - SSDataBlock* pRes; // result SSDataBlock - SSDataBlock* pUpdateRes; // update SSDataBlock - int32_t updateResIndex; - int32_t blockType; // current block type - int32_t validBlockIndex; // Is current data has returned? - uint64_t numOfExec; // execution times - STqReader* tqReader; + SArray* pBlockLists; // multiple SSDatablock. + SSDataBlock* pRes; // result SSDataBlock + SSDataBlock* pUpdateRes; // update SSDataBlock + int32_t updateResIndex; + int32_t blockType; // current block type + int32_t validBlockIndex; // Is current data has returned? + uint64_t numOfExec; // execution times + STqReader* tqReader; - int32_t tsArrayIndex; - SArray* tsArray; - uint64_t groupId; - SUpdateInfo* pUpdateInfo; + int32_t tsArrayIndex; + SArray* tsArray; + uint64_t groupId; + SUpdateInfo* pUpdateInfo; - EStreamScanMode scanMode; - SOperatorInfo* pStreamScanOp; - SOperatorInfo* pTableScanOp; - SArray* childIds; + EStreamScanMode scanMode; + SOperatorInfo* pStreamScanOp; + SOperatorInfo* pTableScanOp; + SArray* childIds; SessionWindowSupporter sessionSup; - bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. - int32_t scanWinIndex; // for state operator - int32_t pullDataResIndex; - SSDataBlock* pPullDataRes; // pull data SSDataBlock - SSDataBlock* pDeleteDataRes; // delete data SSDataBlock - int32_t deleteDataIndex; + bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock + SSDataBlock* pDeleteDataRes; // delete data SSDataBlock + int32_t deleteDataIndex; // status for tmq - //SSchemaWrapper schema; + // SSchemaWrapper schema; STqOffset offset; } SStreamScanInfo; @@ -412,27 +424,27 @@ typedef struct SBlockDistInfo { SSDataBlock* pResBlock; void* pHandle; SReadHandle readHandle; - uint64_t uid; // table uid + uint64_t uid; // table uid } SBlockDistInfo; // todo remove this typedef struct SOptrBasicInfo { - SResultRowInfo resultRowInfo; - SSDataBlock* pRes; + SResultRowInfo resultRowInfo; + SSDataBlock* pRes; } SOptrBasicInfo; typedef struct SAggSupporter { SHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; typedef struct STimeWindowSupp { - int8_t calTrigger; - int64_t waterMark; - TSKEY maxTs; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. + int8_t calTrigger; + int64_t waterMark; + TSKEY maxTs; + SColumnInfoData timeWindowData; // query time window info for scalar function execution. } STimeWindowAggSupp; typedef struct SIntervalAggOperatorInfo { @@ -451,78 +463,78 @@ typedef struct SIntervalAggOperatorInfo { EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] STimeWindowAggSupp twAggSup; bool invertible; - SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. + SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. bool ignoreExpiredData; SArray* pRecycledPages; - SArray* pDelWins; // SWinRes + SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; - SNode *pCondition; + SNode* pCondition; } SIntervalAggOperatorInfo; typedef struct SStreamFinalIntervalOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - int32_t order; // current SSDataBlock scan order + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + int32_t order; // current SSDataBlock scan order STimeWindowAggSupp twAggSup; SArray* pChildren; SSDataBlock* pUpdateRes; bool returnUpdate; - SPhysiNode* pPhyNode; // create new child + SPhysiNode* pPhyNode; // create new child bool isFinal; SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo + SArray* pPullWins; // SPullWindowInfo int32_t pullIndex; SSDataBlock* pPullDataRes; bool ignoreExpiredData; SArray* pRecycledPages; - SArray* pDelWins; // SWinRes + SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; } SStreamFinalIntervalOperatorInfo; typedef struct SAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; - STableQueryInfo *current; - uint64_t groupId; - SGroupResInfo groupResInfo; - SExprSupp scalarExprSup; + STableQueryInfo* current; + uint64_t groupId; + SGroupResInfo groupResInfo; + SExprSupp scalarExprSup; - SNode *pCondition; + SNode* pCondition; } SAggOperatorInfo; typedef struct SProjectOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SNode* pFilterNode; // filter info, which is push down by optimizer - SSDataBlock* existDataBlock; - SArray* pPseudoColInfo; - SLimit limit; - SLimit slimit; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SNode* pFilterNode; // filter info, which is push down by optimizer + SSDataBlock* existDataBlock; + SArray* pPseudoColInfo; + SLimit limit; + SLimit slimit; - uint64_t groupId; - int64_t curSOffset; - int64_t curGroupOutput; + uint64_t groupId; + int64_t curSOffset; + int64_t curGroupOutput; - int64_t curOffset; - int64_t curOutput; + int64_t curOffset; + int64_t curOutput; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SExprSupp scalarSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SExprSupp scalarSup; } SIndefOperatorInfo; typedef struct SFillOperatorInfo { @@ -536,23 +548,23 @@ typedef struct SFillOperatorInfo { typedef struct SGroupbyOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - SNode* pCondition; - bool isInit; // denote if current val is initialized or not - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SGroupResInfo groupResInfo; - SExprSupp scalarSup; + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + SNode* pCondition; + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SGroupResInfo groupResInfo; + SExprSupp scalarSup; } SGroupbyOperatorInfo; typedef struct SDataGroupInfo { - uint64_t groupId; - int64_t numOfRows; - SArray* pPageList; + uint64_t groupId; + int64_t numOfRows; + SArray* pPageList; } SDataGroupInfo; // The sort in partition may be needed later. @@ -564,12 +576,12 @@ typedef struct SPartitionOperatorInfo { int32_t groupKeyLen; // total group by column width SHashObj* pGroupSet; // quick locate the window object for each result - SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file - int32_t rowCapacity; // maximum number of rows for each buffer page - int32_t* columnOffset; // start position for each column data - SArray* sortedGroupArray; // SDataGroupInfo sorted by group id - int32_t groupIndex; // group index - int32_t pageIndex; // page index of current group + SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file + int32_t rowCapacity; // maximum number of rows for each buffer page + int32_t* columnOffset; // start position for each column data + SArray* sortedGroupArray; // SDataGroupInfo sorted by group id + int32_t groupIndex; // group index + int32_t pageIndex; // page index of current group SSDataBlock* pUpdateRes; SExprSupp scalarSup; } SPartitionOperatorInfo; @@ -583,67 +595,67 @@ typedef struct SWindowRowsSup { typedef struct SSessionAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; SGroupResInfo groupResInfo; SWindowRowsSup winSup; - bool reptScan; // next round scan - int64_t gap; // session window gap - int32_t tsSlotId; // primary timestamp slot id + bool reptScan; // next round scan + int64_t gap; // session window gap + int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; } SSessionAggOperatorInfo; typedef struct SResultWindowInfo { SResultRowPosition pos; - STimeWindow win; - bool isOutput; - bool isClosed; + STimeWindow win; + bool isOutput; + bool isClosed; } SResultWindowInfo; typedef struct SStateWindowInfo { SResultWindowInfo winInfo; - SStateKeys stateKey; + SStateKeys stateKey; } SStateWindowInfo; typedef struct SStreamSessionAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SGroupResInfo groupResInfo; - int64_t gap; // session window gap - int32_t primaryTsIndex; // primary timestamp slot id - int32_t endTsIndex; // window end timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SSDataBlock* pWinBlock; // window result - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; // delete result - bool returnDelete; - SSDataBlock* pUpdateRes; // update window - SHashObj* pStDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; final stream operator - SPhysiNode* pPhyNode; // create new child - bool isFinal; - bool ignoreExpiredData; + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SGroupResInfo groupResInfo; + int64_t gap; // session window gap + int32_t primaryTsIndex; // primary timestamp slot id + int32_t endTsIndex; // window end timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SSDataBlock* pWinBlock; // window result + SqlFunctionCtx* pDummyCtx; // for combine + SSDataBlock* pDelRes; // delete result + bool returnDelete; + SSDataBlock* pUpdateRes; // update window + SHashObj* pStDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; final stream operator + SPhysiNode* pPhyNode; // create new child + bool isFinal; + bool ignoreExpiredData; } SStreamSessionAggOperatorInfo; typedef struct STimeSliceOperatorInfo { - SSDataBlock* pRes; - STimeWindow win; - SInterval interval; - int64_t current; - SArray* pPrevRow; // SArray - int32_t fillType; // fill type - SColumn tsCol; // primary timestamp column - SExprSupp scalarSup; // scalar calculation - struct SFillColInfo* pFillColInfo; // fill column info + SSDataBlock* pRes; + STimeWindow win; + SInterval interval; + int64_t current; + SArray* pPrevRow; // SArray + int32_t fillType; // fill type + SColumn tsCol; // primary timestamp column + SExprSupp scalarSup; // scalar calculation + struct SFillColInfo* pFillColInfo; // fill column info } STimeSliceOperatorInfo; typedef struct SStateWindowOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; SGroupResInfo groupResInfo; SWindowRowsSup winSup; @@ -656,52 +668,52 @@ typedef struct SStateWindowOperatorInfo { } SStateWindowOperatorInfo; typedef struct SStreamStateAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SGroupResInfo groupResInfo; - int32_t primaryTsIndex; // primary timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SColumn stateCol; // start row index - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; - SHashObj* pSeDeleted; - void* pDelIterator; - SArray* pScanWindow; - SArray* pChildren; // cache for children's result; - bool ignoreExpiredData; + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SColumn stateCol; // start row index + SqlFunctionCtx* pDummyCtx; // for combine + SSDataBlock* pDelRes; + SHashObj* pSeDeleted; + void* pDelIterator; + SArray* pScanWindow; + SArray* pChildren; // cache for children's result; + bool ignoreExpiredData; } SStreamStateAggOperatorInfo; typedef struct SSortedMergeOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; - SArray* pSortInfo; - int32_t numOfSources; - SSortHandle *pSortHandle; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - int32_t resultRowFactor; - bool hasGroupVal; - SDiskbasedBuf *pTupleStore; // keep the final results - int32_t numOfResPerPage; - char** groupVal; - SArray *groupInfo; + SArray* pSortInfo; + int32_t numOfSources; + SSortHandle* pSortHandle; + int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + int32_t resultRowFactor; + bool hasGroupVal; + SDiskbasedBuf* pTupleStore; // keep the final results + int32_t numOfResPerPage; + char** groupVal; + SArray* groupInfo; } SSortedMergeOperatorInfo; typedef struct SSortOperatorInfo { SOptrBasicInfo binfo; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SArray* pColMatchInfo; // for index map from table scan output - int32_t bufPageSize; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SArray* pColMatchInfo; // for index map from table scan output + int32_t bufPageSize; - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - SNode* pCondition; + SNode* pCondition; } SSortOperatorInfo; typedef struct STagFilterOperatorInfo { @@ -709,17 +721,17 @@ typedef struct STagFilterOperatorInfo { } STagFilterOperatorInfo; typedef struct SJoinOperatorInfo { - SSDataBlock *pRes; - int32_t joinType; + SSDataBlock* pRes; + int32_t joinType; - SSDataBlock *pLeft; - int32_t leftPos; - SColumnInfo leftCol; + SSDataBlock* pLeft; + int32_t leftPos; + SColumnInfo leftCol; - SSDataBlock *pRight; - int32_t rightPos; - SColumnInfo rightCol; - SNode *pCondAfterMerge; + SSDataBlock* pRight; + int32_t rightPos; + SColumnInfo rightCol; + SNode* pCondAfterMerge; } SJoinOperatorInfo; #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) @@ -728,8 +740,8 @@ typedef struct SJoinOperatorInfo { void doDestroyExchangeOperatorInfo(void* param); SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, - __optr_decode_fn_t decode, __optr_explain_fn_t explain); + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_explain_fn_t explain); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); void operatorDummyCloseFn(void* param, int32_t numOfCols); @@ -739,28 +751,30 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); void cleanupExprSupp(SExprSupp* pSup); -int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, +int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey); void initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows); -void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf); -void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, - int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order); +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, + SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, + int32_t numOfTotal, int32_t numOfOutput, int32_t order); int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadInfo, int32_t numOfRows, char* pData, - int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, - SArray* pColList); + int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, + SArray* pColList); void getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key, STimeWindow* win); -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); -void doSetOperatorCompleted(SOperatorInfo* pOperator); -void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); +void doSetOperatorCompleted(SOperatorInfo* pOperator); +void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock); -void cleanupAggSup(SAggSupporter* pAggSup); -void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); -void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); -void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); +void cleanupAggSup(SAggSupporter* pAggSup); +void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); +void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); +void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts); int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts); @@ -769,92 +783,108 @@ SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); -SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, - char* pData, int16_t bytes, bool masterscan, uint64_t groupId, - SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, + int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, + bool isIntervalQuery, SAggSupporter* pSup); SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, + const char* pUser, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t num, SArray* pSortInfo, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, + SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SExprInfo* pExprInfo, + int32_t num, SArray* pSortInfo, SArray* pGroupInfo, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createLastrowScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SArray* pTableList, SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, bool isStream); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + SExecTaskInfo* pTaskInfo); -SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, + int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, + int32_t primaryTsSlotId, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); + STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, STimeWindowAggSupp* pTwAggSupp, - SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, int64_t gap, int32_t tsSlotId, + STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SArray* pGroupColList, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, SBlockDistScanPhysiNode* pBlockScanNode, - SExecTaskInfo* pTaskInfo); +SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* readHandle, uint64_t uid, + SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId); +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, + uint64_t taskId); SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult, SExecTaskInfo* pTaskInfo); SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfCols, - SSDataBlock* pResBlock, STimeWindowAggSupp *pTwAggSupp, int32_t tsSlotId, SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo); + SSDataBlock* pResBlock, STimeWindowAggSupp* pTwAggSupp, int32_t tsSlotId, + SColumn* pStateKeyCol, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode, +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t + numOfCols, SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */ + SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, + SJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild); + +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); #endif int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput, SArray* pPseudoList); + int32_t numOfOutput, SArray* pPseudoList); -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + int32_t scanFlag, bool createDummyCol); bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -void setTaskKilled(SExecTaskInfo* pTaskInfo); -void queryCostStatis(SExecTaskInfo* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo); +void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); int32_t getMaximumIdleDurationSec(); @@ -866,7 +896,7 @@ int32_t getMaximumIdleDurationSec(); * nOptrWithVal: *nOptrWithVal save the number of optr with value * return: result code, 0 means success */ -int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal); +int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t* length, int32_t* nOptrWithVal); /* * ops: root operator, created by caller @@ -879,7 +909,7 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, const char* sql, EOPTR_EXEC_MODEL model); -int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo); +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo** pRes, int32_t* capacity, int32_t* resNum); @@ -889,35 +919,37 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t precision, STimeWindow* win); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, - __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t size); -SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, - TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); -SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, - TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); -bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); -int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, - TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted); -bool functionNeedToExecute(SqlFunctionCtx* pCtx); + int32_t size); +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); +SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex); +SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex); +bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); +int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, + int32_t start, int64_t gap, SHashObj* pStDeleted); +bool functionNeedToExecute(SqlFunctionCtx* pCtx); int32_t compareTimeWindow(const void* p1, const void* p2, const void* param); int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, - SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, - SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); + SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, + const int32_t* rowCellOffset, SSDataBlock* pBlock, + SExecTaskInfo* pTaskInfo); -int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); +int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, + uint64_t taskId); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); -int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); SSDataBlock* createPullDataBlock(); #ifdef __cplusplus diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index a83565cbe0..fc35fba935 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -60,9 +60,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } - } else if (type == STREAM_INPUT__DATA_SCAN) { + } else if (type == STREAM_INPUT__TABLE_SCAN) { // do nothing - ASSERT(pInfo->blockType == STREAM_INPUT__DATA_SCAN); + ASSERT(pInfo->blockType == STREAM_INPUT__TABLE_SCAN); } else { ASSERT(0); } @@ -76,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { return TSDB_CODE_QRY_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__DATA_SCAN, 0, NULL); + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__TABLE_SCAN, 0, NULL); } int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index d5e546f466..d96781d2c2 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -267,7 +267,44 @@ const STqOffset* qExtractStatusFromStreamScanner(void* scanner) { return &pInfo->offset; } -int32_t qStreamPrepareScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { +void* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + return pTaskInfo->streamInfo.metaBlk; +} + +int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); + return 0; +} + +int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + ASSERT(pTaskInfo->execModel == OPTR_EXEC_MODEL_STREAM); + pTaskInfo->streamInfo.prepareStatus = *pOffset; + // TODO: optimize + /*if (pTaskInfo->streamInfo.lastStatus.type != pOffset->type ||*/ + /*pTaskInfo->streamInfo.prepareStatus.version != pTaskInfo->streamInfo.lastStatus.version) {*/ + while (1) { + uint8_t type = pOperator->operatorType; + pOperator->status = OP_OPENED; + if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pInfo = pOperator->info; + tqSeekVer(pInfo->tqReader, pOffset->version); + return 0; + } else { + ASSERT(pOperator->numOfDownstream == 1); + pOperator = pOperator->pDownstream[0]; + } + } + /*}*/ + return 0; +} + +int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if (uid == 0) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9865fbc5cd..a9ecada6e5 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2848,7 +2848,7 @@ int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts) { if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pScanInfo = pOperator->info; - pScanInfo->blockType = STREAM_INPUT__DATA_SCAN; + pScanInfo->blockType = STREAM_INPUT__TABLE_SCAN; pScanInfo->pTableScanOp->status = OP_OPENED; @@ -3283,7 +3283,10 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { + // TODO optimize + /*if (pTaskInfo->execModel != OPTR_EXEC_MODEL_STREAM) {*/ doSetOperatorCompleted(pOperator); + /*}*/ break; } if (pBlock->info.type == STREAM_RETRIEVE) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 68cda52b10..95d7948989 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1123,15 +1123,116 @@ static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32 uidCol[i] = getGroupId(pOperator, uidCol[i]); } } +static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock) { + SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; + SOperatorInfo* pOperator = pInfo->pStreamScanOp; + SExecTaskInfo* pTaskInfo = pInfo->pStreamScanOp->pTaskInfo; + + pInfo->pRes->info.rows = pBlock->info.rows; + pInfo->pRes->info.uid = pBlock->info.uid; + pInfo->pRes->info.type = STREAM_NORMAL; + pInfo->pRes->info.capacity = pBlock->info.rows; + + // for generating rollup SMA result, each time is an independent time serie. + // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this + if (pInfo->assignBlockUid) { + pInfo->pRes->info.groupId = pBlock->info.uid; + } + + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &pBlock->info.uid, sizeof(int64_t)); + if (groupIdPre) { + pInfo->pRes->info.groupId = *groupIdPre; + } else { + pInfo->pRes->info.groupId = 0; + } + + // todo extract method + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { + SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + + bool colExists = false; + for (int32_t j = 0; j < blockDataGetNumOfCols(pBlock); ++j) { + SColumnInfoData* pResCol = bdGetColumnInfoData(pBlock, j); + if (pResCol->info.colId == pColMatchInfo->colId) { + taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); + colExists = true; + break; + } + } + + // the required column does not exists in submit block, let's set it to be all null value + if (!colExists) { + SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); + colDataAppendNNULL(pDst, 0, pBlockInfo->rows); + } + } + + taosArrayDestroy(pBlock->pDataBlock); + + ASSERT(pInfo->pRes->pDataBlock != NULL); +#if 0 + if (pInfo->pRes->pDataBlock == NULL) { + // TODO add log + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); + pOperator->status = OP_EXEC_DONE; + pTaskInfo->code = terrno; + return -1; + } +#endif + + // currently only the tbname pseudo column + if (pInfo->numOfPseudoExpr > 0) { + addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); + } + + doFilter(pInfo->pCondition, pInfo->pRes); + blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); + if (pBlockInfo->rows > 0) { + return 0; + } + return 0; +} static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamScanInfo* pInfo = pOperator->info; - pTaskInfo->code = pOperator->fpSet._openFn(pOperator); - if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { - return NULL; + /*pTaskInfo->code = pOperator->fpSet._openFn(pOperator);*/ + /*if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) {*/ + /*return NULL;*/ + /*}*/ + + if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { + while (1) { + SFetchRet ret = {0}; + tqNextBlock(pInfo->tqReader, &ret); + if (ret.fetchType == FETCH_TYPE__DATA) { + blockDataCleanup(pInfo->pRes); + if (setBlockIntoRes(pInfo, &ret.data) < 0) { + ASSERT(0); + } + pTaskInfo->streamInfo.lastStatus = ret.offset; + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } else { + tDeleteSSDataBlock(&ret.data); + } + } else if (ret.fetchType == FETCH_TYPE__META) { + ASSERT(0); + pTaskInfo->streamInfo.lastStatus = ret.offset; + pTaskInfo->streamInfo.metaBlk = ret.meta; + return NULL; + } else if (ret.fetchType == FETCH_TYPE__NONE) { + pTaskInfo->streamInfo.lastStatus = ret.offset; + return NULL; + } else { + ASSERT(0); + } + } } size_t total = taosArrayGetSize(pInfo->pBlockLists); @@ -1139,7 +1240,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { if (pInfo->validBlockIndex >= total) { /*doClearBufferedBlocks(pInfo);*/ - pOperator->status = OP_EXEC_DONE; + /*pOperator->status = OP_EXEC_DONE;*/ return NULL; } @@ -1286,6 +1387,9 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { } taosArrayDestroy(block.pDataBlock); + + ASSERT(pInfo->pRes->pDataBlock != NULL); +#if 0 if (pInfo->pRes->pDataBlock == NULL) { // TODO add log updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); @@ -1293,6 +1397,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->code = terrno; return NULL; } +#endif // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { @@ -1312,7 +1417,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (pBlockInfo->rows == 0) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); - pOperator->status = OP_EXEC_DONE; + /*pOperator->status = OP_EXEC_DONE;*/ } else if (pInfo->pUpdateInfo) { pInfo->tsArrayIndex = 0; checkUpdateData(pInfo, true, pInfo->pRes, true); @@ -1330,7 +1435,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; - } else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) { + } else if (pInfo->blockType == STREAM_INPUT__TABLE_SCAN) { // check reader last status // if not match, reset status SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index b5c75ce3c4..1ccb9cf152 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -147,7 +147,7 @@ static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) { return 0; } -static int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { +int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { SWal *pWal = pRead->pWal; if (ver == pRead->curVersion) { return 0; @@ -355,22 +355,6 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { return 0; } -int32_t walReadWithHandle_s(SWalReader *pRead, int64_t ver, SWalCont **ppHead) { - taosThreadMutexLock(&pRead->mutex); - if (walReadVer(pRead, ver) < 0) { - taosThreadMutexUnlock(&pRead->mutex); - return -1; - } - *ppHead = taosMemoryMalloc(sizeof(SWalCont) + pRead->pHead->head.bodyLen); - if (*ppHead == NULL) { - taosThreadMutexUnlock(&pRead->mutex); - return -1; - } - memcpy(*ppHead, &pRead->pHead->head, sizeof(SWalCont) + pRead->pHead->head.bodyLen); - taosThreadMutexUnlock(&pRead->mutex); - return 0; -} - int32_t walReadVer(SWalReader *pRead, int64_t ver) { int64_t code; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 4e009e702d..79f0034052 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -80,7 +80,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -./test.sh -f tsim/mnode/basic4.sim +#./test.sh -f tsim/mnode/basic4.sim ./test.sh -f tsim/mnode/basic5.sim # ---- show diff --git a/tests/system-test/failed.txt b/tests/system-test/failed.txt index d0b66b1769..59c4d625d9 100644 --- a/tests/system-test/failed.txt +++ b/tests/system-test/failed.txt @@ -1 +1,2 @@ #python3 ./test.py -f 2-query/last.py -Q 3 +#./test.sh -f tsim/mnode/basic4.sim From e4c45e507f23d65a342794ff589b7ce0722a1d10 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 8 Jul 2022 18:51:44 +0800 Subject: [PATCH 03/13] fix: release handle to keep refs correct --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 946984d9c2..0c5f851d97 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -1256,6 +1256,8 @@ int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) { if (invalidate) { taosLRUCacheRelease(pCache, h, true); + } else { + taosLRUCacheRelease(pCache, h, false); } // void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen); } From a9e75c87aa4546a4ecc9540d97f6c56758f9dc19 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 8 Jul 2022 18:37:16 +0800 Subject: [PATCH 04/13] refactor(stream): internal refactor --- source/dnode/vnode/src/tq/tq.c | 30 +++++++++++++++++------------- source/dnode/vnode/src/tq/tqRead.c | 1 + source/libs/wal/src/walRead.c | 5 +++++ 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index c4dd335aa4..a2f84c6ba3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -244,11 +244,6 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { STqOffsetVal fetchOffsetNew; // 1.find handle - char buf[80]; - tFormatOffset(buf, 80, &reqOffset); - tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch, - TD_VID(pTq->pVnode), buf); - STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); /*ASSERT(pHandle);*/ if (pHandle == NULL) { @@ -270,6 +265,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch); } + char buf[80]; + tFormatOffset(buf, 80, &reqOffset); + tqDebug("tmq poll: consumer %ld (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId, + pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf); + // 2.reset offset if needed if (reqOffset.type > 0) { fetchOffsetNew = reqOffset; @@ -279,7 +279,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { fetchOffsetNew = pOffset->val; char formatBuf[80]; tFormatOffset(formatBuf, 80, &fetchOffsetNew); - tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf); + tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %s", consumerId, pHandle->subKey, formatBuf); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { if (pReq->useSnapshot && pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -295,8 +295,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal)); } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { - tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId, - TD_VID(pTq->pVnode), pReq->subKey); + tqError("tmq poll: subkey %s, no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", + pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; return -1; } @@ -307,14 +307,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); - if (!pHandle->fetchMeta && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + fetchOffsetNew.version++; if (tqScanLog(pTq, &pHandle->execHandle, &dataRsp, &fetchOffsetNew) < 0) { ASSERT(0); code = -1; goto OVER; } if (dataRsp.blockNum == 0) { - // add to async task + // TODO add to async task + /*dataRsp.rspOffset.version--;*/ } if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; @@ -322,7 +324,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { goto OVER; } - if (pHandle->fetchMeta && fetchOffsetNew.type == TMQ_OFFSET__LOG) { + if (pHandle->execHandle.subType != TOPIC_SUB_TYPE__COLUMN && fetchOffsetNew.type == TMQ_OFFSET__LOG) { int64_t fetchVer = fetchOffsetNew.version + 1; SWalCkHead* pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); if (pCkHead == NULL) { @@ -334,8 +336,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { while (1) { consumerEpoch = atomic_load_32(&pHandle->epoch); if (consumerEpoch > reqEpoch) { - tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d", - consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); + tqWarn( + "tmq poll: consumer %ld (epoch %d), subkey %s, vg %d offset %ld, found new consumer epoch %d, discard req " + "epoch %d", + consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); break; } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 504b040ed9..2d51073abf 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -135,6 +135,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { int32_t code = tqRetrieveDataBlock(&ret->data, pReader); if (code != 0 || ret->data.info.rows == 0) { ASSERT(0); + continue; #if 0 if (fromProcessedMsg) { ret->fetchType = FETCH_TYPE__NONE; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 1ccb9cf152..8960703990 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -150,6 +150,7 @@ static int32_t walReadChangeFile(SWalReader *pRead, int64_t fileFirstVer) { int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { SWal *pWal = pRead->pWal; if (ver == pRead->curVersion) { + wDebug("wal version %ld match, no need to reset", ver); return 0; } if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { @@ -177,6 +178,8 @@ int32_t walReadSeekVer(SWalReader *pRead, int64_t ver) { return -1; } + wDebug("wal version reset from %ld to %ld", pRead->curVersion, ver); + pRead->curVersion = ver; return 0; @@ -249,6 +252,7 @@ static int32_t walFetchBodyNew(SWalReader *pRead) { } pRead->curVersion = ver + 1; + wDebug("version advance to %ld, fetch body", pRead->curVersion); return 0; } @@ -265,6 +269,7 @@ static int32_t walSkipFetchBodyNew(SWalReader *pRead) { } pRead->curVersion++; + wDebug("version advance to %ld, skip fetch", pRead->curVersion); return 0; } From 8d01ae00cb9a9f713917ba78c704b776a493d700 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 8 Jul 2022 21:37:15 +0800 Subject: [PATCH 05/13] fix(tmq): offset --- source/dnode/vnode/src/tq/tq.c | 9 +++++++++ source/dnode/vnode/src/tq/tqExec.c | 9 ++++++++- source/dnode/vnode/src/tq/tqRead.c | 1 + source/libs/executor/src/executorMain.c | 4 +++- source/libs/executor/src/scanoperator.c | 21 ++++++++++++--------- source/libs/wal/src/walRead.c | 7 ++++++- 6 files changed, 39 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a2f84c6ba3..d60a1026de 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -294,6 +294,15 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal)); + tqDebug("tmq poll: consumer %ld, subkey %s, offset reset to %ld", consumerId, pHandle->subKey, + fetchOffsetNew.version); + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + dataRsp.rspOffset = fetchOffsetNew; + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + goto OVER; } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey %s, no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index a6e76010e6..d381cfcdc7 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -63,7 +63,9 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff qTaskInfo_t task = pExec->execCol.task[0]; if (qStreamPrepareScan1(task, pOffset) < 0) { - ASSERT(0); + pRsp->rspOffset = *pOffset; + pRsp->rspOffset.version--; + return 0; } while (1) { @@ -91,6 +93,11 @@ int64_t tqScanLog(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, STqOff if (qStreamExtractOffset(task, &pRsp->rspOffset) < 0) { ASSERT(0); } + + if (pRsp->rspOffset.type == TMQ_OFFSET__LOG) { + ASSERT(pRsp->rspOffset.version + 1 >= pRsp->reqOffset.version); + } + ASSERT(pRsp->rspOffset.type != 0); break; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 2d51073abf..4115479e25 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -152,6 +152,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { if (fromProcessedMsg) { ret->offset.type = TMQ_OFFSET__LOG; ret->offset.version = pReader->ver; + ASSERT(pReader->ver != -1); ret->fetchType = FETCH_TYPE__NONE; return 0; } diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index b300d33785..ad33c0ae55 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -293,7 +293,9 @@ int32_t qStreamPrepareScan1(qTaskInfo_t tinfo, const STqOffsetVal* pOffset) { pOperator->status = OP_OPENED; if (type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pInfo = pOperator->info; - tqSeekVer(pInfo->tqReader, pOffset->version); + if (tqSeekVer(pInfo->tqReader, pOffset->version) < 0) { + return -1; + } return 0; } else { ASSERT(pOperator->numOfDownstream == 1); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 501e1979c4..594b0f2b53 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -210,7 +210,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca bool allColumnsHaveAgg = true; SColumnDataAgg** pColAgg = NULL; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } @@ -595,7 +595,7 @@ static void destroyTableScanOperatorInfo(void* param, int32_t numOfOutput) { if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); } - + taosMemoryFreeClear(param); } @@ -745,7 +745,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { static void destroyBlockDistScanOperatorInfo(void* param, int32_t numOfOutput) { SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; blockDataDestroy(pDistInfo->pResBlock); - + taosMemoryFreeClear(param); } @@ -1222,7 +1222,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { if (setBlockIntoRes(pInfo, &ret.data) < 0) { ASSERT(0); } - pTaskInfo->streamInfo.lastStatus = ret.offset; + /*pTaskInfo->streamInfo.lastStatus = ret.offset;*/ if (pInfo->pRes->info.rows > 0) { return pInfo->pRes; } else { @@ -1234,7 +1234,12 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.metaBlk = ret.meta; return NULL; } else if (ret.fetchType == FETCH_TYPE__NONE) { - pTaskInfo->streamInfo.lastStatus = ret.offset; + if (ret.offset.version == -1) { + pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG; + pTaskInfo->streamInfo.lastStatus.version = pTaskInfo->streamInfo.prepareStatus.version - 1; + } else { + pTaskInfo->streamInfo.lastStatus = ret.offset; + } return NULL; } else { ASSERT(0); @@ -1356,8 +1361,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.capacity = block.info.rows; - - uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &block.info.uid, sizeof(int64_t)); if (groupIdPre) { pInfo->pRes->info.groupId = *groupIdPre; @@ -2282,7 +2285,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*)param; pInfo->pRes = blockDataDestroy(pInfo->pRes); - + taosMemoryFreeClear(param); } @@ -2774,7 +2777,7 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) { pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); taosArrayDestroy(pTableScanInfo->pSortInfo); - + taosMemoryFreeClear(param); } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 8960703990..74fa7b0d52 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -190,7 +190,10 @@ void walSetReaderCapacity(SWalReader *pRead, int32_t capacity) { pRead->capacity static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { int64_t contLen; if (pRead->curVersion != fetchVer) { - if (walReadSeekVer(pRead, fetchVer) < 0) return -1; + if (walReadSeekVer(pRead, fetchVer) < 0) { + ASSERT(0); + return -1; + } } contLen = taosReadFile(pRead->pLogFile, pRead->pHead, sizeof(SWalCkHead)); if (contLen != sizeof(SWalCkHead)) { @@ -199,6 +202,7 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } + ASSERT(0); pRead->curVersion = -1; return -1; } @@ -265,6 +269,7 @@ static int32_t walSkipFetchBodyNew(SWalReader *pRead) { if (code < 0) { terrno = TAOS_SYSTEM_ERROR(errno); pRead->curVersion = -1; + ASSERT(0); return -1; } From eb005b9d109552132499c0fbcda369b19aaa8f4a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 8 Jul 2022 21:45:31 +0800 Subject: [PATCH 06/13] fix(tmq): invalid free --- source/libs/executor/src/scanoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 594b0f2b53..65680de347 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1225,8 +1225,8 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { /*pTaskInfo->streamInfo.lastStatus = ret.offset;*/ if (pInfo->pRes->info.rows > 0) { return pInfo->pRes; - } else { - tDeleteSSDataBlock(&ret.data); + /*} else {*/ + /*tDeleteSSDataBlock(&ret.data);*/ } } else if (ret.fetchType == FETCH_TYPE__META) { ASSERT(0); From 0fd4f3b5197df370ac3df41b65f8ec188761672e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 8 Jul 2022 23:05:32 +0800 Subject: [PATCH 07/13] fix(tmq): invalid free --- source/dnode/vnode/src/tq/tq.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index d60a1026de..4fcc47b0ad 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -299,10 +299,21 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqDataRsp dataRsp = {0}; tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); dataRsp.rspOffset = fetchOffsetNew; + code = 0; if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; } - goto OVER; + taosArrayDestroy(dataRsp.blockDataLen); + taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree); + + if (dataRsp.withSchema) { + taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + } + + if (dataRsp.withTbName) { + taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree); + } + return code; } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey %s, no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey); From c5ccdfd4e9e5e1560dbe9717f803aa68c899b543 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Jul 2022 11:06:41 +0800 Subject: [PATCH 08/13] fix(query): set the correct resource free flag. --- include/libs/function/function.h | 8 ++++---- source/libs/scalar/src/scalar.c | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 7722ed7ba1..4d27325d75 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -173,12 +173,12 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); typedef enum { - CREATED_COLDATA = 0x1, // the newly created column data needs to be destroyed. - INPUT_COLDATA = 0x2, // input column data should not be released. -} SCOLDATA_TYPE_E; + SHOULD_FREE_COLDATA = 0x1, // the newly created column data needs to be destroyed. + DELEGATED_MGMT_COLDATA = 0x2, // input column data should not be released. +} ECOLDATA_MGMT_TYPE_E; struct SScalarParam { - SCOLDATA_TYPE_E type; + ECOLDATA_MGMT_TYPE_E type; SColumnInfoData *columnData; SHashObj *pHashFilter; int32_t hashValueType; diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 5c8e148e2d..2cbe1e5c96 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -55,7 +55,7 @@ int32_t sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows, SScalarPara } pParam->columnData = pColumnData; - pParam->type = CREATED_COLDATA; + pParam->type = SHOULD_FREE_COLDATA; return TSDB_CODE_SUCCESS; } @@ -162,7 +162,7 @@ void sclFreeRes(SHashObj *res) { void sclFreeParam(SScalarParam *param) { if (param->columnData != NULL) { colDataDestroy(param->columnData); - taosMemoryFree(param->columnData); + taosMemoryFreeClear(param->columnData); } if (param->pHashFilter != NULL) { @@ -568,9 +568,9 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp _return: for (int32_t i = 0; i < paramNum; ++i) { - if (params[i].type == CREATED_COLDATA) { + if (params[i].type == SHOULD_FREE_COLDATA) { colDataDestroy(params[i].columnData); - taosMemoryFree(params[i].columnData); + taosMemoryFreeClear(params[i].columnData); } } @@ -842,6 +842,7 @@ EDealRes sclWalkFunction(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; @@ -876,6 +877,7 @@ EDealRes sclWalkOperator(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + output.type = DELEGATED_MGMT_COLDATA; if (taosHashPut(ctx->pRes, &pNode, POINTER_BYTES, &output, sizeof(output))) { ctx->code = TSDB_CODE_QRY_OUT_OF_MEMORY; return DEAL_RES_ERROR; From 21742bba095e4aae23a5ab4d793463a02da855e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Jul 2022 12:00:48 +0800 Subject: [PATCH 09/13] fix(query): add error code check. --- source/libs/executor/src/scanoperator.c | 11 +++++- .../libs/scalar/test/filter/filterTests.cpp | 36 +++++++++---------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3fbdcfc308..56e6f393cc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1839,7 +1839,16 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { SMetaReader mr = {0}; metaReaderInit(&mr, pInfo->readHandle.meta, 0); - metaGetTableEntryByUid(&mr, pInfo->pCur->mr.me.ctbEntry.suid); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + longjmp(pTaskInfo->env, terrno); + } // number of columns pColInfoData = taosArrayGet(p->pDataBlock, 3); diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 8f9332cef8..deba8983f3 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -211,7 +211,7 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { TEST(timerangeTest, greater) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -234,7 +234,7 @@ TEST(timerangeTest, greater) { TEST(timerangeTest, greater_and_lower) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -265,7 +265,7 @@ TEST(timerangeTest, greater_and_lower) { TEST(timerangeTest, greater_equal_and_lower_equal) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -297,7 +297,7 @@ TEST(timerangeTest, greater_equal_and_lower_equal) { TEST(timerangeTest, greater_and_lower_not_strict) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode1 = NULL, *logicNode2 = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int64_t tsmall1 = 222, tbig1 = 333; int64_t tsmall2 = 444, tbig2 = 555; SNode *list[2] = {0}; @@ -350,7 +350,7 @@ TEST(columnTest, smallint_column_greater_double_value) { double rightv= 2.5; int8_t eRes[5] = {0, 0, 1, 1, 1}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); @@ -405,7 +405,7 @@ TEST(columnTest, int_column_greater_smallint_value) { int16_t rightv= 4; int8_t eRes[5] = {0, 0, 1, 1, 1}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); @@ -460,7 +460,7 @@ TEST(columnTest, int_column_in_double_list) { double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; bool eRes[5] = {true, true, true, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); @@ -503,7 +503,7 @@ TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; for (int32_t i = 0; i < 5; ++i) { @@ -567,7 +567,7 @@ TEST(columnTest, binary_column_like_binary) { char rightv[64] = {0}; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; bool eRes[5] = {true, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -614,7 +614,7 @@ TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; bool eRes[5] = {false, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -661,7 +661,7 @@ TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; bool eRes[5] = {true, true, true, true, false}; for (int32_t i = 0; i < 5; ++i) { @@ -710,7 +710,7 @@ TEST(opTest, smallint_column_greater_int_column) { int32_t rightv[5]= {0, -5, -4, 23, 100}; bool eRes[5] = {true, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); @@ -747,7 +747,7 @@ TEST(opTest, smallint_value_add_int_column) { int16_t rightv[5]= {0, -1, -4, -1, 100}; bool eRes[5] = {true, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); @@ -790,7 +790,7 @@ TEST(opTest, bigint_column_multi_binary_column) { } bool eRes[5] = {false, true, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); @@ -833,7 +833,7 @@ TEST(opTest, smallint_column_and_binary_column) { } bool eRes[5] = {false, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); @@ -871,7 +871,7 @@ TEST(opTest, smallint_column_or_float_column) { float rightv[5]= {2.0, 3.0, 0, 5.2, 6.0}; bool eRes[5] = {true, true, false, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); @@ -909,7 +909,7 @@ TEST(opTest, smallint_column_or_double_value) { double rightv= 10.2; bool eRes[5] = {true, true, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); @@ -945,7 +945,7 @@ TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {0}; + SScalarParam res = {.type = SHOULD_FREE_COLDATA}; bool eRes[5] = {false, true, false, true, false}; for (int32_t i = 0; i < 5; ++i) { From 5772b0bbf812d67beca3a6fdf12dff5f24494ca6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Jul 2022 12:13:21 +0800 Subject: [PATCH 10/13] fix(query): add error code check. --- source/libs/executor/src/scanoperator.c | 82 ++++++++++++++++++------- 1 file changed, 61 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 56e6f393cc..360a7b2ed0 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -39,8 +39,8 @@ static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capac static int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, const char* dbName); -static void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock); +static int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, + SSDataBlock* pBlock, const char* idStr); static bool processBlockWithProbability(const SSampleExecInfo* pInfo); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { @@ -264,7 +264,11 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca // currently only the tbname pseudo column if (pTableScanInfo->pseudoSup.numOfExprs > 0) { SExprSupp* pSup = &pTableScanInfo->pseudoSup; - addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock); + + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pBlock, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } } int64_t st = taosGetTimestampMs(); @@ -298,16 +302,21 @@ static void prepareForDescendingScan(STableScanInfo* pTableScanInfo, SqlFunction taosqsort(pCond->twindows, pCond->numOfTWindows, sizeof(STimeWindow), pCond, compareTimeWindow); } -void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, - SSDataBlock* pBlock) { +int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, + SSDataBlock* pBlock, const char* idStr) { // currently only the tbname pseudo column if (numOfPseudoExpr == 0) { - return; + return TSDB_CODE_SUCCESS; } SMetaReader mr = {0}; metaReaderInit(&mr, pHandle->meta, 0); - metaGetTableEntryByUid(&mr, pBlock->info.uid); + int32_t code = metaGetTableEntryByUid(&mr, pBlock->info.uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr); + metaReaderClear(&mr); + return terrno; + } for (int32_t j = 0; j < numOfPseudoExpr; ++j) { SExprInfo* pExpr = &pPseudoExpr[j]; @@ -349,6 +358,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ } metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; } void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId) { @@ -680,34 +690,46 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* return pOperator; } -static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) { - int32_t rowLen = 0; +static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { + *rowLen = 0; SMetaReader mr = {0}; metaReaderInit(&mr, pMeta, 0); - metaGetTableEntryByUid(&mr, uid); + int32_t code = metaGetTableEntryByUid(&mr, uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + if (mr.me.type == TSDB_SUPER_TABLE) { int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; for (int32_t i = 0; i < numOfCols; ++i) { - rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; } } else if (mr.me.type == TSDB_CHILD_TABLE) { uint64_t suid = mr.me.ctbEntry.suid; - metaGetTableEntryByUid(&mr, suid); + code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; for (int32_t i = 0; i < numOfCols; ++i) { - rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; } } else if (mr.me.type == TSDB_NORMAL_TABLE) { int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols; for (int32_t i = 0; i < numOfCols; ++i) { - rowLen += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; + (*rowLen) += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; } } metaReaderClear(&mr); - return rowLen; + return TSDB_CODE_SUCCESS; } static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { @@ -716,9 +738,13 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { } SBlockDistInfo* pBlockScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - blockDistInfo.rowSize = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid); + int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, &blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); @@ -1193,7 +1219,10 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { - addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } } doFilter(pInfo->pCondition, pInfo->pRes); @@ -1414,7 +1443,10 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { - addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); + code = addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } } doFilter(pInfo->pCondition, pInfo->pRes); @@ -2243,7 +2275,12 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { STableKeyInfo* item = taosArrayGet(pInfo->pTableList->pTableList, pInfo->curPos); - metaGetTableEntryByUid(&mr, item->uid); + int32_t code = metaGetTableEntryByUid(&mr, item->uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%"PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + longjmp(pTaskInfo->env, terrno); + } for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); @@ -2528,8 +2565,11 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc // currently only the tbname pseudo column if (pTableScanInfo->numOfPseudoExpr > 0) { - addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, pTableScanInfo->numOfPseudoExpr, - pBlock); + int32_t code = addTagPseudoColumnData(&pTableScanInfo->readHandle, pTableScanInfo->pPseudoExpr, pTableScanInfo->numOfPseudoExpr, + pBlock, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } } int64_t st = taosGetTimestampMs(); From 5ee4a196def4ee873e6483be21e2402926ed490e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 9 Jul 2022 14:01:25 +0800 Subject: [PATCH 11/13] fix(query): update the ut. --- .../libs/scalar/test/filter/filterTests.cpp | 59 +++++++++++++------ 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index deba8983f3..d8c64eef55 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -205,13 +205,19 @@ void flttMakeListNode(SNode **pNode, SNodeList *list, int32_t resType) { *pNode = (SNode *)lnode; } +void initScalarParam(SScalarParam* pParam) { + memset(pParam, 0, sizeof(SScalarParam)); + pParam->type = SHOULD_FREE_COLDATA; +} } TEST(timerangeTest, greater) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); + int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -234,7 +240,8 @@ TEST(timerangeTest, greater) { TEST(timerangeTest, greater_and_lower) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -265,7 +272,8 @@ TEST(timerangeTest, greater_and_lower) { TEST(timerangeTest, greater_equal_and_lower_equal) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int64_t tsmall = 222, tbig = 333; flttMakeColumnNode(&pcol, NULL, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 0, NULL); flttMakeValueNode(&pval, TSDB_DATA_TYPE_TIMESTAMP, &tsmall); @@ -297,7 +305,8 @@ TEST(timerangeTest, greater_equal_and_lower_equal) { TEST(timerangeTest, greater_and_lower_not_strict) { SNode *pcol = NULL, *pval = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode1 = NULL, *logicNode2 = NULL; bool eRes[5] = {false, false, true, true, true}; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int64_t tsmall1 = 222, tbig1 = 333; int64_t tsmall2 = 444, tbig2 = 555; SNode *list[2] = {0}; @@ -350,7 +359,8 @@ TEST(columnTest, smallint_column_greater_double_value) { double rightv= 2.5; int8_t eRes[5] = {0, 0, 1, 1, 1}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); @@ -405,7 +415,8 @@ TEST(columnTest, int_column_greater_smallint_value) { int16_t rightv= 4; int8_t eRes[5] = {0, 0, 1, 1, 1}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); @@ -460,7 +471,8 @@ TEST(columnTest, int_column_in_double_list) { double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; bool eRes[5] = {true, true, true, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); @@ -503,7 +515,8 @@ TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; bool eRes[5] = {true, true, false, false, false}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); char leftv[5][5]= {0}; char rightv[3][5]= {0}; for (int32_t i = 0; i < 5; ++i) { @@ -567,7 +580,8 @@ TEST(columnTest, binary_column_like_binary) { char rightv[64] = {0}; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {true, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -614,7 +628,8 @@ TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {false, false, true, false, true}; for (int32_t i = 0; i < 5; ++i) { @@ -661,7 +676,8 @@ TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {true, true, true, true, false}; for (int32_t i = 0; i < 5; ++i) { @@ -710,7 +726,8 @@ TEST(opTest, smallint_column_greater_int_column) { int32_t rightv[5]= {0, -5, -4, 23, 100}; bool eRes[5] = {true, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); @@ -747,7 +764,8 @@ TEST(opTest, smallint_value_add_int_column) { int16_t rightv[5]= {0, -1, -4, -1, 100}; bool eRes[5] = {true, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeValueNode(&pLeft, TSDB_DATA_TYPE_INT, &leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); @@ -790,7 +808,8 @@ TEST(opTest, bigint_column_multi_binary_column) { } bool eRes[5] = {false, true, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); @@ -833,7 +852,8 @@ TEST(opTest, smallint_column_and_binary_column) { } bool eRes[5] = {false, false, true, false, true}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); @@ -871,7 +891,8 @@ TEST(opTest, smallint_column_or_float_column) { float rightv[5]= {2.0, 3.0, 0, 5.2, 6.0}; bool eRes[5] = {true, true, false, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(rightv)/sizeof(rightv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); @@ -909,7 +930,8 @@ TEST(opTest, smallint_column_or_double_value) { double rightv= 10.2; bool eRes[5] = {true, true, true, true, true}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); @@ -945,7 +967,8 @@ TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; SSDataBlock *src = NULL; - SScalarParam res = {.type = SHOULD_FREE_COLDATA}; + SScalarParam res; + initScalarParam(&res); bool eRes[5] = {false, true, false, true, false}; for (int32_t i = 0; i < 5; ++i) { From dd81770841a981fbe15c833e021d68d00682594f Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 9 Jul 2022 15:44:37 +0800 Subject: [PATCH 12/13] fix: rsma ref fix and add debug logs --- source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/inc/sma.h | 8 +- source/dnode/vnode/src/inc/vnodeInt.h | 2 + source/dnode/vnode/src/sma/sma.c | 257 ------------------- source/dnode/vnode/src/sma/smaCommit.c | 3 +- source/dnode/vnode/src/sma/smaEnv.c | 83 +++++- source/dnode/vnode/src/sma/smaRollup.c | 31 ++- source/dnode/vnode/src/sma/smaTimeRange.c | 30 +++ source/dnode/vnode/src/sma/smaUtil.c | 19 ++ source/dnode/vnode/src/vnd/vnodeModule.c | 1 + tests/system-test/0-others/taosShellError.py | 22 +- 11 files changed, 162 insertions(+), 295 deletions(-) delete mode 100644 source/dnode/vnode/src/sma/sma.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 24c4f2912c..18b8f6d79b 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -27,7 +27,6 @@ target_sources( "src/meta/metaSnapshot.c" # sma - "src/sma/sma.c" "src/sma/smaEnv.c" "src/sma/smaUtil.c" "src/sma/smaOpen.c" diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 2d6edae0e7..7183c423fe 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -47,7 +47,8 @@ struct SSmaEnv { }; typedef struct { - int32_t smaRef; + int8_t inited; + int32_t rsetId; } SSmaMgmt; #define SMA_ENV_LOCK(env) ((env)->lock) @@ -95,6 +96,7 @@ enum { TASK_TRIGGER_STAT_CANCELLED = 4, TASK_TRIGGER_STAT_FINISHED = 5, }; + void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -104,6 +106,10 @@ int32_t tdInsertRSmaData(SSma *pSma, char *msg); int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); + +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln); + int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); int32_t tdLockSma(SSma *pSma); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index ae659dc396..34fac045a7 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -163,6 +163,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool const char* stbFullName, int32_t vgId); // sma +int32_t smaInit(); +void smaCleanUp(); int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); int32_t smaBegin(SSma* pSma); diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c deleted file mode 100644 index 12f93f9400..0000000000 --- a/source/dnode/vnode/src/sma/sma.c +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "sma.h" - -// functions for external invocation - -// TODO: Who is responsible for resource allocate and release? -int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg) { - int32_t code = TSDB_CODE_SUCCESS; - - if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { - smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - // TODO: destroy SSDataBlocks(msg) - return code; -} - -int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg) { - int32_t code = TSDB_CODE_SUCCESS; - - if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { - smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); - } - // TODO: destroy SSDataBlocks(msg) - return code; -} - -int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t* days) { - int32_t code = TSDB_CODE_SUCCESS; - if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { - smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); - } - smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); - return code; -} - - -// functions for internal invocation - -#if 0 - -/** - * @brief TODO: Assume that the final generated result it less than 3M - * - * @param pReq - * @param pDataBlocks - * @param vgId - * @param suid // TODO: check with Liao whether suid response is reasonable - * - * TODO: colId should be set - */ -int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, - tb_uid_t suid, const char* stbName, bool isCreateCtb) { - int32_t sz = taosArrayGetSize(pDataBlocks); - int32_t bufSize = sizeof(SSubmitReq); - for (int32_t i = 0; i < sz; ++i) { - SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); - bufSize += sizeof(SSubmitBlk); - } - - *pReq = taosMemoryCalloc(1, bufSize); - if (!(*pReq)) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } - void* pDataBuf = *pReq; - - SArray* pTagArray = NULL; - int32_t msgLen = sizeof(SSubmitReq); - int32_t numOfBlks = 0; - int32_t schemaLen = 0; - SRowBuilder rb = {0}; - tdSRowInit(&rb, pTSchema->version); - - for (int32_t i = 0; i < sz; ++i) { - SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); - SDataBlockInfo* pDataBlkInfo = &pDataBlock->info; - int32_t colNum = pDataBlkInfo->numOfCols; - int32_t rows = pDataBlkInfo->rows; - int32_t rowSize = pDataBlkInfo->rowSize; - int64_t groupId = pDataBlkInfo->groupId; - - if (rb.nCols != colNum) { - tdSRowSetTpInfo(&rb, colNum, pTSchema->flen); - } - - if(isCreateCtb) { - SMetaReader mr = {0}; - const char* ctbName = buildCtbNameByGroupId(stbName, pDataBlock->info.groupId); - if (metaGetTableEntryByName(&mr, ctbName) != 0) { - smaDebug("vgId:%d, no tsma ctb %s exists", vgId, ctbName); - } - SVCreateTbReq ctbReq = {0}; - ctbReq.name = ctbName; - ctbReq.type = TSDB_CHILD_TABLE; - ctbReq.ctb.suid = suid; - - STagVal tagVal = {.cid = colNum + PRIMARYKEY_TIMESTAMP_COL_ID, - .type = TSDB_DATA_TYPE_BIGINT, - .i64 = groupId}; - STag* pTag = NULL; - if(!pTagArray) { - pTagArray = taosArrayInit(1, sizeof(STagVal)); - if (!pTagArray) goto _err; - } - taosArrayClear(pTagArray); - taosArrayPush(pTagArray, &tagVal); - tTagNew(pTagArray, 1, false, &pTag); - if (pTag == NULL) { - tdDestroySVCreateTbReq(&ctbReq); - goto _err; - } - ctbReq.ctb.pTag = (uint8_t*)pTag; - - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &ctbReq, schemaLen, code); - - tdDestroySVCreateTbReq(&ctbReq); - if (code < 0) { - goto _err; - } - } - - - - SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen); - pSubmitBlk->suid = suid; - pSubmitBlk->uid = groupId; - pSubmitBlk->numOfRows = rows; - - msgLen += sizeof(SSubmitBlk); - int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row - tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen)); // set row buf - bool isStartKey = false; - int32_t offset = 0; - for (int32_t k = 0; k < colNum; ++k) { // iterate by column - SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); - STColumn* pCol = &pTSchema->columns[k]; - void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); - switch (pColInfoData->info.type) { - case TSDB_DATA_TYPE_TIMESTAMP: - if (!isStartKey) { - isStartKey = true; - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, true, - offset, k); - - } else { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_TIMESTAMP, TD_VTYPE_NORM, var, - true, offset, k); - } - break; - case TSDB_DATA_TYPE_NCHAR: { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_NCHAR, TD_VTYPE_NORM, var, true, - offset, k); - break; - } - case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, TSDB_DATA_TYPE_VARCHAR, TD_VTYPE_NORM, var, true, - offset, k); - break; - } - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_MEDIUMBLOB: - uError("the column type %" PRIi16 " is defined but not implemented yet", pColInfoData->info.type); - TASSERT(0); - break; - default: - if (pColInfoData->info.type < TSDB_DATA_TYPE_MAX && pColInfoData->info.type > TSDB_DATA_TYPE_NULL) { - if (pCol->type == pColInfoData->info.type) { - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, var, true, offset, - k); - } else { - char tv[8] = {0}; - if (pColInfoData->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (pColInfoData->info.type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else if (IS_SIGNED_NUMERIC_TYPE(pColInfoData->info.type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } else { - uint64_t v = 0; - GET_TYPED_DATA(v, uint64_t, pColInfoData->info.type, var); - SET_TYPED_DATA(&tv, pCol->type, v); - } - tdAppendColValToRow(&rb, PRIMARYKEY_TIMESTAMP_COL_ID + k, pCol->type, TD_VTYPE_NORM, tv, true, offset, - k); - } - } else { - uError("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); - TASSERT(0); - } - break; - } - offset += TYPE_BYTES[pCol->type]; // sum/avg would convert to int64_t/uint64_t/double during aggregation - } - dataLen += TD_ROW_LEN(rb.pBuf); -#ifdef TD_DEBUG_PRINT_ROW - tdSRowPrint(rb.pBuf, pTSchema, __func__); -#endif - } - - ++numOfBlks; - - pSubmitBlk->dataLen = dataLen; - msgLen += pSubmitBlk->dataLen; - } - - (*pReq)->length = msgLen; - - (*pReq)->header.vgId = htonl(vgId); - (*pReq)->header.contLen = htonl(msgLen); - (*pReq)->length = (*pReq)->header.contLen; - (*pReq)->numOfBlocks = htonl(numOfBlks); - SSubmitBlk* blk = (SSubmitBlk*)((*pReq) + 1); - while (numOfBlks--) { - int32_t dataLen = blk->dataLen; - blk->uid = htobe64(blk->uid); - blk->suid = htobe64(blk->suid); - blk->padding = htonl(blk->padding); - blk->sversion = htonl(blk->sversion); - blk->dataLen = htonl(blk->dataLen); - blk->schemaLen = htonl(blk->schemaLen); - blk->numOfRows = htons(blk->numOfRows); - blk = (SSubmitBlk*)(blk->data + dataLen); - } - return TSDB_CODE_SUCCESS; -_err: - taosMemoryFreeClear(*pReq); - taosArrayDestroy(pTagArray); - - return TSDB_CODE_FAILED; -} -#endif diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 6e75176136..e241c14fc7 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -121,7 +121,7 @@ static int32_t tdProcessRSmaPreCommitImpl(SSma *pSma) { // step 3: perform persist task for qTaskInfo tdRSmaPersistExecImpl(pRSmaStat); - smaDebug("vgId:%d, rsma pre commit succeess", SMA_VID(pSma)); + smaDebug("vgId:%d, rsma pre commit success", SMA_VID(pSma)); return TSDB_CODE_SUCCESS; } @@ -173,6 +173,7 @@ static int32_t tdProcessRSmaPostCommitImpl(SSma *pSma) { } if ((pDir = taosOpenDir(dir)) == NULL) { + regfree(®ex); terrno = TAOS_SYSTEM_ERROR(errno); smaWarn("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index c7b938f884..bb8dd48236 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -18,7 +18,7 @@ typedef struct SSmaStat SSmaStat; #define RSMA_TASK_INFO_HASH_SLOT 8 -#define SMA_MGMT_REF_NUM 1024 +#define SMA_MGMT_REF_NUM 10240 extern SSmaMgmt smaMgmt; @@ -30,7 +30,62 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaE static void *tdFreeTSmaStat(STSmaStat *pStat); static void tdDestroyRSmaStat(void *pRSmaStat); +/** + * @brief rsma init + * + * @return int32_t + */ // implementation +int32_t smaInit() { + int8_t old; + int32_t nLoops = 0; + while (1) { + old = atomic_val_compare_exchange_8(&smaMgmt.inited, 0, 2); + if (old != 2) break; + if (++nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + if (old == 0) { + smaMgmt.rsetId = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat); + + if (smaMgmt.rsetId < 0) { + smaError("failed to init sma rset since %s", terrstr()); + atomic_store_8(&smaMgmt.inited, 0); + return TSDB_CODE_FAILED; + } + + smaInfo("sma rset is initialized, rsetId:%d", smaMgmt.rsetId); + atomic_store_8(&smaMgmt.inited, 1); + } + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief rsma cleanup + * + */ +void smaCleanUp() { + int8_t old; + int32_t nLoops = 0; + while (1) { + old = atomic_val_compare_exchange_8(&smaMgmt.inited, 1, 2); + if (old != 2) break; + if (++nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + } + + if (old == 1) { + smaInfo("sma rset is cleaned up, resetId:%d", smaMgmt.rsetId); + taosCloseRef(smaMgmt.rsetId); + atomic_store_8(&smaMgmt.inited, 0); + } +} static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) { SSmaEnv *pEnv = NULL; @@ -135,17 +190,16 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT); // init smaMgmt - smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat); - if (smaMgmt.smaRef < 0) { - smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; - } + smaInit(); - int64_t refId = taosAddRef(smaMgmt.smaRef, pRSmaStat); + int64_t refId = taosAddRef(smaMgmt.rsetId, pRSmaStat); if (refId < 0) { - smaError("taosAddRef smaRef failed, since:%s", tstrerror(terrno)); + smaError("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d failed since:%s", SMA_VID(pSma), + refId, smaMgmt.rsetId, SMA_MGMT_REF_NUM, tstrerror(terrno)); return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, taosAddRef refId:%" PRIi64 " to rsetId rsetId:%d max:%d succeed", SMA_VID(pSma), refId, + smaMgmt.rsetId, SMA_MGMT_REF_NUM); } pRSmaStat->refId = refId; @@ -275,8 +329,13 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat); - if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) { - smaError("remove refId from rsmaRef:0x%" PRIx64 " failed since %s", RSMA_REF_ID(pRSmaStat), terrstr()); + if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) { + smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", SMA_VID(pRSmaStat->pSma), + RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId, terrstr()); + ASSERT(0); + } else { + smaDebug("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " succeed", SMA_VID(pRSmaStat->pSma), + RSMA_REF_ID(pRSmaStat), smaMgmt.rsetId); } } else { ASSERT(0); @@ -323,7 +382,7 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { } break; default: - TASSERT(0); + smaError("vgId:%d undefined smaType:%", SMA_VID(pSma), smaType); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 4c2e606e0e..c44a46ac5a 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -19,7 +19,8 @@ #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid SSmaMgmt smaMgmt = { - .smaRef = -1, + .inited = 0, + .rsetId = -1, }; #define TD_QTASKINFO_FNAME_PREFIX "qtaskinfo.ver" @@ -608,9 +609,11 @@ _err: static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; SSma *pSma = NULL; - SRSmaStat *pStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, pItem->refId); + SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + if (!pStat) { - smaDebug("rsma fetch task not start since already destroyed"); + smaDebug("rsma fetch task not start since already destroyed, rsetId rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + pItem->refId); return; } @@ -622,9 +625,10 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { case TASK_TRIGGER_STAT_PAUSED: case TASK_TRIGGER_STAT_CANCELLED: case TASK_TRIGGER_STAT_FINISHED: { - taosReleaseRef(smaMgmt.smaRef, pItem->refId); - smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled", - SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); + tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is %" PRIi8 + ", rsetId rsetId:%" PRIi64 " refId:%d", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid, rsmaTriggerStat, smaMgmt.rsetId, pItem->refId); return; } default: @@ -665,7 +669,7 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { } _end: - taosReleaseRef(smaMgmt.smaRef, pItem->refId); + tdReleaseSmaRef(smaMgmt.rsetId, pItem->refId, __func__, __LINE__); } static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid, @@ -1258,7 +1262,8 @@ _end: } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); - taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId); + smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d", SMA_VID(pSma), smaMgmt.rsetId, pRSmaStat->refId); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__); taosThreadExit(NULL); return NULL; } @@ -1283,7 +1288,9 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); - taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId); + smaDebug("vgId:%d, release rsetId rsetId:%" PRIi64 " refId:%d)", SMA_VID(pRSmaStat->pSma), smaMgmt.rsetId, + pRSmaStat->refId); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaStat->refId, __func__, __LINE__); } taosThreadAttrDestroy(&thAttr); @@ -1297,8 +1304,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { SRSmaStat *rsmaStat = param; - SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, rsmaStat->refId); - + SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.rsetId, rsmaStat->refId); + ASSERT(0); if (!pRSmaStat) { smaDebug("rsma persistence task not start since already destroyed"); return; @@ -1341,5 +1348,5 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); } break; } - taosReleaseRef(smaMgmt.smaRef, rsmaStat->refId); + taosReleaseRef(smaMgmt.rsetId, rsmaStat->refId); } diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index ecd8db8ea9..6d71f3a250 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -20,6 +20,36 @@ #define SMA_STORAGE_MINUTES_DAY 1440 #define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file +// TODO: Who is responsible for resource allocate and release? +int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + + if ((code = tdProcessTSmaInsertImpl(pSma, indexUid, msg)) < 0) { + smaWarn("vgId:%d, insert tsma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + // TODO: destroy SSDataBlocks(msg) + return code; +} + +int32_t tdProcessTSmaCreate(SSma *pSma, int64_t version, const char *msg) { + int32_t code = TSDB_CODE_SUCCESS; + + if ((code = tdProcessTSmaCreateImpl(pSma, version, msg)) < 0) { + smaWarn("vgId:%d, create tsma failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + // TODO: destroy SSDataBlocks(msg) + return code; +} + +int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdProcessTSmaGetDaysImpl(pCfg, pCont, contLen, days)) < 0) { + smaWarn("vgId:%d, get tsma days failed since %s", pCfg->vgId, tstrerror(terrno)); + } + smaDebug("vgId:%d, get tsma days %d", pCfg->vgId, *days); + return code; +} + /** * @brief Judge the tsma file split days * diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 2ce6d23648..22d82e8df0 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -294,4 +294,23 @@ int32_t tdRemoveTFile(STFile *pTFile) { } // smaXXXUtil ================ +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { + void *pResult = taosAcquireRef(rsetId, refId); + if (!pResult) { + smaWarn("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + } else { + smaDebug("%s:%d taosAcquireRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + } + return pResult; +} + +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId, const char *tags, int32_t ln) { + if (taosReleaseRef(rsetId, refId) < 0) { + smaWarn("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d failed since %s", tags, ln, rsetId, refId, terrstr()); + return TSDB_CODE_FAILED; + } + smaDebug("%s:%d taosReleaseRef for rsetId:%" PRIi64 " refId:%d success", tags, ln, rsetId, refId); + + return TSDB_CODE_SUCCESS; +} // ... \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index d0aede145e..69f87a2c1a 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -100,6 +100,7 @@ void vnodeCleanup() { walCleanUp(); tqCleanUp(); + smaCleanUp(); } int vnodeScheduleTask(int (*execute)(void*), void* arg) { diff --git a/tests/system-test/0-others/taosShellError.py b/tests/system-test/0-others/taosShellError.py index 825ac015fb..5735e55c03 100644 --- a/tests/system-test/0-others/taosShellError.py +++ b/tests/system-test/0-others/taosShellError.py @@ -162,28 +162,28 @@ class TDTestCase: keyDict['h'] = 'abc' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) keyDict['h'] = '\'abc\'' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) keyDict['h'] = '3' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) keyDict['h'] = '\'3\'' retCode, retVal = taos_command(buildPath, "h", keyDict['h'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -h %s test suceess"%keyDict['h']) + tdLog.info("taos -h %s test success"%keyDict['h']) else: tdLog.exit("taos -h %s fail"%keyDict['h']) @@ -193,42 +193,42 @@ class TDTestCase: keyDict['P'] = 'abc' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Invalid port" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '\'abc\'' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Invalid port" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '3' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '\'3\'' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '12ab' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) keyDict['P'] = '\'12ab\'' retCode, retVal = taos_command(buildPath, "P", keyDict['P'], "taos>", keyDict['c'], '') if (retCode == "TAOS_FAIL") and ("Unable to establish connection" in retVal): - tdLog.info("taos -P %s test suceess"%keyDict['P']) + tdLog.info("taos -P %s test success"%keyDict['P']) else: tdLog.exit("taos -P %s fail"%keyDict['P']) @@ -293,7 +293,7 @@ class TDTestCase: keyDict['p'] = 'errorPassword' retCode, retVal = taos_command(buildPath, "u", keyDict['u'], "taos>", keyDict['c'], sqlString, 'p', keyDict['p']) if retCode == "TAOS_FAIL" and "Authentication failure" in retVal: - tdLog.info("taos -p %s test suceess"%keyDict['p']) + tdLog.info("taos -p %s test success"%keyDict['p']) else: tdLog.exit("taos -u %s -p %s"%(keyDict['u'], keyDict['p'])) From 2e63f89f822461492880330cc385ee7386dc1ca2 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 9 Jul 2022 16:13:31 +0800 Subject: [PATCH 13/13] trigger CI