From ccffe94579d3495b6d4e825b308c4358f8f98058 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 26 Apr 2022 18:46:08 +0800 Subject: [PATCH 01/12] test: add client config for test cases --- tests/pytest/util/dnodes.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index c3c2424397..2c50a62117 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -63,7 +63,7 @@ class TDSimClient: if os.system(cmd) != 0: tdLog.exit(cmd) - def deploy(self): + def deploy(self, *updatecfgDict): self.logDir = "%s/sim/psim/log" % (self.path) self.cfgDir = "%s/sim/psim/cfg" % (self.path) self.cfgPath = "%s/sim/psim/cfg/taos.cfg" % (self.path) @@ -95,6 +95,12 @@ class TDSimClient: for key, value in self.cfgDict.items(): self.cfg(key, value) + + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: + print(updatecfgDict[0][0]) + clientCfg = dict (updatecfgDict[0][0].get('clientCfg')) + for key, value in clientCfg.items(): + self.cfg(key, value) tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) @@ -214,9 +220,11 @@ class TDDnode: # self.cfg("logDir",self.logDir) # print(updatecfgDict) isFirstDir = 1 - if updatecfgDict[0] and updatecfgDict[0][0]: + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: print(updatecfgDict[0][0]) for key, value in updatecfgDict[0][0].items(): + if key == "clientCfg": + continue if value == 'dataDir': if isFirstDir: self.cfgDict.pop('dataDir') @@ -491,7 +499,7 @@ class TDDnodes: self.sim.setTestCluster(self.testCluster) if (self.simDeployed == False): - self.sim.deploy() + self.sim.deploy(updatecfgDict) self.simDeployed = True self.check(index) From 09fc0b791defad7c53a121fcf375163646173418 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Apr 2022 20:26:32 +0800 Subject: [PATCH 02/12] refactor: do some internal refactor. --- source/libs/executor/inc/executorimpl.h | 28 ++-- source/libs/executor/src/executorMain.c | 2 +- source/libs/executor/src/executorimpl.c | 183 ++++++++++++++++------- source/libs/executor/src/groupoperator.c | 17 +-- source/libs/executor/src/scanoperator.c | 25 ++-- 5 files changed, 164 insertions(+), 91 deletions(-) diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 00ec92465f..b841224fe4 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -254,6 +254,17 @@ enum { OP_EXEC_DONE = 0x9, }; +typedef struct SOperatorFpSet { + __optr_open_fn_t _openFn; // DO NOT invoke this function directly + __optr_fn_t getNextFn; + __optr_fn_t getStreamResFn; // execute the aggregate in the stream model, todo remove it + __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP + __optr_close_fn_t closeFn; + __optr_encode_fn_t encodeResultRow; + __optr_decode_fn_t decodeResultRow; + __optr_get_explain_fn_t getExplainFn; +} SOperatorFpSet; + typedef struct SOperatorInfo { uint8_t operatorType; bool blockingOptr; // block operator or not @@ -267,15 +278,7 @@ typedef struct SOperatorInfo { SResultInfo resultInfo; struct SOperatorInfo** pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - // todo extract struct - __optr_open_fn_t _openFn; // DO NOT invoke this function directly - __optr_fn_t getNextFn; - __optr_fn_t getStreamResFn; // execute the aggregate in the stream model. - __optr_fn_t cleanupFn; // call this function to release the allocated resources ASAP - __optr_close_fn_t closeFn; - __optr_encode_fn_t encodeResultRow; - __optr_decode_fn_t decodeResultRow; - __optr_get_explain_fn_t getExplainFn; + SOperatorFpSet fpSet; } SOperatorInfo; typedef struct { @@ -609,6 +612,10 @@ typedef struct SJoinOperatorInfo { SNode *pOnCondition; } SJoinOperatorInfo; +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_get_explain_fn_t explain); + int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); void operatorDummyCloseFn(void* param, int32_t numOfCols); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); @@ -653,6 +660,9 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo); SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, int64_t gap, STimeWindowAggSupp *pTwAggSupp, SExecTaskInfo* pTaskInfo); SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 5cbda90733..516afe5553 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -159,7 +159,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) { int64_t st = 0; st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->getNextFn(pTaskInfo->pRoot, &newgroup); + *pRes = pTaskInfo->pRoot->fpSet.getNextFn(pTaskInfo->pRoot, &newgroup); uint64_t el = (taosGetTimestampUs() - st); pTaskInfo->cost.elapsedTime += el; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3541013015..318d87e100 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -226,6 +226,23 @@ int32_t operatorDummyOpenFn(SOperatorInfo* pOperator) { return TSDB_CODE_SUCCESS; } +SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_get_explain_fn_t explain) { + SOperatorFpSet fpSet = { + ._openFn = openFn, + .getNextFn = nextFn, + .getStreamResFn = streamFn, + .cleanupFn = cleanup, + .closeFn = closeFn, + .encodeResultRow = encode, + .decodeResultRow = decode, + .getExplainFn = explain, + }; + + return fpSet; +} + void operatorDummyCloseFn(void* param, int32_t numOfCols) {} static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, @@ -4081,7 +4098,7 @@ static SSDataBlock* doLoadRemoteData(SOperatorInfo* pOperator, bool* newgroup) { SExchangeInfo* pExchangeInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4176,9 +4193,9 @@ SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock pOperator->info = pInfo; pOperator->numOfOutput = pBlock->info.numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = prepareLoadRemoteData; // assign a dummy function. - pOperator->getNextFn = doLoadRemoteData; - pOperator->closeFn = destroyExchangeOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, destroyExchangeOperatorInfo, + NULL, NULL, NULL); #if 1 { // todo refactor @@ -4289,7 +4306,7 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SSDataBlock* loadNextDataBlock(void* param) { SOperatorInfo* pOperator = (SOperatorInfo*)param; bool newgroup = false; - return pOperator->getNextFn(pOperator, &newgroup); + return pOperator->fpSet.getNextFn(pOperator, &newgroup); } static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { @@ -4586,9 +4603,9 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->pExpr = pExprInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doSortedMerge; - pOperator->closeFn = destroySortedMergeOperatorInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSortedMerge, NULL, NULL, destroySortedMergeOperatorInfo, + NULL, NULL, NULL); code = appendDownstream(pOperator, downstream, numOfDownstream); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4667,8 +4684,8 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSDataBlock* pR pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doSort; - pOperator->closeFn = destroyOrderOperatorInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSort, NULL, NULL, destroyOrderOperatorInfo, + NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -4710,7 +4727,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { bool newgroup = true; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -4765,7 +4782,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator, bool* newgroup) } SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } @@ -5007,7 +5024,7 @@ static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator, bool* newgroup) // The downstream exec may change the value of the newgroup, so use a local variable instead. publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5070,7 +5087,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, &newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5121,9 +5138,9 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro SSDataBlock* pBlock = pInfo->binfo.pRes; if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - return pOperator->getStreamResFn(pOperator, newgroup); + return pOperator->fpSet.getStreamResFn(pOperator, newgroup); } else { - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; } @@ -5165,7 +5182,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator, bool* newgroup while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5216,7 +5233,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5265,7 +5282,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5400,7 +5417,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -5451,7 +5468,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -5531,7 +5548,7 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator, bool* newgroup) { SOperatorInfo* pDownstream = pOperator->pDownstream[0]; while (1) { publishOperatorProfEvent(pDownstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = pDownstream->getNextFn(pDownstream, newgroup); + SSDataBlock* pBlock = pDownstream->fpSet.getNextFn(pDownstream, newgroup); publishOperatorProfEvent(pDownstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (*newgroup) { @@ -5603,8 +5620,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { return; } - if (pOperator->closeFn != NULL) { - pOperator->closeFn(pOperator->info, pOperator->numOfOutput); + if (pOperator->fpSet.closeFn != NULL) { + pOperator->fpSet.closeFn(pOperator->info, pOperator->numOfOutput); } if (pOperator->pDownstream != NULL) { @@ -5739,12 +5756,9 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = doOpenAggregateOptr; - pOperator->getNextFn = getAggregateResult; - pOperator->closeFn = destroyAggOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + pOperator->fpSet = createOperatorFpSet(doOpenAggregateOptr, getAggregateResult, NULL, NULL, destroyAggOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5871,9 +5885,9 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SExprInfo* p pOperator->info = pInfo; pOperator->pExpr = pExprInfo; pOperator->numOfOutput = num; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doProjectOperation; - pOperator->closeFn = destroyProjectOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doProjectOperation, NULL, NULL, destroyProjectOperatorInfo, + NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; int32_t code = appendDownstream(pOperator, &downstream, 1); @@ -5929,12 +5943,9 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->pTaskInfo = pTaskInfo; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->_openFn = doOpenIntervalAgg; - pOperator->getNextFn = doBuildIntervalResult; - pOperator->getStreamResFn = doStreamIntervalAgg; - pOperator->closeFn = destroyIntervalOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5951,6 +5962,65 @@ _error: return NULL; } +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + STimeWindowAggSupp *pTwAggSupp, const STableGroupInfo* pTableGroupInfo, SExecTaskInfo* pTaskInfo) { + STableIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STableIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; + pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; + + int32_t numOfRows = 4096; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + + initResultSizeInfo(pOperator, numOfRows); + int32_t code = + initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { + goto _error; + } + + initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); + + pOperator->name = "StreamTimeIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; + pOperator->blockingOptr = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfOutput = numOfCols; + pOperator->info = pInfo; + + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doStreamIntervalAgg, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + + _error: + destroyIntervalOperatorInfo(pInfo, numOfCols); + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; + +} + SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo) { STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); @@ -5969,8 +6039,9 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doAllIntervalAgg; - pOperator->closeFn = destroyBasicOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doAllIntervalAgg, NULL, NULL, destroyBasicOperatorInfo, + NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6010,10 +6081,9 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf pOperator->pTaskInfo = pTaskInfo; pOperator->info = pInfo; - pOperator->getNextFn = doStateWindowAgg; - pOperator->closeFn = destroyStateWindowOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStateWindowAgg, NULL, NULL, destroyStateWindowOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); int32_t code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -6057,10 +6127,9 @@ SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SExprInfo pOperator->pExpr = pExprInfo; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->getNextFn = doSessionWindowAgg; - pOperator->closeFn = destroySWindowOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSessionWindowAgg, NULL, NULL, destroySWindowOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); pOperator->pTaskInfo = pTaskInfo; code = appendDownstream(pOperator, &downstream, 1); @@ -6147,12 +6216,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExp pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doFill; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doFill, NULL, NULL, destroySFillOperatorInfo, + NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; - - pOperator->closeFn = destroySFillOperatorInfo; - code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -7017,8 +7084,8 @@ int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SExplainExecInfo (*pRes)[*resNum].startupCost = operatorInfo->cost.openCost; (*pRes)[*resNum].totalCost = operatorInfo->cost.totalCost; - if (operatorInfo->getExplainFn) { - int32_t code = (*operatorInfo->getExplainFn)(operatorInfo, &(*pRes)->verboseInfo); + if (operatorInfo->fpSet.getExplainFn) { + int32_t code = (*operatorInfo->fpSet.getExplainFn)(operatorInfo, &(*pRes)->verboseInfo); if (code) { qError("operator getExplainFn failed, error:%s", tstrerror(code)); return code; @@ -7055,7 +7122,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) if (pJoinInfo->pLeft == NULL || pJoinInfo->leftPos >= pJoinInfo->pLeft->info.rows) { SOperatorInfo* ds1 = pOperator->pDownstream[0]; publishOperatorProfEvent(ds1, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pLeft = ds1->getNextFn(ds1, newgroup); + pJoinInfo->pLeft = ds1->fpSet.getNextFn(ds1, newgroup); publishOperatorProfEvent(ds1, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->leftPos = 0; @@ -7068,7 +7135,7 @@ static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator, bool* newgroup) if (pJoinInfo->pRight == NULL || pJoinInfo->rightPos >= pJoinInfo->pRight->info.rows) { SOperatorInfo* ds2 = pOperator->pDownstream[1]; publishOperatorProfEvent(ds2, QUERY_PROF_BEFORE_OPERATOR_EXEC); - pJoinInfo->pRight = ds2->getNextFn(ds2, newgroup); + pJoinInfo->pRight = ds2->fpSet.getNextFn(ds2, newgroup); publishOperatorProfEvent(ds2, QUERY_PROF_AFTER_OPERATOR_EXEC); pJoinInfo->rightPos = 0; @@ -7161,9 +7228,9 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOf pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->getNextFn = doMergeJoin; - pOperator->closeFn = destroyBasicOperatorInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeJoin, NULL, NULL, destroyBasicOperatorInfo, + NULL, NULL, NULL); int32_t code = appendDownstream(pOperator, pDownstream, numOfDownstream); return pOperator; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 27c616498e..fd3d6df640 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -277,7 +277,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -360,12 +360,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pOperator->numOfOutput = numOfCols; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = hashGroupbyAggregate; - pOperator->closeFn = destroyGroupOperatorInfo; - pOperator->encodeResultRow = aggEncodeResultRow; - pOperator->decodeResultRow = aggDecodeResultRow; + createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -562,7 +558,7 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator, bool* newgroup) { while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { break; @@ -618,14 +614,13 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->blockingOptr = true; pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pInfo->binfo.pRes = pResultBlock; pOperator->numOfOutput = numOfCols; pOperator->pExpr = pExprInfo; pOperator->info = pInfo; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = hashPartition; - pOperator->closeFn = destroyPartitionOperatorInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, + NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 277ab9bc6f..f4eee01007 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -405,7 +405,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, SQueryTableDataCon pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; + pOperator->fpSet.getNextFn = doTableScan; pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; @@ -429,7 +429,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle) { pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->getNextFn = doTableScanImpl; + pOperator->fpSet.getNextFn = doTableScanImpl; return pOperator; } @@ -502,8 +502,8 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo* // pOperator->operatorType = OP_TableBlockInfoScan; pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doBlockInfoScan; + pOperator->fpSet._openFn = operatorDummyOpenFn; + pOperator->fpSet.getNextFn = doBlockInfoScan; pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; @@ -532,7 +532,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; - pTaskInfo->code = pOperator->_openFn(pOperator); + pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { return NULL; } @@ -659,9 +659,9 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->_openFn = operatorDummyOpenFn; - pOperator->getNextFn = doStreamBlockScan; - pOperator->closeFn = operatorDummyCloseFn; + pOperator->fpSet._openFn = operatorDummyOpenFn; + pOperator->fpSet.getNextFn = doStreamBlockScan; + pOperator->fpSet.closeFn = operatorDummyCloseFn; pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -981,8 +981,8 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->numOfOutput = pResBlock->info.numOfCols; - pOperator->getNextFn = doSysTableScan; - pOperator->closeFn = destroySysScanOperator; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, + NULL, NULL, NULL); pOperator->pTaskInfo = pTaskInfo; return pOperator; @@ -1139,11 +1139,12 @@ SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, pOperator->blockingOptr = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->getNextFn = doTagScan; + + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doTagScan, NULL, NULL, destroyTagScanOperatorInfo, NULL, NULL, NULL); pOperator->pExpr = pExpr; pOperator->numOfOutput = numOfOutput; pOperator->pTaskInfo = pTaskInfo; - pOperator->closeFn = destroyTagScanOperatorInfo; return pOperator; _error: From c0b7b1302b31c22a6d856154fa7a7c96e9b4363b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 27 Apr 2022 09:27:12 +0800 Subject: [PATCH 03/12] refactor: do some internal refactor. --- source/libs/command/src/command.c | 15 ++++++++++++--- source/libs/executor/src/groupoperator.c | 2 +- source/libs/executor/test/executorTests.cpp | 4 ++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 4d4ac6c1e4..621ea7b7fc 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -27,9 +27,14 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } } +// todo : to convert data according to SSDatablock static void buildRspData(const STableMeta* pMeta, char* pData) { - int32_t* pColSizes = (int32_t*)pData; - pData += DESCRIBE_RESULT_COLS * sizeof(int32_t); + int32_t* payloadLen = (int32_t*) pData; + uint64_t* groupId = (uint64_t*)(pData + sizeof(int32_t)); + + int32_t* pColSizes = (int32_t*)(pData + sizeof(int32_t) + sizeof(uint64_t)); + pData = (char*) pColSizes + DESCRIBE_RESULT_COLS * sizeof(int32_t); + int32_t numOfRows = TABLE_TOTAL_COL_NUM(pMeta); // Field @@ -79,6 +84,9 @@ static void buildRspData(const STableMeta* pMeta, char* pData) { for (int32_t i = 0; i < DESCRIBE_RESULT_COLS; ++i) { pColSizes[i] = htonl(pColSizes[i]); } + + + *payloadLen = (int32_t)(pData - (char*)payloadLen); } static int32_t calcRspSize(const STableMeta* pMeta) { @@ -87,7 +95,8 @@ static int32_t calcRspSize(const STableMeta* pMeta) { (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_FIELD_LEN) + (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_TYPE_LEN) + (BitmapLen(numOfRows) + numOfRows * sizeof(int32_t)) + - (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN); + (numOfRows * sizeof(int32_t) + numOfRows * DESCRIBE_RESULT_NOTE_LEN) + + sizeof(int32_t) + sizeof(uint64_t); } static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index fd3d6df640..beee11ec18 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -361,7 +361,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 543df30686..de8df7b916 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -199,9 +199,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_ pOperator->name = "dummyInputOpertor4Test"; if (numOfCols == 1) { - pOperator->getNextFn = getDummyBlock; + pOperator->fpSet.getNextFn = getDummyBlock; } else { - pOperator->getNextFn = get2ColsDummyBlock; + pOperator->fpSet.getNextFn = get2ColsDummyBlock; } SDummyInputInfo *pInfo = (SDummyInputInfo*) taosMemoryCalloc(1, sizeof(SDummyInputInfo)); From 48e379f32048488cc3c46564c610e4467ebe2ac0 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 27 Apr 2022 09:48:33 +0800 Subject: [PATCH 04/12] fix(shell): memory init error. --- contrib/CMakeLists.txt | 26 +++++++++++++------------- source/dnode/mgmt/exe/dmMain.c | 4 ++-- source/libs/function/src/tudf.c | 2 +- source/util/src/tconfig.c | 12 ++++++------ 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 19923a5ad6..6a290dda15 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -110,7 +110,7 @@ execute_process(COMMAND "${CMAKE_COMMAND}" --build . # ================================================================================================ # googletest if(${BUILD_TEST}) - add_subdirectory(googletest) + add_subdirectory(googletest EXCLUDE_FROM_ALL) target_include_directories( gtest PUBLIC $ @@ -143,7 +143,7 @@ set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") option(ENABLE_CJSON_TEST "Enable building cJSON test" OFF) option(CJSON_OVERRIDE_BUILD_SHARED_LIBS "Override BUILD_SHARED_LIBS with CJSON_BUILD_SHARED_LIBS" ON) option(CJSON_BUILD_SHARED_LIBS "Overrides BUILD_SHARED_LIBS if CJSON_OVERRIDE_BUILD_SHARED_LIBS is enabled" OFF) -add_subdirectory(cJson) +add_subdirectory(cJson EXCLUDE_FROM_ALL) target_include_directories( cjson # see https://stackoverflow.com/questions/25676277/cmake-target-include-directories-prints-an-error-when-i-try-to-add-the-source @@ -152,7 +152,7 @@ target_include_directories( unset(CMAKE_PROJECT_INCLUDE_BEFORE) # lz4 -add_subdirectory(lz4/build/cmake) +add_subdirectory(lz4/build/cmake EXCLUDE_FROM_ALL) target_include_directories( lz4_static PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/lz4/lib @@ -160,7 +160,7 @@ target_include_directories( # zlib set(CMAKE_PROJECT_INCLUDE_BEFORE "${TD_SUPPORT_DIR}/EnableCMP0048.txt.in") -add_subdirectory(zlib) +add_subdirectory(zlib EXCLUDE_FROM_ALL) target_include_directories( zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/zlib @@ -176,7 +176,7 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # leveldb if(${BUILD_WITH_LEVELDB}) option(LEVELDB_BUILD_TESTS "" OFF) - add_subdirectory(leveldb) + add_subdirectory(leveldb EXCLUDE_FROM_ALL) target_include_directories( leveldb PUBLIC $ @@ -192,7 +192,7 @@ if(${BUILD_WITH_ROCKSDB}) option(WITH_TOOLS "" OFF) option(WITH_LIBURING "" OFF) option(ROCKSDB_BUILD_SHARED "Build shared versions of the RocksDB libraries" OFF) - add_subdirectory(rocksdb) + add_subdirectory(rocksdb EXCLUDE_FROM_ALL) target_include_directories( rocksdb PUBLIC $ @@ -203,7 +203,7 @@ endif(${BUILD_WITH_ROCKSDB}) # To support build on ubuntu: sudo apt-get install libboost-all-dev if(${BUILD_WITH_LUCENE}) option(ENABLE_TEST "Enable the tests" OFF) - add_subdirectory(lucene) + add_subdirectory(lucene EXCLUDE_FROM_ALL) target_include_directories( lucene++ PUBLIC $ @@ -213,14 +213,14 @@ endif(${BUILD_WITH_LUCENE}) # NuRaft if(${BUILD_WITH_NURAFT}) - add_subdirectory(nuraft) + add_subdirectory(nuraft EXCLUDE_FROM_ALL) endif(${BUILD_WITH_NURAFT}) # pthread if(${BUILD_PTHREAD}) set(CMAKE_BUILD_TYPE release) add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) + add_subdirectory(pthread EXCLUDE_FROM_ALL) set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) add_library(pthread STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) @@ -228,12 +228,12 @@ endif() # iconv if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv) + add_subdirectory(iconv EXCLUDE_FROM_ALL) endif(${BUILD_WITH_ICONV}) # wingetopt if(${BUILD_WINGETOPT}) - add_subdirectory(wingetopt) + add_subdirectory(wingetopt EXCLUDE_FROM_ALL) endif(${BUILD_WINGETOPT}) # msvcregex @@ -293,7 +293,7 @@ if(${BUILD_WITH_UV}) MESSAGE("Windows need set no-sign-compare") add_compile_options(-Wno-sign-compare) endif () - add_subdirectory(libuv) + add_subdirectory(libuv EXCLUDE_FROM_ALL) endif(${BUILD_WITH_UV}) # BDB @@ -334,5 +334,5 @@ endif(${BUILD_WITH_SQLITE}) # Build test # ================================================================================================ if(${BUILD_DEPENDENCY_TESTS}) - add_subdirectory(test) + add_subdirectory(test EXCLUDE_FROM_ALL) endif(${BUILD_DEPENDENCY_TESTS}) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 6d9ead4c87..4a2d02d25d 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -69,8 +69,8 @@ static void dmSetSignalHandle() { static int32_t dmParseArgs(int32_t argc, char const *argv[]) { int32_t cmdEnvIndex = 0; if (argc < 2) return 0; - global.envCmd = taosMemoryMalloc(argc-1); - memset(global.envCmd, 0, argc-1); + global.envCmd = taosMemoryMalloc((argc-1)*sizeof(char*)); + memset(global.envCmd, 0, (argc-1)*sizeof(char*)); for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { if (i < argc - 1) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f8a7e77814..f4a724afa3 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -213,7 +213,7 @@ enum { int32_t getUdfdPipeName(char* pipeName, int32_t size) { char dnodeId[8] = {0}; - size_t dnodeIdSize; + size_t dnodeIdSize = sizeof(dnodeId); int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize); if (err != 0) { dnodeId[0] = '1'; diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index b864f48522..cb9ef87701 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -823,7 +823,7 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } p++; - if (memcmp(url, "jsonFile", 8) == 0) { + if (strncmp(url, "jsonFile", 8) == 0) { char *filepath = p; if (!taosCheckExistFile(filepath)) { uError("fial to load json file: %s", filepath); @@ -893,8 +893,8 @@ int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { } tjsonDelete(pJson); - // } else if (memcmp(url, "jsonUrl", 7) == 0) { - // } else if (memcmp(url, "etcdUrl", 7) == 0) { + // } else if (strncmp(url, "jsonUrl", 7) == 0) { + // } else if (strncmp(url, "etcdUrl", 7) == 0) { } else { uError("Unsupported url: %s", url); return -1; @@ -908,7 +908,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl int32_t index = 0; if (envCmd == NULL) return 0; while (envCmd[index]!=NULL) { - if (memcmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(envCmd[index], "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(envCmd[index], '='); if (p != NULL) { p++; @@ -934,7 +934,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl break; } if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - if (memcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { p++; @@ -975,7 +975,7 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl break; } if(line[_bytes - 1] == '\n') line[_bytes - 1] = 0; - if (memcmp(line, "TAOS_APOLLO_URL", 14) == 0) { + if (strncmp(line, "TAOS_APOLLO_URL", 14) == 0) { char *p = strchr(line, '='); if (p != NULL) { p++; From d4b87d903978318e15d610ccc5bcc03b83884a3a Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Wed, 27 Apr 2022 09:54:50 +0800 Subject: [PATCH 05/12] fix(shell): memory init error. --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 72ff6179db..494b302165 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -87,6 +87,6 @@ # ./test.sh -f tsim/sma/tsmaCreateInsertData.sim # --- valgrind -#./test.sh -f tsim/valgrind/checkError.sim -v +./test.sh -f tsim/valgrind/checkError.sim -v #======================b1-end=============== From c6cc7eff8609c07d7cd79009610ecc36ce2aee97 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 27 Apr 2022 10:02:22 +0800 Subject: [PATCH 06/12] test: add client config for test cases --- tests/pytest/util/dnodes.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index 2c50a62117..8a29a0a7a6 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -96,11 +96,13 @@ class TDSimClient: for key, value in self.cfgDict.items(): self.cfg(key, value) - if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: - print(updatecfgDict[0][0]) - clientCfg = dict (updatecfgDict[0][0].get('clientCfg')) - for key, value in clientCfg.items(): - self.cfg(key, value) + try: + if bool(updatecfgDict) and updatecfgDict[0] and updatecfgDict[0][0]: + clientCfg = dict (updatecfgDict[0][0].get('clientCfg')) + for key, value in clientCfg.items(): + self.cfg(key, value) + except Exception: + pass tdLog.debug("psim is deployed and configured by %s" % (self.cfgPath)) From e8acef45a4e2266a0c770b7892bcfaaeab67d03e Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 27 Apr 2022 10:03:49 +0800 Subject: [PATCH 07/12] [test: add test case for taos shell] --- tests/system-test/0-others/taosShell.py | 233 ++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 tests/system-test/0-others/taosShell.py diff --git a/tests/system-test/0-others/taosShell.py b/tests/system-test/0-others/taosShell.py new file mode 100644 index 0000000000..a946437fed --- /dev/null +++ b/tests/system-test/0-others/taosShell.py @@ -0,0 +1,233 @@ + +import taos +import sys +import time +import socket +import pexpect +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +def taos_command (key, value, expectString, cfgDir, dbName, key1='', value1=''): + if len(key) == 0: + tdLog.exit("taos test key is null!") + + if len(cfgDir) != 0: + taosCmd = 'taos -c ' + cfgDir + ' -' + key + + if len(value) != 0: + if key == 'p': + taosCmd = taosCmd + value + else: + taosCmd = taosCmd + ' ' + value + + if len(key1) != 0: + taosCmd = taosCmd + ' -' + key1 + if key1 == 'p': + taosCmd = taosCmd + value1 + else: + if len(value1) != 0: + taosCmd = taosCmd + ' ' + value1 + + tdLog.info ("taos cmd: %s" % taosCmd) + + child = pexpect.spawn(taosCmd, timeout=3) + #output = child.readline() + #print (output.decode()) + i = child.expect([expectString, pexpect.TIMEOUT, pexpect.EOF], timeout=1) + retResult = child.before.decode() + print(retResult) + #print(child.after.decode()) + if i == 0: + print ('taos login success! Here can run sql, taos> ') + if len(dbName) != 0: + child.sendline ('create database %s;'%(dbName)) + w = child.expect(["Query OK", pexpect.TIMEOUT, pexpect.EOF], timeout=1) + if w == 0: + return "TAOS_OK" + else: + return "TAOS_FAIL" + else: + return "TAOS_OK" + else: + if key == 'A' or key1 == 'A': + return "TAOS_OK", retResult + else: + return "TAOS_FAIL" + +class TDTestCase: + + #updatecfgDict = {'serverPort': 7080, 'firstEp': 'localhost:7080'} + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring + tdSql.prepare() + # time.sleep(2) + tdSql.query("create user testpy pass 'testpy'") + + hostname = socket.gethostname() + tdLog.info ("hostname: %s" % hostname) + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + checkNetworkStatus = ['0: unavailable', '1: network ok', '2: service ok', '3: service degraded', '4: exiting'] + netrole = ['client', 'server'] + + keyDict = {'h':'', 'P':'6030', 'p':'testpy', 'u':'testpy', 'a':'', 'A':'', 'c':'', 'C':'', 's':'', 'r':'', 'f':'', \ + 'k':'', 't':'', 'n':'', 'l':'1024', 'N':'100', 'V':'', 'd':'db', 'w':'30', '-help':'', '-usage':'', '?':''} + + keyDict['h'] = hostname + keyDict['c'] = cfgPath + + tdLog.printNoPrefix("================================ parameter: -h") + newDbName="dbh" + retCode = taos_command("h", keyDict['h'], "taos>", keyDict['c'], newDbName) + if retCode != "TAOS_OK": + tdLog.exit("taos -h %s fail"%keyDict['h']) + else: + #dataDbName = ["information_schema", "performance_schema", "db", newDbName] + tdSql.query("show databases") + #tdSql.getResult("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -h %s fail"%keyDict['h']) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -P") + #tdDnodes.stop(1) + #sleep(3) + #tdDnodes.start(1) + #sleep(3) + #keyDict['P'] = 6030 + newDbName = "dbpp" + retCode = taos_command("P", keyDict['P'], "taos>", keyDict['c'], newDbName) + if retCode != "TAOS_OK": + tdLog.exit("taos -P %s fail"%keyDict['P']) + else: + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -P %s fail"%keyDict['P']) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -u") + newDbName="dbu" + retCode = taos_command("u", keyDict['u'], "taos>", keyDict['c'], newDbName, "p", keyDict['p']) + if retCode != "TAOS_OK": + tdLog.exit("taos -u %s -p%s fail"%(keyDict['u'], keyDict['p'])) + else: + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -u %s -p%s fail"%(keyDict['u'], keyDict['p'])) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -A") + newDbName="dbaa" + retCode, retVal = taos_command("p", keyDict['p'], "taos>", keyDict['c'], '', "A", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -A fail") + + retCode = taos_command("u", keyDict['u'], "taos>", keyDict['c'], newDbName, 'a', retVal) + if retCode != "TAOS_OK": + tdLog.exit("taos -u %s -a %s"%(keyDict['u'], retVal)) + + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -u %s -a %s fail"%(keyDict['u'], retVal)) + + tdSql.query('drop database %s'%newDbName) + + tdLog.printNoPrefix("================================ parameter: -s") + newDbName="dbss" + keyDict['s'] = "\"create database " + newDbName + "\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + print ("========== check new db ==========") + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.getData(i, 0) == newDbName: + break + else: + tdLog.exit("create db fail after taos -s %s fail"%(keyDict['s'])) + + keyDict['s'] = "\"create table " + newDbName + ".stb (ts timestamp, c int) tags (t int)\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s create table fail") + + keyDict['s'] = "\"create table " + newDbName + ".ctb0 using " + newDbName + ".stb tags (0) " + newDbName + ".ctb1 using " + newDbName + ".stb tags (1)\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s create table fail") + + keyDict['s'] = "\"insert into " + newDbName + ".ctb0 values('2021-04-01 08:00:00.000', 10)('2021-04-01 08:00:01.000', 20) " + newDbName + ".ctb1 values('2021-04-01 08:00:00.000', 11)('2021-04-01 08:00:01.000', 21)\"" + retCode = taos_command("s", keyDict['s'], "Query OK", keyDict['c'], '', '', '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s insert data fail") + + sqlString = "select * from " + newDbName + ".ctb0" + tdSql.query(sqlString) + tdSql.checkData(0, 0, '2021-04-01 08:00:00.000') + tdSql.checkData(0, 1, 10) + tdSql.checkData(1, 0, '2021-04-01 08:00:01.000') + tdSql.checkData(1, 1, 20) + sqlString = "select * from " + newDbName + ".ctb1" + tdSql.query(sqlString) + tdSql.checkData(0, 0, '2021-04-01 08:00:00.000') + tdSql.checkData(0, 1, 11) + tdSql.checkData(1, 0, '2021-04-01 08:00:01.000') + tdSql.checkData(1, 1, 21) + + #tdSql.query('drop database %s'%newDbName) + + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 02cee2ba62a0fe9f4e35f12f12f5864b106649a4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 27 Apr 2022 10:11:32 +0800 Subject: [PATCH 08/12] refactor: do some internal refactor. --- source/common/src/tdatablock.c | 15 ++++---- source/libs/executor/src/executorimpl.c | 50 +++++++++++++++++-------- 2 files changed, 43 insertions(+), 22 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d6a6ba5b13..63e009ed0a 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -117,22 +117,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { int32_t dataLen = varDataTLen(pData); - if(type == TSDB_DATA_TYPE_JSON) { - if(*pData == TSDB_DATA_TYPE_NULL) { + if (type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { dataLen = 0; - }else if(*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData+CHAR_BYTES); - }else if(*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { dataLen = LONG_BYTES; - }else if(*pData == TSDB_DATA_TYPE_BOOL) { + } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; } dataLen += CHAR_BYTES; } + SVarColAttr* pAttr = &pColumnInfoData->varmeta; if (pAttr->allocLen < pAttr->length + dataLen) { uint32_t newSize = pAttr->allocLen; - if (newSize == 0) { + if (newSize <= 1) { newSize = 8; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 318d87e100..44c46785cc 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1034,7 +1034,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlo } } -static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t type, +static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunctParam* pFuncParam, int32_t paramIndex, int32_t numOfRows) { SColumnInfoData* pColInfo = NULL; if (pInput->pData[paramIndex] == NULL) { @@ -1044,17 +1044,17 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc } // Set the correct column info (data type and bytes) - pColInfo->info.type = type; - pColInfo->info.bytes = tDataTypes[type].bytes; + pColInfo->info.type = pFuncParam->param.nType; + pColInfo->info.bytes = pFuncParam->param.nLen; pInput->pData[paramIndex] = pColInfo; } else { pColInfo = pInput->pData[paramIndex]; } - ASSERT(!IS_VAR_DATA_TYPE(type)); colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); + int8_t type = pFuncParam->param.nType; if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { int64_t v = pFuncParam->param.i; for (int32_t i = 0; i < numOfRows; ++i) { @@ -1065,6 +1065,12 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc for (int32_t i = 0; i < numOfRows; ++i) { colDataAppendDouble(pColInfo, i, &v); } + } else if (type == TSDB_DATA_TYPE_VARCHAR) { + char *tmp = taosMemoryMalloc(pFuncParam->param.nLen + VARSTR_HEADER_SIZE); + STR_WITH_SIZE_TO_VARSTR(tmp, pFuncParam->param.pz, pFuncParam->param.nLen); + for(int32_t i = 0; i < numOfRows; ++i) { + colDataAppend(pColInfo, i, tmp, false); + } } return TSDB_CODE_SUCCESS; @@ -1104,7 +1110,7 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt pInput->numOfRows = pBlock->info.rows; pInput->startRowIndex = 0; - code = doCreateConstantValColumnInfo(pInput, pFuncParam, pFuncParam->param.nType, j, pBlock->info.rows); + code = doCreateConstantValColumnInfo(pInput, pFuncParam, j, pBlock->info.rows); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -6684,18 +6690,32 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - // todo extract method - SColumn c = {0}; - c.slotId = pColNode->slotId; - c.colId = pColNode->colId; - c.type = pColNode->node.resType.type; - c.bytes = pColNode->node.resType.bytes; - c.precision = pColNode->node.resType.precision; - c.scale = pColNode->node.resType.scale; + if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - taosArrayPush(pList, &c); + // todo extract method + SColumn c = {0}; + c.slotId = pColNode->slotId; + c.colId = pColNode->colId; + c.type = pColNode->node.resType.type; + c.bytes = pColNode->node.resType.bytes; + c.scale = pColNode->node.resType.scale; + c.precision = pColNode->node.resType.precision; + + taosArrayPush(pList, &c); + } else if (nodeType(pNode->pExpr) == QUERY_NODE_VALUE) { + SValueNode* pValNode = (SValueNode*) pNode->pExpr; + SColumn c = {0}; + c.slotId = pNode->slotId; + c.colId = pNode->slotId; + c.type = pValNode->node.type; + c.bytes = pValNode->node.resType.bytes; + c.scale = pValNode->node.resType.scale; + c.precision = pValNode->node.resType.precision; + + taosArrayPush(pList, &c); + } } return pList; From 0136cae76079862fe08aa313693dd5fb51ebea66 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Apr 2022 10:35:21 +0800 Subject: [PATCH 09/12] refactor(cluster): add cb func on transaction start and stop --- include/dnode/mnode/sdb/sdb.h | 29 ++++- source/dnode/mnode/impl/inc/mndDef.h | 8 +- source/dnode/mnode/impl/inc/mndTrans.h | 11 +- source/dnode/mnode/impl/src/mndTrans.c | 79 +++++++++++--- source/dnode/mnode/impl/src/mndUser.c | 145 ++++++++++++------------- source/dnode/mnode/sdb/inc/sdbInt.h | 24 ---- source/dnode/mnode/sdb/src/sdb.c | 2 +- source/dnode/mnode/sdb/src/sdbHash.c | 16 +-- source/dnode/mnode/sdb/src/sdbRaw.c | 4 +- source/dnode/mnode/sdb/src/sdbRow.c | 4 +- 10 files changed, 192 insertions(+), 130 deletions(-) diff --git a/include/dnode/mnode/sdb/sdb.h b/include/dnode/mnode/sdb/sdb.h index d534b0405f..1613339d10 100644 --- a/include/dnode/mnode/sdb/sdb.h +++ b/include/dnode/mnode/sdb/sdb.h @@ -18,6 +18,11 @@ #include "os.h" +#include "thash.h" +#include "tlockfree.h" +#include "tlog.h" +#include "tmsg.h" + #ifdef __cplusplus extern "C" { #endif @@ -135,7 +140,7 @@ typedef enum { typedef struct SSdb SSdb; typedef int32_t (*SdbInsertFp)(SSdb *pSdb, void *pObj); typedef int32_t (*SdbUpdateFp)(SSdb *pSdb, void *pSrcObj, void *pDstObj); -typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj); +typedef int32_t (*SdbDeleteFp)(SSdb *pSdb, void *pObj, bool callFunc); typedef int32_t (*SdbDeployFp)(SMnode *pMnode); typedef SSdbRow *(*SdbDecodeFp)(SSdbRaw *pRaw); typedef SSdbRaw *(*SdbEncodeFp)(void *pObj); @@ -326,9 +331,29 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver); int32_t sdbGetRawTotalSize(SSdbRaw *pRaw); SSdbRow *sdbAllocRow(int32_t objSize); -void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow); +void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc); void *sdbGetRowObj(SSdbRow *pRow); +typedef struct SSdb { + SMnode *pMnode; + char *currDir; + char *syncDir; + char *tmpDir; + int64_t lastCommitVer; + int64_t curVer; + int64_t tableVer[SDB_MAX]; + int64_t maxId[SDB_MAX]; + EKeyType keyTypes[SDB_MAX]; + SHashObj *hashObjs[SDB_MAX]; + SRWLatch locks[SDB_MAX]; + SdbInsertFp insertFps[SDB_MAX]; + SdbUpdateFp updateFps[SDB_MAX]; + SdbDeleteFp deleteFps[SDB_MAX]; + SdbDeployFp deployFps[SDB_MAX]; + SdbEncodeFp encodeFps[SDB_MAX]; + SdbDecodeFp decodeFps[SDB_MAX]; +} SSdb; + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 36a8173adf..b1a33e54aa 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -126,8 +126,6 @@ typedef enum { DND_REASON_OTHERS } EDndReason; -typedef void (*TransCbFp)(SMnode* pMnode, void* param); - typedef struct { int32_t id; ETrnStage stage; @@ -150,8 +148,10 @@ typedef struct { int64_t dbUid; char dbname[TSDB_DB_FNAME_LEN]; char lastError[TSDB_TRANS_ERROR_LEN]; - TransCbFp transCbFp; - void* transCbParam; + int32_t startFunc; + int32_t stopFunc; + int32_t paramLen; + void* param; } STrans; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndTrans.h b/source/dnode/mnode/impl/inc/mndTrans.h index 2fcc82e861..5b5aff7c86 100644 --- a/source/dnode/mnode/impl/inc/mndTrans.h +++ b/source/dnode/mnode/impl/inc/mndTrans.h @@ -33,6 +33,15 @@ typedef struct { void *pCont; } STransAction; +typedef enum { + TEST_TRANS_START_FUNC = 1, + TEST_TRANS_STOP_FUNC = 2, + CONSUME_TRANS_START_FUNC = 3, + CONSUME_TRANS_STOP_FUNC = 4, +} ETrnFuncType; + +typedef void (*TransCbFp)(SMnode *pMnode, void *param, int32_t paramLen); + int32_t mndInitTrans(SMnode *pMnode); void mndCleanupTrans(SMnode *pMnode); @@ -44,7 +53,7 @@ int32_t mndTransAppendCommitlog(STrans *pTrans, SSdbRaw *pRaw); int32_t mndTransAppendRedoAction(STrans *pTrans, STransAction *pAction); int32_t mndTransAppendUndoAction(STrans *pTrans, STransAction *pAction); void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen); -void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param); +void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen); void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb); int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 582b60f7b2..c08b0f6db9 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -29,7 +29,7 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans); static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw); static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans); static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *OldTrans, STrans *pOld); -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans); +static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc); static int32_t mndTransAppendLog(SArray *pArray, SSdbRaw *pRaw); static int32_t mndTransAppendAction(SArray *pArray, STransAction *pAction); @@ -174,6 +174,13 @@ static SSdbRaw *mndTransActionEncode(STrans *pTrans) { SDB_SET_BINARY(pRaw, dataPos, (void *)pAction->pCont, pAction->contLen, TRANS_ENCODE_OVER) } + SDB_SET_INT32(pRaw, dataPos, pTrans->startFunc, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->stopFunc, TRANS_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pTrans->paramLen, TRANS_ENCODE_OVER) + if (pTrans->param != NULL) { + SDB_SET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_ENCODE_OVER) + } + SDB_SET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, TRANS_ENCODE_OVER) @@ -305,6 +312,14 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { action.pCont = NULL; } + SDB_GET_INT32(pRaw, dataPos, &pTrans->startFunc, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->stopFunc, TRANS_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pTrans->paramLen, TRANS_DECODE_OVER) + if (pTrans->paramLen != 0) { + pTrans->param = taosMemoryMalloc(pTrans->paramLen); + SDB_GET_BINARY(pRaw, dataPos, pTrans->param, pTrans->paramLen, TRANS_DECODE_OVER); + } + SDB_GET_RESERVE(pRaw, dataPos, MND_TRANS_RESERVE_SIZE, TRANS_DECODE_OVER) terrno = 0; @@ -413,9 +428,36 @@ static const char *mndTransType(ETrnType type) { } } +static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) { + mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen); +} + +static void mndTransTestStopFunc(SMnode *pMnode, void *param, int32_t paramLen) { + mInfo("test trans stop, param:%s, len:%d", (char *)param, paramLen); +} + +static TransCbFp mndTransGetCbFp(ETrnFuncType ftype) { + switch (ftype) { + case TEST_TRANS_START_FUNC: + return mndTransTestStartFunc; + case TEST_TRANS_STOP_FUNC: + return mndTransTestStopFunc; + default: + return NULL; + } +} + static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) { // pTrans->stage = TRN_STAGE_PREPARE; mTrace("trans:%d, perform insert action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); + + if (pTrans->startFunc > 0) { + TransCbFp fp = mndTransGetCbFp(pTrans->startFunc); + if (fp) { + (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); + } + } + return 0; } @@ -430,10 +472,23 @@ static void mndTransDropData(STrans *pTrans) { pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; } + if (pTrans->param != NULL) { + taosMemoryFree(pTrans->param); + pTrans->param = NULL; + pTrans->paramLen = 0; + } } -static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) { - mTrace("trans:%d, perform delete action, row:%p stage:%s", pTrans->id, pTrans, mndTransStr(pTrans->stage)); +static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans, bool callFunc) { + mDebug("trans:%d, perform delete action, row:%p stage:%s callfunc:%d", pTrans->id, pTrans, mndTransStr(pTrans->stage), + callFunc); + if (pTrans->stopFunc > 0 && callFunc) { + TransCbFp fp = mndTransGetCbFp(pTrans->stopFunc); + if (fp) { + (*fp)(pSdb->pMnode, pTrans->param, pTrans->paramLen); + } + } + mndTransDropData(pTrans); return 0; } @@ -498,7 +553,7 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, ETrnType type, const S return NULL; } - mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local var is created, data:%p", pTrans->id, pTrans); return pTrans; } @@ -525,7 +580,7 @@ static void mndTransDropActions(SArray *pArray) { void mndTransDrop(STrans *pTrans) { if (pTrans != NULL) { mndTransDropData(pTrans); - mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans); + mDebug("trans:%d, local var is freed, data:%p", pTrans->id, pTrans); taosMemoryFreeClear(pTrans); } } @@ -574,9 +629,11 @@ void mndTransSetRpcRsp(STrans *pTrans, void *pCont, int32_t contLen) { pTrans->rpcRspLen = contLen; } -void mndTransSetCb(STrans *pTrans, TransCbFp fp, void *param) { - pTrans->transCbFp = fp; - pTrans->transCbParam = param; +void mndTransSetCb(STrans *pTrans, ETrnFuncType startFunc, ETrnFuncType stopFunc, void *param, int32_t paramLen) { + pTrans->startFunc = startFunc; + pTrans->stopFunc = stopFunc; + pTrans->param = param; + pTrans->paramLen = paramLen; } void mndTransSetDbInfo(STrans *pTrans, SDbObj *pDb) { @@ -712,8 +769,6 @@ int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) { pNew->rpcRefId = pTrans->rpcRefId; pNew->rpcRsp = pTrans->rpcRsp; pNew->rpcRspLen = pTrans->rpcRspLen; - pNew->transCbFp = pTrans->transCbFp; - pNew->transCbParam = pTrans->transCbParam; pTrans->rpcRsp = NULL; pTrans->rpcRspLen = 0; @@ -1125,10 +1180,6 @@ static bool mndTransPerfromFinishedStage(SMnode *pMnode, STrans *pTrans) { mDebug("trans:%d, finished, code:0x%04x, failedTimes:%d", pTrans->id, pTrans->code, pTrans->failedTimes); - if (pTrans->transCbFp != NULL) { - (*pTrans->transCbFp)(pMnode, pTrans->transCbParam); - } - return continueExec; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index d6876782f6..1500b3d7d8 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -96,36 +96,36 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN; SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); - if (pRaw == NULL) goto USER_ENCODE_OVER; + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, USER_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, USER_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pUser->superUser, USER_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, USER_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, USER_ENCODE_OVER) + SDB_SET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pUser->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pUser->updateTime, _OVER) + SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) char *db = taosHashIterate(pUser->readDbs, NULL); while (db != NULL) { - SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER); db = taosHashIterate(pUser->readDbs, db); } db = taosHashIterate(pUser->writeDbs, NULL); while (db != NULL) { - SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_ENCODE_OVER); + SDB_SET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER); db = taosHashIterate(pUser->writeDbs, db); } - SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, USER_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -USER_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("user:%s, failed to encode to raw:%p since %s", pUser->user, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -140,55 +140,54 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto USER_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; if (sver != USER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto USER_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SUserObj)); - if (pRow == NULL) goto USER_DECODE_OVER; + if (pRow == NULL) goto _OVER; SUserObj *pUser = sdbGetRowObj(pRow); - if (pUser == NULL) goto USER_DECODE_OVER; - - pUser->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); - pUser->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); - if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto USER_DECODE_OVER; + if (pUser == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, USER_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, USER_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, USER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, USER_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, USER_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, USER_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, pUser->user, TSDB_USER_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pUser->pass, TSDB_PASSWORD_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pUser->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pUser->updateTime, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, USER_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, USER_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfReadDbs, _OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfWriteDbs, _OVER) + pUser->readDbs = taosHashInit(numOfReadDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); + pUser->writeDbs = taosHashInit(numOfWriteDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, true); + if (pUser->readDbs == NULL || pUser->writeDbs == NULL) goto _OVER; for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER) int32_t len = strlen(db) + 1; taosHashPut(pUser->readDbs, db, len, db, TSDB_DB_FNAME_LEN); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, USER_DECODE_OVER) + SDB_GET_BINARY(pRaw, dataPos, db, TSDB_DB_FNAME_LEN, _OVER) int32_t len = strlen(db) + 1; taosHashPut(pUser->writeDbs, db, len, db, TSDB_DB_FNAME_LEN); } - SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, USER_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) terrno = 0; -USER_DECODE_OVER: +_OVER: if (terrno != 0) { mError("user:%s, failed to decode from raw:%p since %s", pUser->user, pRaw, terrstr()); taosHashCleanup(pUser->readDbs); @@ -220,6 +219,8 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { mTrace("user:%s, perform delete action, row:%p", pUser->user, pUser); taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); + pUser->readDbs = NULL; + pUser->writeDbs = NULL; return 0; } @@ -228,13 +229,8 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); pOld->updateTime = pNew->updateTime; - void *tmp1 = pOld->readDbs; - pOld->readDbs = pNew->readDbs; - pNew->readDbs = tmp1; - - void *tmp2 = pOld->writeDbs; - pOld->writeDbs = pNew->writeDbs; - pNew->writeDbs = tmp2; + TSWAP(pOld->readDbs, pNew->readDbs, (void *)); + TSWAP(pOld->writeDbs, pNew->writeDbs, (void *)); return 0; } @@ -277,6 +273,9 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate } sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + char *param = strdup("====> test code to be deleted later <====="); + mndTransSetCb(pTrans, TEST_TRANS_START_FUNC, TEST_TRANS_STOP_FUNC, param, strlen(param) + 1); + if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); @@ -296,41 +295,41 @@ static int32_t mndProcessCreateUserReq(SNodeMsg *pReq) { if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto CREATE_USER_OVER; + goto _OVER; } mDebug("user:%s, start to create", createReq.user); if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto CREATE_USER_OVER; + goto _OVER; } if (createReq.pass[0] == 0) { terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; - goto CREATE_USER_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, createReq.user); if (pUser != NULL) { terrno = TSDB_CODE_MND_USER_ALREADY_EXIST; - goto CREATE_USER_OVER; + goto _OVER; } pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto CREATE_USER_OVER; + goto _OVER; } if (mndCheckCreateUserAuth(pOperUser) != 0) { - goto CREATE_USER_OVER; + goto _OVER; } code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -CREATE_USER_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to create since %s", createReq.user, terrstr()); } @@ -399,38 +398,38 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_USER_OVER; + goto _OVER; } mDebug("user:%s, start to alter", alterReq.user); if (alterReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto ALTER_USER_OVER; + goto _OVER; } if (alterReq.pass[0] == 0) { terrno = TSDB_CODE_MND_INVALID_PASS_FORMAT; - goto ALTER_USER_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, alterReq.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto ALTER_USER_OVER; + goto _OVER; } memcpy(&newUser, pUser, sizeof(SUserObj)); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { - goto ALTER_USER_OVER; + goto _OVER; } int32_t len = strlen(alterReq.dbname) + 1; @@ -446,50 +445,50 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } if (taosHashPut(newUser.readDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { taosHashClear(newUser.readDbs); } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } if (taosHashPut(newUser.writeDbs, alterReq.dbname, len, alterReq.dbname, TSDB_DB_FNAME_LEN) != 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_USER_OVER; + goto _OVER; } } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { taosHashClear(newUser.writeDbs); } else { terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; - goto ALTER_USER_OVER; + goto _OVER; } newUser.updateTime = taosGetTimestampMs(); if (mndCheckAlterUserAuth(pOperUser, pUser, pDb, &alterReq) != 0) { - goto ALTER_USER_OVER; + goto _OVER; } code = mndUpdateUser(pMnode, pUser, &newUser, pReq); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_USER_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to alter since %s", alterReq.user, terrstr()); } @@ -537,36 +536,36 @@ static int32_t mndProcessDropUserReq(SNodeMsg *pReq) { if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto DROP_USER_OVER; + goto _OVER; } mDebug("user:%s, start to drop", dropReq.user); if (dropReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; - goto DROP_USER_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, dropReq.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto DROP_USER_OVER; + goto _OVER; } pOperUser = mndAcquireUser(pMnode, pReq->user); if (pOperUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; - goto DROP_USER_OVER; + goto _OVER; } if (mndCheckDropUserAuth(pOperUser) != 0) { - goto DROP_USER_OVER; + goto _OVER; } code = mndDropUser(pMnode, pReq, pUser); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -DROP_USER_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to drop since %s", dropReq.user, terrstr()); } @@ -586,7 +585,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto GET_AUTH_OVER; + goto _OVER; } mTrace("user:%s, start to get auth", authReq.user); @@ -594,7 +593,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { pUser = mndAcquireUser(pMnode, authReq.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_USER_NOT_EXIST; - goto GET_AUTH_OVER; + goto _OVER; } memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); @@ -622,7 +621,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto GET_AUTH_OVER; + goto _OVER; } tSerializeSGetUserAuthRsp(pRsp, contLen, &authRsp); @@ -631,7 +630,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { pReq->rspLen = contLen; code = 0; -GET_AUTH_OVER: +_OVER: mndReleaseUser(pMnode, pUser); taosHashCleanup(authRsp.readDbs); taosHashCleanup(authRsp.writeDbs); diff --git a/source/dnode/mnode/sdb/inc/sdbInt.h b/source/dnode/mnode/sdb/inc/sdbInt.h index d268b93051..de71ac47b6 100644 --- a/source/dnode/mnode/sdb/inc/sdbInt.h +++ b/source/dnode/mnode/sdb/inc/sdbInt.h @@ -19,10 +19,6 @@ #include "os.h" #include "sdb.h" -#include "thash.h" -#include "tlockfree.h" -#include "tlog.h" -#include "tmsg.h" #ifdef __cplusplus extern "C" { @@ -53,26 +49,6 @@ typedef struct SSdbRow { char pObj[]; } SSdbRow; -typedef struct SSdb { - SMnode *pMnode; - char *currDir; - char *syncDir; - char *tmpDir; - int64_t lastCommitVer; - int64_t curVer; - int64_t tableVer[SDB_MAX]; - int64_t maxId[SDB_MAX]; - EKeyType keyTypes[SDB_MAX]; - SHashObj *hashObjs[SDB_MAX]; - SRWLatch locks[SDB_MAX]; - SdbInsertFp insertFps[SDB_MAX]; - SdbUpdateFp updateFps[SDB_MAX]; - SdbDeleteFp deleteFps[SDB_MAX]; - SdbDeployFp deployFps[SDB_MAX]; - SdbEncodeFp encodeFps[SDB_MAX]; - SdbDecodeFp decodeFps[SDB_MAX]; -} SSdb; - const char *sdbTableName(ESdbType type); void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper); diff --git a/source/dnode/mnode/sdb/src/sdb.c b/source/dnode/mnode/sdb/src/sdb.c index bc878d7b31..f85ff7a26e 100644 --- a/source/dnode/mnode/sdb/src/sdb.c +++ b/source/dnode/mnode/sdb/src/sdb.c @@ -87,7 +87,7 @@ void sdbCleanup(SSdb *pSdb) { SSdbRow *pRow = *ppRow; if (pRow == NULL) continue; - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, true); ppRow = taosHashIterate(hash, ppRow); } } diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 1158bedc21..b34c2a82d8 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -137,7 +137,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize); if (pOldRow != NULL) { taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; } @@ -148,7 +148,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) { taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE; return terrno; } @@ -164,7 +164,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosWLockLatch(pLock); taosHashRemove(hash, pRow->pObj, keySize); taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = code; return terrno; } @@ -202,7 +202,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj); } - sdbFreeRow(pSdb, pNewRow); + sdbFreeRow(pSdb, pNewRow, false); pSdb->tableVer[pOldRow->type]++; return code; @@ -215,7 +215,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize); if (ppOldRow == NULL || *ppOldRow == NULL) { taosWUnLockLatch(pLock); - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); terrno = TSDB_CODE_SDB_OBJ_NOT_THERE; return terrno; } @@ -228,7 +228,7 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow * taosWUnLockLatch(pLock); pSdb->tableVer[pOldRow->type]++; - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, false); sdbCheck(pSdb, pOldRow); // sdbRelease(pSdb, pOldRow->pObj); @@ -322,7 +322,7 @@ static void sdbCheck(SSdb *pSdb, SSdbRow *pRow) { int32_t ref = atomic_load_32(&pRow->refCount); sdbPrintOper(pSdb, pRow, "check"); if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, true); } taosRUnLockLatch(pLock); @@ -340,7 +340,7 @@ void sdbRelease(SSdb *pSdb, void *pObj) { int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1); sdbPrintOper(pSdb, pRow, "release"); if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) { - sdbFreeRow(pSdb, pRow); + sdbFreeRow(pSdb, pRow, true); } taosRUnLockLatch(pLock); diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 326fe53fc7..45464c3bd3 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -107,7 +107,9 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32_ return -1; } - memcpy(pRaw->pData + dataPos, pVal, valLen); + if (pVal != NULL) { + memcpy(pRaw->pData + dataPos, pVal, valLen); + } return 0; } diff --git a/source/dnode/mnode/sdb/src/sdbRow.c b/source/dnode/mnode/sdb/src/sdbRow.c index 94f87cb350..43f70cb245 100644 --- a/source/dnode/mnode/sdb/src/sdbRow.c +++ b/source/dnode/mnode/sdb/src/sdbRow.c @@ -36,11 +36,11 @@ void *sdbGetRowObj(SSdbRow *pRow) { return pRow->pObj; } -void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) { +void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc) { // remove attached object such as trans SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type]; if (deleteFp != NULL) { - (*deleteFp)(pSdb, pRow->pObj); + (*deleteFp)(pSdb, pRow->pObj, callFunc); } sdbPrintOper(pSdb, pRow, "free"); From debb35da495969a4a5f5d0bb7115e71512a04edc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Apr 2022 11:27:27 +0800 Subject: [PATCH 10/12] fix: make taos -c xx -C work --- tools/shell/src/shellMain.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index d277920877..6672cee367 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -42,6 +42,8 @@ int main(int argc, char *argv[]) { return 0; } + taos_init(); + if (shell.args.is_dump_config) { shellDumpConfig(); taos_cleanup(); From c3066c9412c02e3db3ea8148358f3f6ff5246a19 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 27 Apr 2022 11:51:07 +0800 Subject: [PATCH 11/12] [test: modify test case] --- tests/script/jenkins/basic.txt | 2 +- tests/script/tsim/db/alter_option.sim | 86 +++++++++++++-------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 494b302165..44147f477d 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -6,7 +6,7 @@ # ---- db ./test.sh -f tsim/db/create_all_options.sim -#./test.sh -f tsim/db/alter_option.sim +./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 3b61b907de..1f0c2043e8 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,7 +66,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 +sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -86,7 +86,7 @@ endi if $data3_db != 0 then # ntables return -1 endi -if $data4_db != 3 then # replica +if $data4_db != 1 then # replica return -1 endi if $data5_db != nostrict then # strict @@ -127,47 +127,47 @@ if $data16_db != ns then # precision endi sleep 3000 -sql show db.vgroups -if $data[0][4] == LEADER then - if $data[0][6] != FOLLOWER then - return -1 - endi - if $data[0][8] != FOLLOWER then - return -1 - endi -endi -if $data[0][6] == LEADER then - if $data[0][4] != FOLLOWER then - return -1 - endi - if $data[0][8] != FOLLOWER then - return -1 - endi -endi -if $data[0][8] == LEADER then - if $data[0][4] != FOLLOWER then - return -1 - endi - if $data[0][6] != FOLLOWER then - return -1 - endi -endi - -if $data[0][4] != LEADER then - if $data[0][4] != FOLLOWER then - return -1 - endi -endi -if $data[0][6] != LEADER then - if $data[0][6] != FOLLOWER then - return -1 - endi -endi -if $data[0][8] != LEADER then - if $data[0][8] != FOLLOWER then - return -1 - endi -endi +#sql show db.vgroups +#if $data[0][4] == LEADER then +# if $data[0][6] != FOLLOWER then +# return -1 +# endi +# if $data[0][8] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][6] == LEADER then +# if $data[0][4] != FOLLOWER then +# return -1 +# endi +# if $data[0][8] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][8] == LEADER then +# if $data[0][4] != FOLLOWER then +# return -1 +# endi +# if $data[0][6] != FOLLOWER then +# return -1 +# endi +#endi +# +#if $data[0][4] != LEADER then +# if $data[0][4] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][6] != LEADER then +# if $data[0][6] != FOLLOWER then +# return -1 +# endi +#endi +#if $data[0][8] != LEADER then +# if $data[0][8] != FOLLOWER then +# return -1 +# endi +#endi print ============== not support modify options: name, create_time, vgroups, ntables sql_error alter database db name dba From 01dd33ae5810ebcac413a64b9d7516d838e3c59c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 27 Apr 2022 13:41:28 +0800 Subject: [PATCH 12/12] fix: invalid func in mndVgroup.c --- source/dnode/mnode/impl/src/mndVgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 3a4fde992f..aab9ac61a2 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -45,8 +45,8 @@ int32_t mndInitVgroup(SMnode *pMnode) { .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, .insertFp = (SdbInsertFp)mndVgroupActionInsert, - .updateFp = (SdbUpdateFp)mndVgroupActionDelete, - .deleteFp = (SdbDeleteFp)mndVgroupActionUpdate}; + .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete}; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp);