From 898a9bd9e50081389b248a8eab8a63c2a83b9f5d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 28 May 2022 16:01:00 +0800 Subject: [PATCH 01/47] feature: tsma expired windows retrieval --- include/common/tmsg.h | 11 +++++++++++ include/common/tmsgdef.h | 1 + 2 files changed, 12 insertions(+) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index faf4addb4b..fd0ed1cf15 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2360,6 +2360,17 @@ static int32_t tDecodeTSmaWrapper(SDecoder* pDecoder, STSmaWrapper* pReq) { return 0; } +typedef struct { + int64_t tsmaIndexUid; + STimeWindow queryWindow; +} SVGetTsmaExpWndsReq; + +typedef struct { + int64_t tsmaIndexUid; + int32_t numExpWnds; + TSKEY* expWndsStartTs; +} SVGetTsmaExpWndsRsp; + typedef struct { int idx; } SMCreateFullTextReq; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 51a15c1489..90239bac3e 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -206,6 +206,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT_RSMA, "vnode-submit-rsma", SSubmitReq, SSubmitRsp) + TD_DEF_MSG_TYPE(TDMT_VND_GET_TSMA_EXP_WNDS, "vnode-get-tsma-expired-windows", SVGetTsmaExpWndsReq, SVGetTsmaExpWndsRsp) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_TIMEOUT, "vnode-sync-timeout", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_SYNC_PING, "vnode-sync-ping", NULL, NULL) From 1652cd0ec2a47955dffdfc0ee126a1a9b608eaef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 29 May 2022 12:35:11 +0800 Subject: [PATCH 02/47] enh(query): enable twa function in select clause. --- include/common/tmsg.h | 7 - include/libs/executor/executor.h | 12 - include/libs/function/function.h | 47 +-- include/util/tlist.h | 2 +- source/libs/executor/inc/executil.h | 1 + source/libs/executor/inc/executorimpl.h | 6 +- source/libs/executor/src/executil.c | 28 +- source/libs/executor/src/executorimpl.c | 104 +---- source/libs/executor/src/groupoperator.c | 12 +- source/libs/executor/src/scanoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 318 ++++++++++----- source/libs/function/inc/builtinsimpl.h | 4 + source/libs/function/inc/taggfunction.h | 7 - source/libs/function/src/builtins.c | 13 +- source/libs/function/src/builtinsimpl.c | 373 ++++++++++++++++-- source/libs/function/src/taggfunction.c | 3 +- source/util/src/tlist.c | 2 +- 17 files changed, 595 insertions(+), 348 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index faf4addb4b..c4abfffc61 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -571,13 +571,6 @@ int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pR int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); -typedef struct { - int16_t colId; // column id - int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag - int16_t flag; // denote if it is a tag or a normal column - char name[TSDB_DB_FNAME_LEN]; -} SColIndex; - typedef struct { int16_t lowerRelOptr; int16_t upperRelOptr; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 5379a8f712..288248422b 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -156,18 +156,6 @@ int64_t qGetQueriedTableUid(qTaskInfo_t tinfo); */ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList); -/** - * Create the table group according to the group by tags info - * @param pTableIdList - * @param skey - * @param groupInfo - * @param groupByIndex - * @param numOfIndex - * @return - */ -// int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* -// groupByIndex, int32_t numOfIndex); - /** * Update the table id list of a given query. * @param uid child table uid diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 21b7309055..01f347fd05 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -67,50 +67,6 @@ typedef struct SFileBlockInfo { #define TOP_BOTTOM_QUERY_LIMIT 100 #define FUNCTIONS_NAME_MAX_LENGTH 16 -#define FUNCTION_INVALID_ID -1 -#define FUNCTION_COUNT 0 -#define FUNCTION_SUM 1 -#define FUNCTION_AVG 2 -#define FUNCTION_MIN 3 -#define FUNCTION_MAX 4 -#define FUNCTION_STDDEV 5 -#define FUNCTION_PERCT 6 -#define FUNCTION_APERCT 7 -#define FUNCTION_FIRST 8 -#define FUNCTION_LAST 9 -#define FUNCTION_LAST_ROW 10 -#define FUNCTION_TOP 11 -#define FUNCTION_BOTTOM 12 -#define FUNCTION_SPREAD 13 -#define FUNCTION_TWA 14 -#define FUNCTION_LEASTSQR 15 - -#define FUNCTION_TS 16 -#define FUNCTION_TS_DUMMY 17 -#define FUNCTION_TAG_DUMMY 18 -#define FUNCTION_TS_COMP 19 - -#define FUNCTION_TAG 20 -#define FUNCTION_PRJ 21 - -#define FUNCTION_TAGPRJ 22 -#define FUNCTION_ARITHM 23 -#define FUNCTION_DIFF 24 - -#define FUNCTION_FIRST_DST 25 -#define FUNCTION_LAST_DST 26 -#define FUNCTION_STDDEV_DST 27 -#define FUNCTION_INTERP 28 - -#define FUNCTION_RATE 29 -#define FUNCTION_IRATE 30 -#define FUNCTION_TID_TAG 31 -#define FUNCTION_DERIVATIVE 32 -#define FUNCTION_BLKINFO 33 - - -#define FUNCTION_COV 38 - typedef struct SResultRowEntryInfo { bool initialized:1; // output buffer has been initialized bool complete:1; // query has completed @@ -180,10 +136,9 @@ typedef struct SqlFunctionCtx { char *pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - int64_t *ptsList; // corresponding timestamp array list + int64_t *ptsList; // corresponding timestamp array list, todo remove it SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ int32_t offset; - SVariant tag; struct SResultRowEntryInfo *resultInfo; SSubsidiaryResInfo subsidiaries; SPoint1 start; diff --git a/include/util/tlist.h b/include/util/tlist.h index 43833d7ecd..1954bda145 100644 --- a/include/util/tlist.h +++ b/include/util/tlist.h @@ -229,7 +229,7 @@ int32_t tdListAppend(SList *list, void *data); SListNode *tdListPopHead(SList *list); SListNode *tdListPopTail(SList *list); SListNode *tdListGetHead(SList *list); -SListNode *tsListGetTail(SList *list); +SListNode *tdListGetTail(SList *list); SListNode *tdListPopNode(SList *list, SListNode *node); void tdListMove(SList *src, SList *dst); void tdListDiscard(SList *list); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 63c398618f..dcd4e8f918 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -75,6 +75,7 @@ typedef struct SResultRowInfo { int32_t size; // number of result set int32_t capacity; // max capacity SResultRowPosition cur; + SList* openWindow; } SResultRowInfo; struct SqlFunctionCtx; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9e15f3b12d..3f25939bd2 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -451,15 +451,15 @@ typedef struct SIntervalAggOperatorInfo { int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. STimeWindow win; // query time range bool timeWindowInterpo; // interpolation needed or not - char** pRow; // previous row/tuple of already processed datablock + SArray* pInterpCols; // interpolation columns SAggSupporter aggSup; // aggregate supporter STableQueryInfo* pCurrent; // current tableQueryInfo struct int32_t order; // current SSDataBlock scan order EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] SArray* pUpdatedWindow; // updated time window due to the input data block from the downstream operator. STimeWindowAggSupp twAggSup; - struct SFillInfo* pFillInfo; // fill info bool invertible; + SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. } SIntervalAggOperatorInfo; typedef struct SStreamFinalIntervalOperatorInfo { @@ -802,7 +802,7 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t initCacheSupporter(SCatchSupporter* pCatchSup, size_t rowSize, const char* pKey, const char* pDir); int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey); -SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize); SResultWindowInfo* getSessionTimeWindow(SArray* pWinInfos, TSKEY ts, int64_t gap, int32_t* pIndex); int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pTs, int32_t rows, diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5a02547f58..b8915ae218 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -258,32 +258,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); } -static int64_t getNumOfResultWindowRes(STaskRuntimeEnv* pRuntimeEnv, SResultRowPosition *pos, int32_t* rowCellInfoOffset) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - ASSERT(0); - - for (int32_t j = 0; j < pQueryAttr->numOfOutput; ++j) { - int32_t functionId = 0;//pQueryAttr->pExpr1[j].base.functionId; - - /* - * ts, tag, tagprj function can not decide the output number of current query - * the number of output result is decided by main output - */ - if (functionId == FUNCTION_TS || functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ) { - continue; - } - -// SResultRowEntryInfo *pResultInfo = getResultCell(pResultRow, j, rowCellInfoOffset); -// assert(pResultInfo != NULL); -// -// if (pResultInfo->numOfRes > 0) { -// return pResultInfo->numOfRes; -// } - } - - return 0; -} - static int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) { int32_t left = *(int32_t *)pLeft; int32_t right = *(int32_t *)pRight; @@ -381,7 +355,7 @@ static int32_t mergeIntoGroupResultImplRv(STaskRuntimeEnv *pRuntimeEnv, SGroupRe } - int64_t num = getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset); + int64_t num = 0;//getNumOfResultWindowRes(pRuntimeEnv, &pResultRowCell->pos, rowCellInfoOffset); if (num <= 0) { continue; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 3e52ff1836..9c5f93613a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -239,36 +239,6 @@ static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) { return true; } -static void prepareResultListBuffer(SResultRowInfo* pResultRowInfo, jmp_buf env) { - int64_t newCapacity = 0; - - // more than the capacity, reallocate the resources - if (pResultRowInfo->size < pResultRowInfo->capacity) { - return; - } - - if (pResultRowInfo->capacity > 10000) { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.25); - } else { - newCapacity = (int64_t)(pResultRowInfo->capacity * 1.5); - } - - if (newCapacity <= pResultRowInfo->capacity) { - newCapacity += 4; - } - - char* p = taosMemoryRealloc(pResultRowInfo->pPosition, newCapacity * sizeof(SResultRowPosition)); - if (p == NULL) { - longjmp(env, TSDB_CODE_OUT_OF_MEMORY); - } - - pResultRowInfo->pPosition = (SResultRowPosition*)p; - - int32_t inc = (int32_t)newCapacity - pResultRowInfo->capacity; - memset(&pResultRowInfo->pPosition[pResultRowInfo->capacity], 0, sizeof(SResultRowPosition) * inc); - pResultRowInfo->capacity = (int32_t)newCapacity; -} - static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, char* pData, int16_t bytes, bool masterscan, uint64_t uid) { bool existed = false; @@ -306,7 +276,7 @@ static bool chkResultRowFromKey(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pR return p1 != NULL; } -SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) { +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, int32_t interBufSize) { SFilePage* pData = NULL; // in the first scan, new space needed for results @@ -392,25 +362,22 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR } // allocate a new buffer page - prepareResultListBuffer(pResultRowInfo, pTaskInfo->env); if (pResult == NULL) { ASSERT(pSup->resultRowSize > 0); - pResult = getNewResultRow_rv(pResultBuf, groupId, pSup->resultRowSize); + pResult = getNewResultRow(pResultBuf, groupId, pSup->resultRowSize); + initResultRow(pResult); // add a new result set for a new group SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; - taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, - sizeof(SResultRowPosition)); + taosHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, sizeof(SResultRowPosition)); } // 2. set the new time window to be the new active time window - pResultRowInfo->pPosition[pResultRowInfo->size++] = - (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; // too many time window in query - if (pResultRowInfo->size > MAX_INTERVAL_TIME_WINDOW) { + if (taosHashGetSize(pSup->pResultRowHashTable) > MAX_INTERVAL_TIME_WINDOW) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } @@ -1218,7 +1185,6 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { taosVariantDestroy(&pCtx[i].param[j].param); } - taosVariantDestroy(&pCtx[i].tag); taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx); taosMemoryFree(pCtx[i].input.pData); taosMemoryFree(pCtx[i].input.pColumnDataAgg); @@ -1248,9 +1214,9 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo) { pTaskInfo->code = TSDB_CODE_TSC_Q static bool isCachedLastQuery(STaskAttr* pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { - continue; - } +// if (functionId == FUNCTION_LAST || functionId == FUNCTION_LAST_DST) { +// continue; +// } return false; } @@ -1300,7 +1266,7 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = getExprFunctionId(&pQuery->pExpr1[i]); - +#if 0 if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || functionId == FUNCTION_TAG_DUMMY) { continue; @@ -1311,6 +1277,8 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { } else { hasOtherFunc = true; } +#endif + } if (hasFirstLastFunc && status == BLK_DATA_NOT_LOAD) { @@ -1786,41 +1754,13 @@ void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOf // set the correct pointer after the memory buffer reallocated. int32_t functionId = pBInfo->pCtx[i].functionId; - +#if 0 if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { // if (i > 0) pBInfo->pCtx[i].pTsOutput = pBInfo->pCtx[i - 1].pOutput; } - } -} +#endif -void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput) { - bool needCopyTs = false; - int32_t tsNum = 0; - char* src = NULL; - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if (functionId == FUNCTION_DIFF || functionId == FUNCTION_DERIVATIVE) { - needCopyTs = true; - if (i > 0 && pCtx[i - 1].functionId == FUNCTION_TS_DUMMY) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i - 1); // find ts data - src = pColRes->pData; - } - } else if (functionId == FUNCTION_TS_DUMMY) { - tsNum++; - } - } - - if (!needCopyTs) return; - if (tsNum < 2) return; - if (src == NULL) return; - - for (int32_t i = 0; i < numOfOutput; i++) { - int32_t functionId = pCtx[i].functionId; - if (functionId == FUNCTION_TS_DUMMY) { - SColumnInfoData* pColRes = taosArrayGet(pRes->pDataBlock, i); - memcpy(pColRes->pData, src, pColRes->info.bytes * pRes->info.rows); - } } } @@ -3569,7 +3509,7 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi offset += sizeof(int32_t); uint64_t tableGroupId = *(uint64_t*)(result + offset); - SResultRow* resultRow = getNewResultRow_rv(pSup->pResultBuf, tableGroupId, pSup->resultRowSize); + SResultRow* resultRow = getNewResultRow(pSup->pResultBuf, tableGroupId, pSup->resultRowSize); if (!resultRow) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_INVALID_INPUT); } @@ -3592,10 +3532,6 @@ bool aggDecodeResultRow(SOperatorInfo* pOperator, SAggSupporter* pSup, SOptrBasi offset += valueLen; initResultRow(resultRow); - prepareResultListBuffer(&pInfo->resultRowInfo, pOperator->pTaskInfo->env); - // pInfo->resultRowInfo.cur = pInfo->resultRowInfo.size; - // pInfo->resultRowInfo.pPosition[pInfo->resultRowInfo.size++] = - // (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; } @@ -3887,18 +3823,6 @@ static SSDataBlock* doFill(SOperatorInfo* pOperator) { } } -// todo set the attribute of query scan count -static int32_t getNumOfScanTimes(STaskAttr* pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = getExprFunctionId(&pQueryAttr->pExpr1[i]); - if (functionId == FUNCTION_STDDEV || functionId == FUNCTION_PERCT) { - return 2; - } - } - - return 1; -} - static void destroyOperatorInfo(SOperatorInfo* pOperator) { if (pOperator == NULL) { return; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index ef770e8afc..7ba37c282d 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -110,9 +110,11 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo return true; } -static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { +static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex) { SColumnDataAgg* pColAgg = NULL; + size_t numOfGroupCols = taosArrayGetSize(pGroupCols); + for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); @@ -208,7 +210,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { for (int32_t j = 0; j < pBlock->info.rows; ++j) { // Compare with the previous row of this column, and do not set the output buffer again if they are identical. if (!pInfo->isInit) { - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); pInfo->isInit = true; num++; continue; @@ -223,7 +225,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { // The first row of a new block does not belongs to the previous existed group if (j == 0) { num++; - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); continue; } @@ -238,7 +240,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { // assign the group keys or user input constant values if required doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex); - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); num = 1; } @@ -396,7 +398,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t numOfGroupCols = taosArrayGetSize(pInfo->pGroupCols); for (int32_t j = 0; j < pBlock->info.rows; ++j) { - recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j, numOfGroupCols); + recordNewGroupKeys(pInfo->pGroupCols, pInfo->pGroupColVals, pBlock, j); int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); SDataGroupInfo* pGInfo = NULL; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index fe7668f370..11b50a38bc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -534,8 +534,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->pPseudoCtx = createSqlFunctionCtx(pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, &pInfo->rowCellInfoOffset); } - pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; - // pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose +// pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; + pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose pInfo->readHandle = *readHandle; pInfo->interval = extractIntervalInfo(pTableScanNode); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 5d0a0a0270..a1a7aecf5f 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1,3 +1,4 @@ +#include #include "executorimpl.h" #include "functionMgt.h" #include "tdatablock.h" @@ -339,34 +340,40 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o tw->ekey -= 1; } -void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, SArray* pDataBlock, TSKEY prevTs, +void doTimeWindowInterpolation(SIntervalAggOperatorInfo *pInfo, int32_t numOfExprs, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { - SExprInfo* pExpr = pOperator->pExpr; + SqlFunctionCtx* pCtx = pInfo->binfo.pCtx; - SqlFunctionCtx* pCtx = pInfo->pCtx; + int32_t index = 1; + for (int32_t k = 0; k < numOfExprs; ++k) { - for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { - int32_t functionId = pCtx[k].functionId; - if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { + // todo use flag instead of function name + if (strcmp(pCtx[k].pExpr->pExpr->_function.functionName, "twa") != 0) { pCtx[k].start.key = INT64_MIN; continue; } - SColIndex* pColIndex = NULL /*&pExpr[k].base.colInfo*/; - int16_t index = pColIndex->colIndex; - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); +// if (functionId != FUNCTION_TWA && functionId != FUNCTION_INTERP) { +// pCtx[k].start.key = INT64_MIN; +// continue; +// } + + SFunctParam* pParam = &pCtx[k].param[0]; + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, pParam->pCol->slotId); + + ASSERT(pColInfo->info.colId == pParam->pCol->colId && curTs != windowKey); - // assert(pColInfo->info.colId == pColIndex->info.colId && curTs != windowKey); double v1 = 0, v2 = 0, v = 0; - if (prevRowIndex == -1) { - // GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pRuntimeEnv->prevRow[index]); + SGroupKeys* p = taosArrayGet(pInfo->pPrevValues, index); + GET_TYPED_DATA(v1, double, pColInfo->info.type, p->pData); } else { - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char*)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + GET_TYPED_DATA(v1, double, pColInfo->info.type, colDataGetData(pColInfo, prevRowIndex)); } - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char*)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + GET_TYPED_DATA(v2, double, pColInfo->info.type, colDataGetData(pColInfo, curRowIndex)); +#if 0 if (functionId == FUNCTION_INTERP) { if (type == RESULT_ROW_START_INTERP) { pCtx[k].start.key = prevTs; @@ -386,6 +393,8 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, } } } else if (functionId == FUNCTION_TWA) { +#endif + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; SPoint point2 = (SPoint){.key = curTs, .val = &v2}; SPoint point = (SPoint){.key = windowKey, .val = &v}; @@ -399,8 +408,13 @@ void doTimeWindowInterpolation(SOperatorInfo* pOperator, SOptrBasicInfo* pInfo, pCtx[k].end.key = point.key; pCtx[k].end.val = v; } + + index += 1; } +#if 0 } +#endif + } static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { @@ -415,53 +429,53 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in } } -static bool setTimeWindowInterpolationStartTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t pos, - int32_t numOfRows, SArray* pDataBlock, const TSKEY* tsCols, - STimeWindow* win) { - bool ascQuery = true; +static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo *pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, int32_t pos, + SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win) { + int32_t numOfRows = pBlock->info.rows; + + bool ascQuery = (pInfo->order == TSDB_ORDER_ASC); + int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); + TSKEY curTs = tsCols[pos]; - TSKEY lastTs = 0; //*(TSKEY*)pRuntimeEnv->prevRow[0]; + + SGroupKeys* pTsKey = taosArrayGet(pInfo->pPrevValues, 0); + TSKEY lastTs = *(int64_t*) pTsKey->pData; // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. // start exactly from this point, no need to do interpolation TSKEY key = ascQuery ? win->skey : win->ekey; if (key == curTs) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP); return true; } - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery))) { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == ( - 1) && !ascQuery))) { + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP); return true; } - int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, prevTs, pos - step, curTs, pos, key, - RESULT_ROW_START_INTERP); + doTimeWindowInterpolation(pInfo, numOfExprs, pBlock->pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); return true; } -static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFunctionCtx* pCtx, int32_t endRowIndex, - SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, - STimeWindow* win) { - int32_t order = TSDB_ORDER_ASC; - int32_t numOfOutput = pOperatorInfo->numOfExprs; +static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo *pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, int32_t endRowIndex, + SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { + int32_t order = pInfo->order; TSKEY actualEndKey = tsCols[endRowIndex]; - TSKEY key = order ? win->ekey : win->skey; + TSKEY key = (order == TSDB_ORDER_ASC) ? win->ekey : win->skey; // not ended in current data block, do not invoke interpolation - if ((key > blockEkey /*&& QUERY_IS_ASC_QUERY(pQueryAttr)*/) || - (key < blockEkey /*&& !QUERY_IS_ASC_QUERY(pQueryAttr)*/)) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + if ((key > blockEkey && (order == TSDB_ORDER_ASC)) || (key < blockEkey && (order == TSDB_ORDER_DESC))) { + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP); return false; } // there is actual end point of current time window, no interpolation need if (key == actualEndKey) { - setNotInterpoWindowKey(pCtx, numOfOutput, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP); return true; } @@ -470,7 +484,7 @@ static bool setTimeWindowInterpolationEndTs(SOperatorInfo* pOperatorInfo, SqlFun assert(nextRowIndex >= 0); TSKEY nextKey = tsCols[nextRowIndex]; - doTimeWindowInterpolation(pOperatorInfo, pOperatorInfo->info, pDataBlock, actualEndKey, endRowIndex, nextKey, + doTimeWindowInterpolation(pInfo, numOfExprs, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, RESULT_ROW_END_INTERP); return true; } @@ -542,8 +556,8 @@ static int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, return startPos; } -static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { - assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); +static bool isResultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { + ASSERT(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { return pResult->startInterp == true; } else { @@ -560,34 +574,33 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } } -static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBlock* pBlock, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep, - int32_t order, bool timeWindowInterpo) { - if (!timeWindowInterpo) { +static void doWindowBorderInterpolation(SIntervalAggOperatorInfo *pInfo, SSDataBlock* pBlock, int32_t numOfExprs, SqlFunctionCtx* pCtx, + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + if (!pInfo->timeWindowInterpo) { return; } assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInfo->order); if (pBlock->pDataBlock == NULL) { // tscError("pBlock->pDataBlock == NULL"); return; } + // todo set the correct primary timestamp column index SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); TSKEY* tsCols = (TSKEY*)(pColInfo->pData); - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + bool done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP); if (!done) { // it is not interpolated, now start to generated the interpolated value int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pOperatorInfo, pCtx, startRowIndex, pBlock->info.rows, - pBlock->pDataBlock, tsCols, win); + bool interp = setTimeWindowInterpolationStartTs(pInfo, pCtx, numOfExprs, startRowIndex, pBlock, tsCols, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP); } // point interpolation does not require the end key time window interpolation. @@ -596,29 +609,48 @@ static void doWindowBorderInterpolation(SOperatorInfo* pOperatorInfo, SSDataBloc // } // interpolation query does not generate the time window end interpolation - done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP); if (!done) { int32_t endRowIndex = startPos + (forwardStep - 1) * step; - TSKEY endKey = (order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; + TSKEY endKey = (pInfo->order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; bool interp = - setTimeWindowInterpolationEndTs(pOperatorInfo, pCtx, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); + setTimeWindowInterpolationEndTs(pInfo, pCtx, numOfExprs, endRowIndex, pBlock->pDataBlock, tsCols, endKey, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } } else { - setNotInterpoWindowKey(pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP); } } -static void saveDataBlockLastRow(char** pRow, SArray* pDataBlock, int32_t rowIndex, int32_t numOfCols) { - if (pDataBlock == NULL) { +static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, SArray* pCols) { + if (pBlock->pDataBlock == NULL) { return; } - for (int32_t k = 0; k < numOfCols; ++k) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); + size_t num = taosArrayGetSize(pPrevKeys); + for (int32_t k = 0; k < num; ++k) { + SColumn* pc = taosArrayGet(pCols, k); + + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pc->slotId); + + SGroupKeys* pkey = taosArrayGet(pPrevKeys, k); + for(int32_t i = pBlock->info.rows - 1; i >= 0; --i) { + if (colDataIsNull_s(pColInfo, i)) { + continue; + } + + char* val = colDataGetData(pColInfo, i); + if (IS_VAR_DATA_TYPE(pkey->type)) { + memcpy(pkey->pData, val, varDataTLen(val)); + ASSERT(varDataTLen(val) <= pkey->bytes); + } else { + memcpy(pkey->pData, val, pkey->bytes); + } + + break; + } } } @@ -635,7 +667,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } int32_t step = 1; - bool ascScan = (pInfo->order == TSDB_ORDER_ASC); + + bool ascScan = (pInfo->order == TSDB_ORDER_ASC); // int32_t prevIndex = pResultRowInfo->curPos; @@ -679,48 +712,69 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe ASSERT(forwardStep > 0); // prev time window not interpolation yet. - // int32_t curIndex = pResultRowInfo->curPos; + SResultRowPosition pos = {0}; + if (pInfo->timeWindowInterpo) { + pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); -#if 0 - if (prevIndex != -1 && prevIndex < curIndex && pInfo->timeWindowInterpo) { - for (int32_t j = prevIndex; j < curIndex; ++j) { // previous time window may be all closed already. - SResultRow* pRes = getResultRow(pResultRowInfo, j); - if (pRes->closed) { - assert(resultRowInterpolated(pRes, RESULT_ROW_START_INTERP) && resultRowInterpolated(pRes, RESULT_ROW_END_INTERP)); + SResultRowPosition* px = (SResultRowPosition*)pn->data; + + // do nothing + if (pn != NULL && px->pageId == pos.pageId && px->offset == pos.offset) { + + } else { + tdListAppend(pResultRowInfo->openWindow, &pos); + } + } + + if (pInfo->timeWindowInterpo) { + while (1) { + SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); + SResultRowPosition* p1 = (SResultRowPosition*)pn->data; + if (p1->pageId == pos.pageId && p1->offset == pos.offset) { + break; + } + + SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1); + if (pr->closed) { + ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); + tdListPopHead(pResultRowInfo->openWindow); continue; } - STimeWindow w = pRes->win; - ret = setTimeWindowOutputBuf(pResultRowInfo, pBlock->info.uid, &w, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); + STimeWindow w = pr->win; + ret = setTimeWindowOutputBuf(pResultRowInfo, &w, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - assert(!resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - doTimeWindowInterpolation(pOperatorInfo, &pInfo->binfo, pBlock->pDataBlock, *(TSKEY*)pInfo->pRow[0], -1, - tsCols[startPos], startPos, w.ekey, RESULT_ROW_END_INTERP); + ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + SGroupKeys *pTsKey = taosArrayGet(pInfo->pPrevValues, 0); + int64_t prevTs = *(int64_t*) pTsKey->pData; + doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, + w.ekey, RESULT_ROW_END_INTERP); setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); - doApplyFunctions(pInfo->binfo.pCtx, &w, &pInfo->timeWindowData, startPos, 0, tsCols, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + tdListPopHead(pResultRowInfo->openWindow); } + // todo keep current outputbuffer info to avoid repeatly set the buffer info // restore current time window - ret = setTimeWindowOutputBuf(pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, tableGroupId, - pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, - pTaskInfo); + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - } -#endif - // window start key interpolation - doWindowBorderInterpolation(pOperatorInfo, pBlock, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep, - pInfo->order, false); + // window start key interpolation + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep); + } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, @@ -743,12 +797,12 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; + SResKeyPos* pPos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pPos->groupId = tableGroupId; + pPos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*)pPos->key = pResult->win.skey; - taosArrayPush(pUpdated, &pos); + taosArrayPush(pUpdated, &pPos); } ekey = ascScan? nextWin.ekey:nextWin.skey; @@ -756,8 +810,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, @@ -765,8 +818,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } if (pInfo->timeWindowInterpo) { - int32_t rowIndex = ascScan ? (pBlock->info.rows - 1) : 0; - saveDataBlockLastRow(pInfo->pRow, pBlock->pDataBlock, rowIndex, pBlock->info.numOfCols); + int32_t rowIndex = ascScan? (pBlock->info.rows - 1) : 0; + saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols); } return pUpdated; @@ -1040,8 +1093,7 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } } -void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, - SOptrBasicInfo* pBinfo, int32_t numOfOutput) { +void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SOptrBasicInfo* pBinfo, int32_t numOfOutput) { SResultRow* pResult = getResultRowByPos(pResultBuf, p1); SqlFunctionCtx* pCtx = pBinfo->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1169,7 +1221,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { } } -bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { +static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { for (int32_t i = 0; i < numOfCols; i++) { if (!fmIsInvertible(pFCtx[i].functionId)) { return false; @@ -1178,6 +1230,50 @@ bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { return true; } +static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SArray** pInterpList, SArray** pPrevCols) { + // the primary timestamp column + bool needed = false; + *pInterpList = taosArrayInit(4, sizeof(SColumn)); + *pPrevCols = taosArrayInit(4, sizeof(SGroupKeys)); + + { // ts column + SColumn c = {0}; + c.colId = 1; + c.slotId = 0; + c.type = TSDB_DATA_TYPE_TIMESTAMP; + c.bytes = sizeof(int64_t); + taosArrayPush(*pInterpList, &c); + + SGroupKeys key = {0}; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(*pPrevCols, &key); + } + + for(int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = pCtx[i].pExpr; + + if (strcmp(pExpr->pExpr->_function.functionName, "twa") == 0) { + SFunctParam* pParam = &pExpr->base.pParam[0]; + + SColumn c = *pParam->pCol; + taosArrayPush(*pInterpList, &c); + needed = true; + + SGroupKeys key = {0}; + key.bytes = c.bytes; + key.type = c.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, c.bytes); + taosArrayPush(*pPrevCols, &key); + } + } + + return needed; +} + SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, STimeWindowAggSupp* pTwAggSupp, SExecTaskInfo* pTaskInfo) { @@ -1187,11 +1283,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = *pTwAggSupp; + pInfo->primaryTsIndex = primaryTsSlotId; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; @@ -1201,23 +1298,30 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initAggInfo(&pInfo->binfo, &pInfo->aggSup, pExprInfo, numOfCols, pResBlock, keyBufSize, pTaskInfo->id.str); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); + pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols); pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API - if (code != TSDB_CODE_SUCCESS) { + pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, &pInfo->pInterpCols, &pInfo->pPrevValues); + if (pInfo->timeWindowInterpo) { + pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); + } + + // pInfo->pTableQueryInfo = initTableQueryInfo(pTableGroupInfo); + if (code != TSDB_CODE_SUCCESS /* || pInfo->pTableQueryInfo == NULL*/) { goto _error; } initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)1); - pOperator->name = "TimeIntervalAggOperator"; + pOperator->name = "TimeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERVAL; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->pExpr = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; - pOperator->numOfExprs = numOfCols; - pOperator->info = pInfo; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->pExpr = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->numOfExprs = numOfCols; + pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); @@ -1704,8 +1808,8 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB forwardStep = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // window start(end) key interpolation - doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, - pInfo->order, false); + // disable it temporarily +// doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); @@ -2082,7 +2186,7 @@ static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pRes } if (pWinInfo->pos.pageId == -1) { - *pResult = getNewResultRow_rv(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize); + *pResult = getNewResultRow(pAggSup->pResultBuf, groupId, pAggSup->resultRowSize); if (*pResult == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index cac86be917..68b83f4a19 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -140,6 +140,10 @@ bool uniqueFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) int32_t uniqueFunction(SqlFunctionCtx *pCtx); //int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); +bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool twaFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); +int32_t twaFunction(SqlFunctionCtx *pCtx); +int32_t twaFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index d779cf50f4..c3d61d426d 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -52,13 +52,6 @@ typedef struct SInterpInfoDetail { int8_t primaryCol; } SInterpInfoDetail; -typedef struct STwaInfo { - int8_t hasResult; // flag to denote has value - double dOutput; - SPoint1 p; - STimeWindow win; -} STwaInfo; - bool topbot_datablock_filter(SqlFunctionCtx *pCtx, const char *minval, const char *maxval); /** diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index c2b1fcd549..85e0d1d521 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -18,7 +18,6 @@ #include "querynodes.h" #include "scalar.h" #include "taoserror.h" -#include "tdatablock.h" static int32_t buildFuncErrMsg(char* pErrBuf, int32_t len, int32_t errCode, const char* pFormat, ...) { va_list vArgList; @@ -314,7 +313,7 @@ static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len pValue->notReserved = true; - uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; if (!IS_INTEGER_TYPE(paraType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1119,6 +1118,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = firstLastFinalize, .combineFunc = lastCombine, }, + { + .name = "twa", + .type = FUNCTION_TYPE_TWA, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateInNumOutDou, + .getEnvFunc = getTwaFuncEnv, + .initFunc = twaFunctionSetup, + .processFunc = twaFunction, + .finalizeFunc = twaFinalize + }, { .name = "histogram", .type = FUNCTION_TYPE_HISTOGRAM, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e2ee6cc6fe..78e057f7a8 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -225,7 +225,6 @@ typedef struct SUniqueInfo { int32_t numOfPoints; uint8_t colType; int16_t colBytes; - bool hasNull; //null is not hashable, handle separately SHashObj *pHash; char pItems[]; } SUniqueInfo; @@ -300,7 +299,7 @@ int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - //pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; char* in = GET_ROWCELL_INTERBUF(pResInfo); colDataAppend(pCol, pBlock->info.rows, in, pResInfo->isNullRes); @@ -829,8 +828,10 @@ int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) { int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; - SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + + int32_t type = pInput->pData[0]->info.type; + SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + if (IS_INTEGER_TYPE(type)) { pAvgRes->result = pAvgRes->sum.isum / ((double)pAvgRes->count); } else { @@ -1827,7 +1828,7 @@ bool percentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultI } int32_t percentileFunction(SqlFunctionCtx* pCtx) { - int32_t notNullElems = 0; + int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -1905,11 +1906,11 @@ int32_t percentileFunction(SqlFunctionCtx* pCtx) { } char* data = colDataGetData(pCol, i); - notNullElems += 1; + numOfElems += 1; tMemBucketPut(pInfo->pMemBucket, data, 1); } - SET_VAL(pResInfo, notNullElems, 1); + SET_VAL(pResInfo, numOfElems, 1); } return TSDB_CODE_SUCCESS; @@ -1983,7 +1984,7 @@ bool apercentileFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResult } int32_t apercentileFunction(SqlFunctionCtx* pCtx) { - int32_t notNullElems = 0; + int32_t numOfElems = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -2000,7 +2001,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; } - notNullElems += 1; + numOfElems += 1; char* data = colDataGetData(pCol, i); double v = 0; // value @@ -2013,7 +2014,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { if (colDataIsNull_f(pCol->nullbitmap, i)) { continue; } - notNullElems += 1; + numOfElems += 1; char* data = colDataGetData(pCol, i); double v = 0; @@ -2022,7 +2023,7 @@ int32_t apercentileFunction(SqlFunctionCtx* pCtx) { } } - SET_VAL(pResInfo, notNullElems, 1); + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -3690,7 +3691,6 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { TSKEY* tsList = (int64_t*)pInput->pPTS->pData; SColumnInfoData* pInputCol = pInput->pData[0]; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; int32_t startOffset = pCtx->offset; @@ -3713,24 +3713,6 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { return pInfo->numSampled; } -//int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { -// SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); -// SSampleInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); -// int32_t slotId = pCtx->pExpr->base.resSchema.slotId; -// SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); -// -// //int32_t currentRow = pBlock->info.rows; -// pResInfo->numOfRes = pInfo->numSampled; -// -// for (int32_t i = 0; i < pInfo->numSampled; ++i) { -// colDataAppend(pCol, i, pInfo->data + i * pInfo->colBytes, false); -// //TODO: handle ts output -// } -// -// return pResInfo->numOfRes; -//} - - bool getTailFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); @@ -3806,7 +3788,6 @@ int32_t tailFunction(SqlFunctionCtx* pCtx) { TSKEY* tsList = (int64_t*)pInput->pPTS->pData; SColumnInfoData* pInputCol = pInput->pData[0]; - SColumnInfoData* pTsOutput = pCtx->pTsOutput; SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; int32_t startOffset = pCtx->offset; @@ -3873,7 +3854,6 @@ bool uniqueFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { pInfo->numOfPoints = 0; pInfo->colType = pCtx->resDataInfo.type; pInfo->colBytes = pCtx->resDataInfo.bytes; - pInfo->hasNull = false; if (pInfo->pHash != NULL) { taosHashClear(pInfo->pHash); } else { @@ -3887,11 +3867,10 @@ static void doUniqueAdd(SUniqueInfo* pInfo, char *data, TSKEY ts, bool isNull) { if (isNull == true) { int32_t size = sizeof(SUniqueItem) + pInfo->colBytes; SUniqueItem *pItem = (SUniqueItem *)(pInfo->pItems + pInfo->numOfPoints * size); - if (pInfo->hasNull == false && pItem->isNull == false) { + if (pItem->isNull == false) { pItem->timestamp = ts; pItem->isNull = true; pInfo->numOfPoints++; - pInfo->hasNull = true; } else if (pItem->timestamp > ts && pItem->isNull == true) { pItem->timestamp = ts; } @@ -3911,8 +3890,6 @@ static void doUniqueAdd(SUniqueInfo* pInfo, char *data, TSKEY ts, bool isNull) { } else if (pHashItem->timestamp > ts) { pHashItem->timestamp = ts; } - - return; } int32_t uniqueFunction(SqlFunctionCtx* pCtx) { @@ -3954,7 +3931,7 @@ int32_t uniqueFunction(SqlFunctionCtx* pCtx) { int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); - SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SUniqueInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); @@ -3967,3 +3944,325 @@ int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } +typedef struct STwaInfo { + double dOutput; + int8_t hasResult; // flag to denote has value + SPoint1 p; + STimeWindow win; +} STwaInfo; + +bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(STwaInfo); + return true; +} + +bool twaFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { + if (!functionSetup(pCtx, pResultInfo)) { + return false; + } + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pInfo->p.key = INT64_MIN; + pInfo->win = TSWINDOW_INITIALIZER; + return true; +} + +static double twa_get_area(SPoint1 s, SPoint1 e) { + if ((s.val >= 0 && e.val >= 0)|| (s.val <=0 && e.val <= 0)) { + return (s.val + e.val) * (e.key - s.key) / 2; + } + + double x = (s.key * e.val - e.key * s.val)/(e.val - s.val); + double val = (s.val * (x - s.key) + e.val * (e.key - x)) / 2; + return val; +} + +int32_t twaFunction(SqlFunctionCtx* pCtx) { + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + int32_t numOfElems = 0; + + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SPoint1* last = &pInfo->p; + + int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pCtx->order); + int32_t i = pInput->startRowIndex; + for (i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (!colDataIsNull_f(pInputCol->nullbitmap, i)) { + break; + } + } + + if (pCtx->start.key != INT64_MIN) { + ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); + + ASSERT(last->key == INT64_MIN); + last->key = tsList[i]; + + GET_TYPED_DATA(last->val, double, pInputCol->info.type, colDataGetData(pInputCol, i)); + + pInfo->dOutput += twa_get_area(pCtx->start, *last); + + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = pCtx->start.key; + numOfElems++; + i += step; + } else if (pInfo->p.key == INT64_MIN) { + last->key = tsList[i]; + GET_TYPED_DATA(last->val, double, pInputCol->info.type, colDataGetData(pInputCol, i)); + + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = last->key; + numOfElems++; + i += step; + } + + // calculate the value of + switch(pInputCol->info.type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t *val = (int8_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *val = (int16_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *val = (int32_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t *val = (int64_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = (double)val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *val = (float*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = (double)val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double *val = (double*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: { + uint8_t *val = (uint8_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + uint16_t *val = (uint16_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UINT: { + uint32_t *val = (uint32_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: { + uint64_t *val = (uint64_t*) colDataGetData(pInputCol, 0); + for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + +#ifndef _TD_NINGSI_60 + SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; +#else + SPoint1 st; + st.key = tsList[i]; + st.val = (double) val[i]; +#endif + pInfo->dOutput += twa_get_area(pInfo->p, st); + pInfo->p = st; + } + break; + } + + default: assert(0); + } + + // the last interpolated time window value + if (pCtx->end.key != INT64_MIN) { + pInfo->dOutput += twa_get_area(pInfo->p, pCtx->end); + pInfo->p = pCtx->end; + } + + pInfo->win.ekey = pInfo->p.key; + + SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + return TSDB_CODE_SUCCESS; +} + +/* + * To copy the input to interResBuf to avoid the input buffer space be over writen + * by next input data. The TWA function only applies to each table, so no merge procedure + * is required, we simply copy to the resut ot interResBuffer. + */ +//void twa_function_copy(SQLFunctionCtx *pCtx) { +// assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); +// SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); +// +// memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); +// pResInfo->hasResult = ((STwaInfo *)pCtx->pInput)->hasResult; +//} + +int32_t twaFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { + SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); + if (pInfo->hasResult != DATA_SET_FLAG) { + pResInfo->isNullRes = 1; + } else { + // assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); + if (pInfo->win.ekey == pInfo->win.skey) { + pInfo->dOutput = pInfo->p.val; + } else { + pInfo->dOutput = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); + } + + pResInfo->numOfRes = 1; + } + + return functionFinalize(pCtx, pBlock); +} + diff --git a/source/libs/function/src/taggfunction.c b/source/libs/function/src/taggfunction.c index 950655e480..e683a38cbd 100644 --- a/source/libs/function/src/taggfunction.c +++ b/source/libs/function/src/taggfunction.c @@ -236,7 +236,7 @@ bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry) { bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry) { return pEntry->initialized; } - +#if 0 int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, bool isSuperTable/*, SUdfInfo* pUdfInfo*/) { if (!isValidDataType(dataType)) { @@ -470,6 +470,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } +#endif static bool function_setup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { if (pResultInfo->initialized) { diff --git a/source/util/src/tlist.c b/source/util/src/tlist.c index 1d17b4a9e1..b1c0188051 100644 --- a/source/util/src/tlist.c +++ b/source/util/src/tlist.c @@ -95,7 +95,7 @@ SListNode *tdListPopTail(SList *list) { SListNode *tdListGetHead(SList *list) { return TD_DLIST_HEAD(list); } -SListNode *tsListGetTail(SList *list) { return TD_DLIST_TAIL(list); } +SListNode *tdListGetTail(SList *list) { return TD_DLIST_TAIL(list); } SListNode *tdListPopNode(SList *list, SListNode *node) { TD_DLIST_POP(list, node); From f7c21c45023100a7556a0dd152229c073b1292b9 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 30 May 2022 18:00:03 +0800 Subject: [PATCH 03/47] feat:add schemaless db paras --- include/common/tmsg.h | 2 ++ include/libs/parser/parser.h | 1 + include/util/taoserror.h | 1 + source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 1 + source/client/src/clientSml.c | 33 +++++++++++++++++++++++++ source/common/src/tmsg.c | 2 ++ source/dnode/mnode/impl/inc/mndDef.h | 1 + source/dnode/mnode/impl/src/mndDb.c | 7 ++++++ source/dnode/mnode/impl/src/mndVgroup.c | 1 + source/libs/parser/inc/parInt.h | 1 + source/libs/parser/src/parInsert.c | 24 +++++++++++++++++- source/libs/parser/src/parTranslater.c | 11 +++++++++ source/util/src/terror.c | 1 + tests/test/c/sdbDump.c | 1 + 16 files changed, 88 insertions(+), 1 deletion(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index faf4addb4b..68480e4ffd 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -779,6 +779,7 @@ typedef struct { int8_t cacheLastRow; int32_t numOfRetensions; SArray* pRetensions; + int8_t schemaless; } SDbCfgRsp; int32_t tSerializeSDbCfgRsp(void* buf, int32_t bufLen, const SDbCfgRsp* pRsp); @@ -1009,6 +1010,7 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; int32_t numOfRetensions; SArray* pRetensions; // SRetention + int8_t schemaless; // for tsma int8_t isTsma; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 1466bb7400..774a23ff4b 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -49,6 +49,7 @@ typedef struct SParseContext { const char* pUser; bool isSuperUser; bool async; + int8_t schemalessType; } SParseContext; int32_t qParseSql(SParseContext* pCxt, SQuery** pQuery); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 65cfe8de0b..b9eae6962c 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -673,6 +673,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) #define TSDB_CODE_SML_INVALID_PRECISION_TYPE TAOS_DEF_ERROR_CODE(0, 0x3001) #define TSDB_CODE_SML_INVALID_DATA TAOS_DEF_ERROR_CODE(0, 0x3002) +#define TSDB_CODE_SML_INVALID_DB_CONF TAOS_DEF_ERROR_CODE(0, 0x3003) #ifdef __cplusplus } diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d9f3351008..4c5bac71e4 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -149,6 +149,7 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; + int8_t schemalessType; } STscObj; typedef struct SResultColumn { diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 669b2bc97e..35e354e952 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -161,6 +161,7 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosAddRef(clientConnRefPool, pObj); + pObj->schemalessType = 0; tscDebug("connObj created, 0x%" PRIx64, pObj->id); return pObj; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index eb4c4cb59f..ba25f2f4ca 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -175,6 +175,7 @@ int32_t parseSql(SRequestObj* pRequest, bool topicQuery, SQuery** pQuery, SStmtC .pTransporter = pTscObj->pAppInfo->pTransporter, .pStmtCb = pStmtCb, .pUser = pTscObj->user, + .schemalessType = pTscObj->schemalessType, .isSuperUser = (0 == strcmp(pTscObj->user, TSDB_DEFAULT_USER))}; cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7d623072d6..29c7608126 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2318,6 +2318,28 @@ cleanup: return code; } +static int32_t isSchemalessDb(SSmlHandle* info){ + SName name; + tNameSetDbName(&name, info->taos->acctId, info->taos->db, strlen(info->taos->db)); + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + SDbCfgInfo pInfo = {0}; + SEpSet ep = getEpSet_s(&info->taos->pAppInfo->mgmtEp); + + int32_t code = catalogGetDBCfg(info->pCatalog, info->taos->pAppInfo->pTransporter, &ep, dbFname, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + info->pRequest->code = code; + smlBuildInvalidDataMsg(&info->msgBuf, "catalogGetDBCfg error, code:", tstrerror(code)); + return code; + } + if (!pInfo.schemaless){ + info->pRequest->code = TSDB_CODE_SML_INVALID_DB_CONF; + smlBuildInvalidDataMsg(&info->msgBuf, "can not insert into schemaless db:", dbFname); + return TSDB_CODE_SML_INVALID_DB_CONF; + } + return TSDB_CODE_SUCCESS; +} + /** * taos_schemaless_insert() parse and insert data points into database according to * different protocol. @@ -2351,6 +2373,17 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr return (TAOS_RES*)request; } + info->taos->schemalessType = 1; + if(request->pDb == NULL){ + request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + smlBuildInvalidDataMsg(&info->msgBuf, "Database not specified", NULL); + goto end; + } + + if(isSchemalessDb(info) != TSDB_CODE_SUCCESS){ + goto end; + } + if (!lines) { request->code = TSDB_CODE_SML_INVALID_DATA; smlBuildInvalidDataMsg(&info->msgBuf, "lines is null", NULL); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7615f7b070..b001953ee9 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2226,6 +2226,7 @@ int32_t tSerializeSDbCfgRsp(void *buf, int32_t bufLen, const SDbCfgRsp *pRsp) { if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } + if (tEncodeI8(&encoder, pRsp->schemaless) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2274,6 +2275,7 @@ int32_t tDeserializeSDbCfgRsp(void *buf, int32_t bufLen, SDbCfgRsp *pRsp) { return -1; } } + if (tDecodeI8(&decoder, &pRsp->schemaless) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9e036d7b25..6d839ea622 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -302,6 +302,7 @@ typedef struct { int8_t hashMethod; // default is 1 int32_t numOfRetensions; SArray* pRetensions; + int8_t schemaless; } SDbCfg; typedef struct { diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 95d3383ee1..cc884fb18b 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -115,6 +115,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT8(pRaw, dataPos, pRetension->freqUnit, _OVER) SDB_SET_INT8(pRaw, dataPos, pRetension->keepUnit, _OVER) } + SDB_SET_INT8(pRaw, dataPos, pDb->cfg.schemaless, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -192,6 +193,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { } } } + SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.schemaless, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); @@ -380,6 +382,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->replications < TSDB_MIN_DB_REPLICA || pCfg->replications > TSDB_MAX_DB_REPLICA) return -1; if (pCfg->replications != 1 && pCfg->replications != 3) return -1; if (pCfg->strict < TSDB_DB_STRICT_OFF || pCfg->strict > TSDB_DB_STRICT_ON) return -1; + if (pCfg->schemaless < TSDB_DB_SCHEMALESS_OFF || pCfg->schemaless > TSDB_DB_SCHEMALESS_ON) return -1; if (pCfg->cacheLastRow < TSDB_MIN_DB_CACHE_LAST_ROW || pCfg->cacheLastRow > TSDB_MAX_DB_CACHE_LAST_ROW) return -1; if (pCfg->hashMethod != 1) return -1; if (pCfg->replications > mndGetDnodeSize(pMnode)) { @@ -411,6 +414,8 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->strict < 0) pCfg->strict = TSDB_DEFAULT_DB_STRICT; if (pCfg->cacheLastRow < 0) pCfg->cacheLastRow = TSDB_DEFAULT_CACHE_LAST_ROW; if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; + if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; + } static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { @@ -521,6 +526,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .strict = pCreate->strict, .cacheLastRow = pCreate->cacheLastRow, .hashMethod = 1, + .schemaless = pCreate->schemaless, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; @@ -890,6 +896,7 @@ static int32_t mndProcessGetDbCfgReq(SRpcMsg *pReq) { cfgRsp.cacheLastRow = pDb->cfg.cacheLastRow; cfgRsp.numOfRetensions = pDb->cfg.numOfRetensions; cfgRsp.pRetensions = pDb->cfg.pRetensions; + cfgRsp.schemaless = pDb->cfg.schemaless; int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &cfgRsp); void *pRsp = rpcMallocCont(contLen); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e05b38a7c0..a00841f911 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -218,6 +218,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashMethod = pDb->cfg.hashMethod; createReq.numOfRetensions = pDb->cfg.numOfRetensions; createReq.pRetensions = pDb->cfg.pRetensions; + createReq.schemaless = pDb->cfg.schemaless; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index 184ed7d8b2..c74567f1a0 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -31,6 +31,7 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); +int32_t isNotSchemalessDb(SParseContext* pContext); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 58a6d1483f..c78733c3c4 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1204,6 +1204,23 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } +int32_t isNotSchemalessDb(SParseContext* pContext){ + SName name; + tNameSetDbName(&name, pContext->acctId, pContext->db, strlen(pContext->db)); + char dbFname[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, dbFname); + SDbCfgInfo pInfo = {0}; + int32_t code = catalogGetDBCfg(pContext->pCatalog, pContext->pTransporter, &pContext->mgmtEpSet, dbFname, &pInfo); + if (code != TSDB_CODE_SUCCESS) { + parserError("catalogGetDBCfg error, code:%s, dbFName:%s", tstrerror(code), dbFname); + return code; + } + if (pInfo.schemaless){ + parserError("can not insert into schemaless db:%s", dbFname); + return TSDB_CODE_SML_INVALID_DB_CONF; + } + return TSDB_CODE_SUCCESS; +} // INSERT INTO // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] @@ -1252,6 +1269,11 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { (*pQuery)->pRoot = (SNode*)context.pOutput; } + int32_t code = isNotSchemalessDb(pContext); + if(code != TSDB_CODE_SUCCESS){ + return code; + } + if (NULL == (*pQuery)->pTableList) { (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); if (NULL == (*pQuery)->pTableList) { @@ -1261,7 +1283,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { context.pOutput->payloadType = PAYLOAD_TYPE_KV; - int32_t code = skipInsertInto(&context); + code = skipInsertInto(&context); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(&context); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d84b005f7f..31363401fe 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2619,6 +2619,11 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt if (TSDB_CODE_SUCCESS == code) { code = checkTableSchema(pCxt, pStmt); } + if (TSDB_CODE_SUCCESS == code) { + if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ + code = TSDB_CODE_SML_INVALID_DB_CONF; + } + } return code; } @@ -4350,6 +4355,9 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { + if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ + return TSDB_CODE_SML_INVALID_DB_CONF; + } SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -4766,6 +4774,9 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p } static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { + if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ + return TSDB_CODE_SML_INVALID_DB_CONF; + } SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; STableMeta* pTableMeta = NULL; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 178d6e8d2b..0ffcebca5b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -469,6 +469,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output ty TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE, "Invalid line protocol type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE, "Invalid timestamp precision type") TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DATA, "Invalid data type") +TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_DB_CONF, "Invalid schemaless db config") #ifdef TAOS_ERROR_C }; diff --git a/tests/test/c/sdbDump.c b/tests/test/c/sdbDump.c index 8be2822c0a..6a7255b33c 100644 --- a/tests/test/c/sdbDump.c +++ b/tests/test/c/sdbDump.c @@ -86,6 +86,7 @@ void dumpDb(SSdb *pSdb, SJson *json) { tjsonAddIntegerToObject(item, "cacheLastRow", pObj->cfg.cacheLastRow); tjsonAddIntegerToObject(item, "hashMethod", pObj->cfg.hashMethod); tjsonAddIntegerToObject(item, "numOfRetensions", pObj->cfg.numOfRetensions); + tjsonAddIntegerToObject(item, "schemaless", pObj->cfg.schemaless); sdbRelease(pSdb, pObj); } From 70927458e53f0d4c118eb8b5de82d85d371f7135 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 30 May 2022 20:10:30 +0800 Subject: [PATCH 04/47] refactor(query): do some internal refactor. --- include/libs/function/function.h | 30 -- source/common/src/tdatablock.c | 11 +- source/libs/executor/inc/executil.h | 12 +- source/libs/executor/src/executil.c | 14 +- source/libs/executor/src/executorimpl.c | 58 ++-- source/libs/executor/src/scanoperator.c | 4 +- source/libs/executor/src/timewindowoperator.c | 315 ++++++++++-------- source/libs/function/src/builtinsimpl.c | 135 ++------ source/libs/function/src/texpr.c | 14 - 9 files changed, 253 insertions(+), 340 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 01f347fd05..e8cb363e08 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -61,9 +61,6 @@ typedef struct SFileBlockInfo { #define TSDB_BLOCK_DIST_STEP_ROWS 8 #define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results -#define FUNCTION_TYPE_SCALAR 1 -#define FUNCTION_TYPE_AGG 2 - #define TOP_BOTTOM_QUERY_LIMIT 100 #define FUNCTIONS_NAME_MAX_LENGTH 16 @@ -165,9 +162,6 @@ enum { typedef struct tExprNode { int32_t nodeType; union { - SSchema *pSchema;// column node - struct SVariant *pVal; // value node - struct {// function node char functionName[FUNCTIONS_NAME_MAX_LENGTH]; // todo refactor int32_t functionId; @@ -210,47 +204,23 @@ struct SScalarParam { int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength, bool isSuperTable); -bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId); - void resetResultRowEntryResult(SqlFunctionCtx* pCtx, int32_t num); void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock); bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); -/////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -// fill api -struct SFillInfo; -struct SFillColInfo; - typedef struct SPoint { int64_t key; void * val; } SPoint; -//void taosFillSetStartInfo(struct SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); -//void taosResetFillInfo(struct SFillInfo* pFillInfo, TSKEY startTimestamp); -//void taosFillSetInputDataBlock(struct SFillInfo* pFillInfo, const struct SSDataBlock* pInput); -//struct SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfOutput, const SValueNode* val); -//bool taosFillHasMoreResults(struct SFillInfo* pFillInfo); -// -//struct SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols, -// SInterval* pInterval, int32_t fillType, -// struct SFillColInfo* pCol, const char* id); -// -//void* taosDestroyFillInfo(struct SFillInfo *pFillInfo); -//int64_t taosFillResultDataBlock(struct SFillInfo* pFillInfo, void** output, int32_t capacity); -//int64_t getFillInfoStart(struct SFillInfo *pFillInfo); - int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // udf api struct SUdfInfo; -void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); -void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); - /** * create udfd proxy, called once in process that call doSetupUdf/callUdfxxx/doTeardownUdf * @return error code diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 49c438d8cf..1bd35bdefe 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -354,14 +354,19 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) return -1; } - int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; + int32_t index = (tsColumnIndex == -1) ? 0 : tsColumnIndex; + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, index); if (pColInfoData->info.type != TSDB_DATA_TYPE_TIMESTAMP) { return 0; } - pDataBlock->info.window.skey = *(TSKEY*)colDataGetData(pColInfoData, 0); - pDataBlock->info.window.ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); + TSKEY skey = *(TSKEY*)colDataGetData(pColInfoData, 0); + TSKEY ekey = *(TSKEY*)colDataGetData(pColInfoData, (pDataBlock->info.rows - 1)); + + pDataBlock->info.window.skey = TMIN(skey, ekey); + pDataBlock->info.window.ekey = TMAX(skey, ekey); + return 0; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index dcd4e8f918..b8975854c9 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -80,11 +80,10 @@ typedef struct SResultRowInfo { struct SqlFunctionCtx; -size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); +size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); int32_t initResultRowInfo(SResultRowInfo* pResultRowInfo, int32_t size); void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -int32_t numOfClosedResultRows(SResultRowInfo* pResultRowInfo); void closeAllResultRows(SResultRowInfo* pResultRowInfo); void initResultRow(SResultRow *pResultRow); @@ -93,15 +92,6 @@ bool isResultRowClosed(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultCell(const SResultRow* pRow, int32_t index, const int32_t* offset); -static FORCE_INLINE SResultRow *getResultRow(SDiskbasedBuf* pBuf, SResultRowInfo *pResultRowInfo, int32_t slot) { - ASSERT(pResultRowInfo != NULL && slot >= 0 && slot < pResultRowInfo->size); - SResultRowPosition* pos = &pResultRowInfo->pPosition[slot]; - - SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); - return pRow; -} - static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index b8915ae218..3f3fa9836f 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -101,20 +101,8 @@ void resetResultRowInfo(STaskRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow pResultRowInfo->size = 0; } -int32_t numOfClosedResultRows(SResultRowInfo *pResultRowInfo) { - int32_t i = 0; -// while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { -// ++i; -// } - - return i; -} - void closeAllResultRows(SResultRowInfo *pResultRowInfo) { - assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); - - for (int32_t i = 0; i < pResultRowInfo->size; ++i) { - } +// do nothing } bool isResultRowClosed(SResultRow* pRow) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9c5f93613a..23493ae647 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -345,6 +345,8 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // In case of group by column query, the required SResultRow object must be existInCurrentResusltRowInfo in the // pResultRowInfo object. if (p1 != NULL) { + + // todo pResult = getResultRowByPos(pResultBuf, p1); ASSERT(pResult->pageId == p1->pageId && pResult->offset == p1->offset); } @@ -353,11 +355,9 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR // 1. close current opened time window if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId && pResult->offset != pResultRowInfo->cur.offset))) { - // todo extract function SResultRowPosition pos = pResultRowInfo->cur; - SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)pPage + pos.offset); - closeResultRow(pRow); + SFilePage* + pPage = getBufPage(pResultBuf, pos.pageId); releaseBufPage(pResultBuf, pPage); } @@ -552,11 +552,13 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } + void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset, int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) { for (int32_t k = 0; k < numOfOutput; ++k) { // keep it temporarily + // todo no need this?? bool hasAgg = pCtx[k].input.colDataAggIsSet; int32_t numOfRows = pCtx[k].input.numOfRows; int32_t startOffset = pCtx[k].input.startRowIndex; @@ -576,7 +578,8 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow if (fmIsWindowPseudoColumnFunc(pCtx[k].functionId)) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[k]); - char* p = GET_ROWCELL_INTERBUF(pEntryInfo); + + char* p = GET_ROWCELL_INTERBUF(pEntryInfo); SColumnInfoData idata = {0}; idata.info.type = TSDB_DATA_TYPE_BIGINT; @@ -587,22 +590,23 @@ void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; pCtx[k].sfp.process(&tw, 1, &out); pEntryInfo->numOfRes = 1; - continue; - } - int32_t code = TSDB_CODE_SUCCESS; - if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { - code = pCtx[k].fpSet.process(&pCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); - taskInfo->code = code; - longjmp(taskInfo->env, code); - } - } + } else { + int32_t code = TSDB_CODE_SUCCESS; + if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) { + code = pCtx[k].fpSet.process(&pCtx[k]); - // restore it - pCtx[k].input.colDataAggIsSet = hasAgg; - pCtx[k].input.startRowIndex = startOffset; - pCtx[k].input.numOfRows = numOfRows; + if (code != TSDB_CODE_SUCCESS) { + qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code)); + taskInfo->code = code; + longjmp(taskInfo->env, code); + } + } + + // restore it + pCtx[k].input.colDataAggIsSet = hasAgg; + pCtx[k].input.startRowIndex = startOffset; + pCtx[k].input.numOfRows = numOfRows; + } } } @@ -741,12 +745,14 @@ static int32_t doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunct for (int32_t k = 0; k < pOperator->numOfExprs; ++k) { if (functionNeedToExecute(&pCtx[k])) { // todo add a dummy funtion to avoid process check - if (pCtx[k].fpSet.process != NULL) { - int32_t code = pCtx[k].fpSet.process(&pCtx[k]); - if (code != TSDB_CODE_SUCCESS) { - qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); - return code; - } + if (pCtx[k].fpSet.process == NULL) { + continue; + } + + int32_t code = pCtx[k].fpSet.process(&pCtx[k]); + if (code != TSDB_CODE_SUCCESS) { + qError("%s aggregate function error happens, code: %s", GET_TASKID(pOperator->pTaskInfo), tstrerror(code)); + return code; } } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 11b50a38bc..c84883823d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -534,8 +534,8 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, pInfo->pPseudoCtx = createSqlFunctionCtx(pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, &pInfo->rowCellInfoOffset); } -// pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; - pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose + pInfo->scanInfo = (SScanInfo){.numOfAsc = pTableScanNode->scanSeq[0], .numOfDesc = pTableScanNode->scanSeq[1]}; +// pInfo->scanInfo = (SScanInfo){.numOfAsc = 0, .numOfDesc = 1}; // for debug purpose pInfo->readHandle = *readHandle; pInfo->interval = extractIntervalInfo(pTableScanNode); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index a1a7aecf5f..6595456c17 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -12,6 +12,11 @@ typedef enum SResultTsInterpType { static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator); static SSDataBlock* doStreamSessionWindowAgg(SOperatorInfo* pOperator); +static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); + +static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult); +static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult); + /* * There are two cases to handle: * @@ -135,8 +140,10 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo // set time window for current result pResultRow->win = (*win); + *pResult = pResultRow; setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + return TSDB_CODE_SUCCESS; } @@ -164,38 +171,38 @@ static void doKeepNewWindowStartInfo(SWindowRowsSup* pRowSup, const int64_t* tsL static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, int16_t order, int64_t* pData) { - int32_t forwardStep = 0; + int32_t forwardRows = 0; if (order == TSDB_ORDER_ASC) { int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); if (end >= 0) { - forwardStep = end; + forwardRows = end; if (pData[end + pos] == ekey) { - forwardStep += 1; + forwardRows += 1; } } } else { int32_t end = searchFn((char*)&pData[pos], numOfRows - pos, ekey, order); if (end >= 0) { - forwardStep = end; + forwardRows = end; if (pData[end + pos] == ekey) { - forwardStep += 1; + forwardRows += 1; } } // int32_t end = searchFn((char*)pData, pos + 1, ekey, order); // if (end >= 0) { -// forwardStep = pos - end; +// forwardRows = pos - end; // // if (pData[end] == ekey) { -// forwardStep += 1; +// forwardRows += 1; // } // } } - assert(forwardStep >= 0); - return forwardStep; + assert(forwardRows >= 0); + return forwardRows; } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -431,10 +438,7 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo *pInfo, SqlFunctionCtx* pCtx, int32_t numOfExprs, int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols, STimeWindow* win) { - int32_t numOfRows = pBlock->info.rows; - bool ascQuery = (pInfo->order == TSDB_ORDER_ASC); - int32_t step = 1; // GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); TSKEY curTs = tsCols[pos]; @@ -449,14 +453,15 @@ static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo *pInfo, S return true; } - if (lastTs == INT64_MIN && ((pos == 0 && ascQuery) || (pos == ( - 1) && !ascQuery))) { + // it is the first time window, no need to do interpolation + if (pTsKey->isNull && pos == 0) { setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_START_INTERP); - return true; + } else { + TSKEY prevTs = ((pos == 0) ? lastTs : tsCols[pos - 1]); + doTimeWindowInterpolation(pInfo, numOfExprs, pBlock->pDataBlock, prevTs, pos - 1, curTs, pos, key, + RESULT_ROW_START_INTERP); } - TSKEY prevTs = ((pos == 0 && ascQuery) || (pos == (numOfRows - 1) && !ascQuery)) ? lastTs : tsCols[pos - step]; - - doTimeWindowInterpolation(pInfo, numOfExprs, pBlock->pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); return true; } @@ -473,14 +478,13 @@ static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo *pInfo, Sql return false; } - // there is actual end point of current time window, no interpolation need + // there is actual end point of current time window, no interpolation needs if (key == actualEndKey) { setNotInterpoWindowKey(pCtx, numOfExprs, RESULT_ROW_END_INTERP); return true; } - int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); - int32_t nextRowIndex = endRowIndex + step; + int32_t nextRowIndex = endRowIndex + 1; assert(nextRowIndex >= 0); TSKEY nextKey = tsCols[nextRowIndex]; @@ -575,27 +579,23 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } static void doWindowBorderInterpolation(SIntervalAggOperatorInfo *pInfo, SSDataBlock* pBlock, int32_t numOfExprs, SqlFunctionCtx* pCtx, - SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) { + SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardRows) { if (!pInfo->timeWindowInterpo) { return; } - assert(pBlock != NULL); - int32_t step = GET_FORWARD_DIRECTION_FACTOR(pInfo->order); - + ASSERT(pBlock != NULL); if (pBlock->pDataBlock == NULL) { // tscError("pBlock->pDataBlock == NULL"); return; } - // todo set the correct primary timestamp column index - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* tsCols = (TSKEY*)(pColInfo->pData); bool done = isResultRowInterpolated(pResult, RESULT_ROW_START_INTERP); if (!done) { // it is not interpolated, now start to generated the interpolated value - int32_t startRowIndex = startPos; - bool interp = setTimeWindowInterpolationStartTs(pInfo, pCtx, numOfExprs, startRowIndex, pBlock, tsCols, win); + bool interp = setTimeWindowInterpolationStartTs(pInfo, pCtx, numOfExprs, startPos, pBlock, tsCols, win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } @@ -611,7 +611,7 @@ static void doWindowBorderInterpolation(SIntervalAggOperatorInfo *pInfo, SSDataB // interpolation query does not generate the time window end interpolation done = isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP); if (!done) { - int32_t endRowIndex = startPos + (forwardStep - 1) * step; + int32_t endRowIndex = startPos + forwardRows - 1; TSKEY endKey = (pInfo->order == TSDB_ORDER_ASC) ? pBlock->info.window.ekey : pBlock->info.window.skey; bool interp = @@ -654,43 +654,87 @@ static void saveDataBlockLastRow(SArray* pPrevKeys, const SSDataBlock* pBlock, S } } -static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - uint64_t tableGroupId) { +static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t numOfExprs, SResultRowInfo* pResultRowInfo, + SSDataBlock* pBlock, int32_t scanFlag, int64_t* tsCols, SResultRowPosition* p) { + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - int32_t numOfOutput = pOperatorInfo->numOfExprs; + int32_t startPos = 0; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + uint64_t groupId = pBlock->info.groupId; + + SResultRow* pResult = NULL; + + while (1) { + SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); + + SResultRowPosition* p1 = (SResultRowPosition*)pn->data; + if (p->pageId == p1->pageId && p->offset == p1->offset) { + break; + } + + SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1); + ASSERT(pr->offset == p1->offset && pr->pageId == p1->pageId); + + if (pr->closed) { + ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); + tdListPopHead(pResultRowInfo->openWindow); + continue; + } + + STimeWindow w = pr->win; + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pInfo->binfo.pCtx, + numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + SGroupKeys *pTsKey = taosArrayGet(pInfo->pPrevValues, 0); + int64_t prevTs = *(int64_t*) pTsKey->pData; + doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, + w.ekey, RESULT_ROW_END_INTERP); + + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pInfo->binfo.pCtx, numOfExprs, RESULT_ROW_START_INTERP); + + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols, + pBlock->info.rows, numOfExprs, pInfo->order); + + if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { + closeResultRow(pr); + tdListPopHead(pResultRowInfo->openWindow); + } else { // the remains are can not be closed yet. + break; + } + } +} + +static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, + int32_t scanFlag) { + SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SArray* pUpdated = NULL; if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { pUpdated = taosArrayInit(4, POINTER_BYTES); } - int32_t step = 1; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - bool ascScan = (pInfo->order == TSDB_ORDER_ASC); - - // int32_t prevIndex = pResultRowInfo->curPos; - - TSKEY* tsCols = NULL; - if (pBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - - if (tsCols != NULL) { - blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); - } - } - - int32_t startPos = 0; - TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols, pBlock->info.rows, ascScan); + int32_t startPos = 0; + int32_t numOfOutput = pOperatorInfo->numOfExprs; + int64_t *tsCols = extractTsCol(pBlock, pInfo); + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = (pInfo->order == TSDB_ORDER_ASC); + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols, pBlock->info.rows, ascScan); + SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); - bool masterScan = true; - SResultRow* pResult = NULL; - int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -705,84 +749,35 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe taosArrayPush(pUpdated, &pos); } - int32_t forwardStep = 0; TSKEY ekey = ascScan? win.ekey:win.skey; - forwardStep = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); - ASSERT(forwardStep > 0); + int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); + ASSERT(forwardRows > 0); // prev time window not interpolation yet. - SResultRowPosition pos = {0}; if (pInfo->timeWindowInterpo) { - pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); + SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult); + doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); - SResultRowPosition* px = (SResultRowPosition*)pn->data; - - // do nothing - if (pn != NULL && px->pageId == pos.pageId && px->offset == pos.offset) { - - } else { - tdListAppend(pResultRowInfo->openWindow, &pos); - } - } - - if (pInfo->timeWindowInterpo) { - while (1) { - SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); - SResultRowPosition* p1 = (SResultRowPosition*)pn->data; - if (p1->pageId == pos.pageId && p1->offset == pos.offset) { - break; - } - - SResultRow* pr = getResultRowByPos(pInfo->aggSup.pResultBuf, p1); - if (pr->closed) { - ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); - tdListPopHead(pResultRowInfo->openWindow); - continue; - } - - STimeWindow w = pr->win; - ret = setTimeWindowOutputBuf(pResultRowInfo, &w, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, - numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - ASSERT(!isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - - SGroupKeys *pTsKey = taosArrayGet(pInfo->pPrevValues, 0); - int64_t prevTs = *(int64_t*) pTsKey->pData; - doTimeWindowInterpolation(pInfo, numOfOutput, pBlock->pDataBlock, prevTs, -1, tsCols[startPos], startPos, - w.ekey, RESULT_ROW_END_INTERP); - - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - setNotInterpoWindowKey(pInfo->binfo.pCtx, pOperatorInfo->numOfExprs, RESULT_ROW_START_INTERP); - - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - tdListPopHead(pResultRowInfo->openWindow); - } - - // todo keep current outputbuffer info to avoid repeatly set the buffer info // restore current time window - ret = setTimeWindowOutputBuf(pResultRowInfo, &win, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // window start key interpolation - doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardStep); + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &win, startPos, forwardRows); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, pInfo->order); + + doCloseWindow(pResultRowInfo, pInfo, pResult); STimeWindow nextWin = win; while (1) { - int32_t prevEndPos = (forwardStep - 1) * step + startPos; + int32_t prevEndPos = forwardRows - 1 + startPos; startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { break; @@ -790,7 +785,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe // null data, failed to allocate more memory buffer int32_t code = - setTimeWindowOutputBuf(pResultRowInfo, &nextWin, masterScan, &pResult, tableGroupId, pInfo->binfo.pCtx, + setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pInfo->binfo.pCtx, numOfOutput, pInfo->binfo.rowCellInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -806,24 +801,61 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } ekey = ascScan? nextWin.ekey:nextWin.skey; - forwardStep = + forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->order); // window start(end) key interpolation - doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep); + doWindowBorderInterpolation(pInfo, pBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, - pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, pInfo->order); + doCloseWindow(pResultRowInfo, pInfo, pResult); } if (pInfo->timeWindowInterpo) { - int32_t rowIndex = ascScan? (pBlock->info.rows - 1) : 0; saveDataBlockLastRow(pInfo->pPrevValues, pBlock, pInfo->pInterpCols); } return pUpdated; - // updateResultRowInfoActiveIndex(pResultRowInfo, &pInfo->win, pRuntimeEnv->current->lastKey, true, false); +} + +void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult) { + // current result is done in computing final results. + if (pInfo->timeWindowInterpo && isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { + closeResultRow(pResult); + tdListPopHead(pResultRowInfo->openWindow); + } +} + +SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult) { + SResultRowPosition pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); + if (pn == NULL) { + tdListAppend(pResultRowInfo->openWindow, &pos); + return pos; + } + + SResultRowPosition* px = (SResultRowPosition*)pn->data; + if (px->pageId != pos.pageId || px->offset != pos.offset) { + tdListAppend(pResultRowInfo->openWindow, &pos); + } + + return pos; +} + +int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo) { + TSKEY* tsCols = NULL; + if (pBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + + if (tsCols != NULL) { + blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); + } + } + + return tsCols; } static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { @@ -852,7 +884,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag); #if 0 // test for encode/decode result info if(pOperator->encodeResultRow){ @@ -983,8 +1015,9 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { } SStateWindowOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SOptrBasicInfo* pBInfo = &pInfo->binfo; + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); @@ -1165,7 +1198,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { break; } - // The timewindows that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the // caller. Note that all the time window are not close till now. // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pInfo->order, MAIN_SCAN, true); @@ -1180,7 +1213,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } - pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); + pUpdated = hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN); } finalizeUpdatedResult(pOperator->numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); @@ -1230,26 +1263,26 @@ static bool allInvertible(SqlFunctionCtx* pFCtx, int32_t numOfCols) { return true; } -static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SArray** pInterpList, SArray** pPrevCols) { +static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SIntervalAggOperatorInfo* pInfo) { // the primary timestamp column bool needed = false; - *pInterpList = taosArrayInit(4, sizeof(SColumn)); - *pPrevCols = taosArrayInit(4, sizeof(SGroupKeys)); + pInfo->pInterpCols = taosArrayInit(4, sizeof(SColumn)); + pInfo->pPrevValues = taosArrayInit(4, sizeof(SGroupKeys)); { // ts column SColumn c = {0}; c.colId = 1; - c.slotId = 0; + c.slotId = pInfo->primaryTsIndex; c.type = TSDB_DATA_TYPE_TIMESTAMP; c.bytes = sizeof(int64_t); - taosArrayPush(*pInterpList, &c); + taosArrayPush(pInfo->pInterpCols, &c); SGroupKeys key = {0}; key.bytes = c.bytes; key.type = c.type; - key.isNull = false; + key.isNull = true; // to denote no value is assigned yet key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(*pPrevCols, &key); + taosArrayPush(pInfo->pPrevValues, &key); } for(int32_t i = 0; i < numOfCols; ++i) { @@ -1259,7 +1292,7 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SArr SFunctParam* pParam = &pExpr->base.pParam[0]; SColumn c = *pParam->pCol; - taosArrayPush(*pInterpList, &c); + taosArrayPush(pInfo->pInterpCols, &c); needed = true; SGroupKeys key = {0}; @@ -1267,7 +1300,7 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SArr key.type = c.type; key.isNull = false; key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(*pPrevCols, &key); + taosArrayPush(pInfo->pPrevValues, &key); } } @@ -1302,7 +1335,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->invertible = allInvertible(pInfo->binfo.pCtx, numOfCols); pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API - pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, &pInfo->pInterpCols, &pInfo->pPrevValues); + pInfo->timeWindowInterpo = timeWindowinterpNeeded(pInfo->binfo.pCtx, numOfCols, pInfo); if (pInfo->timeWindowInterpo) { pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); } @@ -1780,7 +1813,7 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB bool ascScan = true; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; - int32_t forwardStep = 0; + int32_t forwardRows = 0; if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); @@ -1805,15 +1838,15 @@ static SArray* doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataB pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; *(int64_t*)pos->key = pResult->win.skey; taosArrayPush(pUpdated, &pos); - forwardStep = + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); // window start(end) key interpolation // disable it temporarily -// doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep); +// doWindowBorderInterpolation(pInfo, pSDataBlock, numOfOutput, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols, + doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); - int32_t prevEndPos = (forwardStep - 1) * step + startPos; + int32_t prevEndPos = (forwardRows - 1) * step + startPos; startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); if (startPos < 0) { break; @@ -2321,7 +2354,7 @@ static void doStreamSessionWindowAggImpl(SOperatorInfo* pOperator, longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } // window start(end) key interpolation - // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardStep, + // doWindowBorderInterpolation(pOperatorInfo, pSDataBlock, pInfo->binfo.pCtx, pResult, &nextWin, startPos, forwardRows, // pInfo->order, false); int32_t winNum = getNumCompactWindow(pAggSup->pResultRows, winIndex, gap); if (winNum > 0) { diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 78e057f7a8..6694a8cdb3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3946,7 +3946,6 @@ int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { typedef struct STwaInfo { double dOutput; - int8_t hasResult; // flag to denote has value SPoint1 p; STimeWindow win; } STwaInfo; @@ -3977,28 +3976,25 @@ static double twa_get_area(SPoint1 s, SPoint1 e) { return val; } +#define INIT_INTP_POINT(_p, _k, _v) \ + do { \ + (_p).key = (_k); \ + (_p).val = (_v); \ + } while (0) + int32_t twaFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - int32_t numOfElems = 0; - TSKEY* tsList = (int64_t*)pInput->pPTS->pData; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + SPoint1 *last = &pInfo->p; + int32_t numOfElems = 0; - SPoint1* last = &pInfo->p; - - int32_t step = 1;//GET_FORWARD_DIRECTION_FACTOR(pCtx->order); int32_t i = pInput->startRowIndex; - for (i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += step) { - if (!colDataIsNull_f(pInputCol->nullbitmap, i)) { - break; - } - } - if (pCtx->start.key != INT64_MIN) { ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); @@ -4009,37 +4005,30 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { GET_TYPED_DATA(last->val, double, pInputCol->info.type, colDataGetData(pInputCol, i)); pInfo->dOutput += twa_get_area(pCtx->start, *last); - - pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = pCtx->start.key; numOfElems++; - i += step; + i += 1; } else if (pInfo->p.key == INT64_MIN) { last->key = tsList[i]; GET_TYPED_DATA(last->val, double, pInputCol->info.type, colDataGetData(pInputCol, i)); - pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = last->key; numOfElems++; - i += step; + i += 1; } + SPoint1 st = {0}; + // calculate the value of switch(pInputCol->info.type) { case TSDB_DATA_TYPE_TINYINT: { int8_t *val = (int8_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4048,18 +4037,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { case TSDB_DATA_TYPE_SMALLINT: { int16_t *val = (int16_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4067,18 +4050,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_INT: { int32_t *val = (int32_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4086,18 +4063,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_BIGINT: { int64_t *val = (int64_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4105,18 +4076,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_FLOAT: { float *val = (float*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double)val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4124,18 +4089,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_DOUBLE: { double *val = (double*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4143,18 +4102,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_UTINYINT: { uint8_t *val = (uint8_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4162,18 +4115,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_USMALLINT: { uint16_t *val = (uint16_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4181,18 +4128,12 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_UINT: { uint32_t *val = (uint32_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -4200,25 +4141,19 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { } case TSDB_DATA_TYPE_UBIGINT: { uint64_t *val = (uint64_t*) colDataGetData(pInputCol, 0); - for (; i < pInput->numOfRows + pInput->startRowIndex; i += step) { + for (; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_f(pInputCol->nullbitmap, i)) { continue; } -#ifndef _TD_NINGSI_60 - SPoint1 st = {.key = tsList[i], .val = (double) val[i]}; -#else - SPoint1 st; - st.key = tsList[i]; - st.val = (double) val[i]; -#endif + INIT_INTP_POINT(st, tsList[i], val[i]); pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } break; } - default: assert(0); + default: ASSERT(0); } // the last interpolated time window value @@ -4229,7 +4164,7 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { pInfo->win.ekey = pInfo->p.key; - SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); + SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -4250,7 +4185,7 @@ int32_t twaFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->hasResult != DATA_SET_FLAG) { + if (pResInfo->numOfRes == 0) { pResInfo->isNullRes = 1; } else { // assert(pInfo->win.ekey == pInfo->p.key && pInfo->hasResult == pResInfo->hasResult); diff --git a/source/libs/function/src/texpr.c b/source/libs/function/src/texpr.c index b91af2d157..703b19ced7 100644 --- a/source/libs/function/src/texpr.c +++ b/source/libs/function/src/texpr.c @@ -36,12 +36,7 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) { if (pNode->nodeType == TEXPR_BINARYEXPR_NODE || pNode->nodeType == TEXPR_UNARYEXPR_NODE) { doExprTreeDestroy(&pNode, fp); - } else if (pNode->nodeType == TEXPR_VALUE_NODE) { - taosVariantDestroy(pNode->pVal); - } else if (pNode->nodeType == TEXPR_COL_NODE) { - taosMemoryFreeClear(pNode->pSchema); } - taosMemoryFree(pNode); } @@ -49,15 +44,6 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) { if (*pExpr == NULL) { return; } - - int32_t type = (*pExpr)->nodeType; - if (type == TEXPR_VALUE_NODE) { - taosVariantDestroy((*pExpr)->pVal); - taosMemoryFree((*pExpr)->pVal); - } else if (type == TEXPR_COL_NODE) { - taosMemoryFree((*pExpr)->pSchema); - } - taosMemoryFree(*pExpr); *pExpr = NULL; } From bb2146cd75d41eb6fea407684c67c72d4412eea0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 30 May 2022 22:35:01 +0800 Subject: [PATCH 05/47] refactor(query): do some internal refactor. --- include/common/tdatablock.h | 51 +------------ source/client/test/clientTests.cpp | 40 +++++------ source/common/src/tdatablock.c | 95 +++++++++++++++++++++++++ source/libs/executor/src/executorimpl.c | 44 +----------- 4 files changed, 120 insertions(+), 110 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 9b71e8c454..66b81efc5b 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -227,6 +227,9 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); + void blockDebugShowData(const SArray* dataBlocks); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, @@ -246,54 +249,6 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32 colSize + COMP_OVERFLOW_BYTES, compressed, NULL, 0); } -static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, - int8_t needCompress) { - int32_t* actualLen = (int32_t*)data; - data += sizeof(int32_t); - - uint64_t* groupId = (uint64_t*)data; - data += sizeof(uint64_t); - - int32_t* colSizes = (int32_t*)data; - data += numOfCols * sizeof(int32_t); - - *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); - - int32_t numOfRows = pBlock->info.rows; - for (int32_t col = 0; col < numOfCols; ++col) { - SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col); - - // copy the null bitmap - if (IS_VAR_DATA_TYPE(pColRes->info.type)) { - size_t metaSize = numOfRows * sizeof(int32_t); - memcpy(data, pColRes->varmeta.offset, metaSize); - data += metaSize; - (*dataLen) += metaSize; - } else { - int32_t len = BitmapLen(numOfRows); - memcpy(data, pColRes->nullbitmap, len); - data += len; - (*dataLen) += len; - } - - if (needCompress) { - colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); - data += colSizes[col]; - (*dataLen) += colSizes[col]; - } else { - colSizes[col] = colDataGetLength(pColRes, numOfRows); - (*dataLen) += colSizes[col]; - memmove(data, pColRes->pData, colSizes[col]); - data += colSizes[col]; - } - - colSizes[col] = htonl(colSizes[col]); - } - - *actualLen = *dataLen; - *groupId = pBlock->info.groupId; -} - #ifdef __cplusplus } #endif diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b5b6ea65e0..914e5aefc2 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -567,6 +567,7 @@ TEST(testCase, insert_test) { taos_free_result(pRes); taos_close(pConn); } +#endif TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -625,23 +626,23 @@ TEST(testCase, projection_query_tables) { printf("start to insert next table\n"); - for(int32_t i = 0; i < 1000000; i += 20) { - char sql[1024] = {0}; - sprintf(sql, - "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" - "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", - i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, - i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, - i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); - TAOS_RES* p = taos_query(pConn, sql); - if (taos_errno(p) != 0) { - printf("failed to insert data, reason:%s\n", taos_errstr(p)); - } - - taos_free_result(p); - } +// for(int32_t i = 0; i < 1000000; i += 20) { +// char sql[1024] = {0}; +// sprintf(sql, +// "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" +// "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" +// "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" +// "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", +// i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, +// i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, +// i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); +// TAOS_RES* p = taos_query(pConn, sql); +// if (taos_errno(p) != 0) { +// printf("failed to insert data, reason:%s\n", taos_errstr(p)); +// } +// +// taos_free_result(p); +// } // pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { @@ -663,7 +664,7 @@ TEST(testCase, projection_query_tables) { // taos_free_result(pRes); taos_close(pConn); } - +#if 0 TEST(testCase, projection_query_stables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); @@ -692,8 +693,6 @@ TEST(testCase, projection_query_stables) { taos_close(pConn); } -#endif - TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); @@ -734,5 +733,6 @@ TEST(testCase, agg_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif #pragma GCC diagnostic pop diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1bd35bdefe..199fc35c72 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1764,3 +1764,98 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo ret->length = htonl(ret->length); return ret; } + +void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { + int32_t* actualLen = (int32_t*)data; + data += sizeof(int32_t); + + uint64_t* groupId = (uint64_t*)data; + data += sizeof(uint64_t); + + int32_t* colSizes = (int32_t*)data; + data += numOfCols * sizeof(int32_t); + + *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); + + int32_t numOfRows = pBlock->info.rows; + for (int32_t col = 0; col < numOfCols; ++col) { + SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, col); + + // copy the null bitmap + if (IS_VAR_DATA_TYPE(pColRes->info.type)) { + size_t metaSize = numOfRows * sizeof(int32_t); + memcpy(data, pColRes->varmeta.offset, metaSize); + data += metaSize; + (*dataLen) += metaSize; + } else { + int32_t len = BitmapLen(numOfRows); + memcpy(data, pColRes->nullbitmap, len); + data += len; + (*dataLen) += len; + } + + if (needCompress) { + colSizes[col] = blockCompressColData(pColRes, numOfRows, data, needCompress); + data += colSizes[col]; + (*dataLen) += colSizes[col]; + } else { + colSizes[col] = colDataGetLength(pColRes, numOfRows); + (*dataLen) += colSizes[col]; + memmove(data, pColRes->pData, colSizes[col]); + data += colSizes[col]; + } + + colSizes[col] = htonl(colSizes[col]); + } + + *actualLen = *dataLen; + *groupId = pBlock->info.groupId; +} + +const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { + blockDataEnsureCapacity(pBlock, numOfRows); + const char* pStart = pData; + + int32_t dataLen = *(int32_t*)pStart; + pStart += sizeof(int32_t); + + pBlock->info.groupId = *(uint64_t*)pStart; + pStart += sizeof(uint64_t); + + int32_t* colLen = (int32_t*)pStart; + pStart += sizeof(int32_t) * numOfCols; + + for (int32_t i = 0; i < numOfCols; ++i) { + colLen[i] = htonl(colLen[i]); + ASSERT(colLen[i] >= 0); + + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pColInfoData->varmeta.length = colLen[i]; + pColInfoData->varmeta.allocLen = colLen[i]; + + memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); + pStart += sizeof(int32_t) * numOfRows; + + if (colLen[i] > 0) { + pColInfoData->pData = taosMemoryMalloc(colLen[i]); + } + } else { + memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); + pStart += BitmapLen(numOfRows); + } + + if (colLen[i] > 0) { + memcpy(pColInfoData->pData, pStart, colLen[i]); + } + + // TODO + // setting this flag to true temporarily so aggregate function on stable will + // examine NULL value for non-primary key column + pColInfoData->hasNull = true; + pStart += colLen[i]; + } + + ASSERT(pStart - pData == dataLen); + return pStart; +} \ No newline at end of file diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 23493ae647..9128167065 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -356,8 +356,7 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR if (pResultRowInfo->cur.pageId != -1 && ((pResult == NULL) || (pResult->pageId != pResultRowInfo->cur.pageId && pResult->offset != pResultRowInfo->cur.offset))) { SResultRowPosition pos = pResultRowInfo->cur; - SFilePage* - pPage = getBufPage(pResultBuf, pos.pageId); + SFilePage* pPage = getBufPage(pResultBuf, pos.pageId); releaseBufPage(pResultBuf, pPage); } @@ -2523,46 +2522,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { if (pColList == NULL) { // data from other sources - blockDataEnsureCapacity(pRes, numOfRows); - - int32_t dataLen = *(int32_t*)pData; - pData += sizeof(int32_t); - - pRes->info.groupId = *(uint64_t*)pData; - pData += sizeof(uint64_t); - - int32_t* colLen = (int32_t*)pData; - - char* pStart = pData + sizeof(int32_t) * numOfOutput; - for (int32_t i = 0; i < numOfOutput; ++i) { - colLen[i] = htonl(colLen[i]); - ASSERT(colLen[i] >= 0); - - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colLen[i]; - pColInfoData->varmeta.allocLen = colLen[i]; - - memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); - pStart += sizeof(int32_t) * numOfRows; - - if (colLen[i] > 0) { - pColInfoData->pData = taosMemoryMalloc(colLen[i]); - } - } else { - memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); - pStart += BitmapLen(numOfRows); - } - - if (colLen[i] > 0) { - memcpy(pColInfoData->pData, pStart, colLen[i]); - } - - // TODO setting this flag to true temporarily so aggregate function on stable will - // examine NULL value for non-primary key column - pColInfoData->hasNull = true; - pStart += colLen[i]; - } + blockCompressDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; From 9cea8dbaf7563d9c39d9fcb7191b316bed79beef Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 31 May 2022 00:21:47 +0800 Subject: [PATCH 06/47] fix(query): remove duplicated function call. --- source/libs/executor/src/timewindowoperator.c | 4 +--- tests/script/tsim/stream/basic0.sim | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 57a94cdc35..97ee124a6d 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -767,7 +767,7 @@ static int32_t saveResult(SResultRow* result, uint64_t groupId, SArray* pUpdated } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SArray* pUpdated) { + int32_t scanFlag, SArray* pUpdated) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -1287,8 +1287,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { continue; } - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } diff --git a/tests/script/tsim/stream/basic0.sim b/tests/script/tsim/stream/basic0.sim index 9edad991dc..29775a5ef1 100644 --- a/tests/script/tsim/stream/basic0.sim +++ b/tests/script/tsim/stream/basic0.sim @@ -63,7 +63,8 @@ if $data02 != 234 then return -1 endi -if $data03 != 234 then +if $data03 != 234 then + print expect 234, actual $data03 return -1 endi From 35635ea68c76727a8d0628a1b9c95875a19af291 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 10:42:18 +0800 Subject: [PATCH 07/47] feat:add schemaless db paras --- include/common/tmsg.h | 1 - source/common/src/systable.c | 2 ++ source/dnode/mnode/impl/src/mndDb.c | 4 ++++ source/dnode/mnode/impl/src/mndVgroup.c | 1 - 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 68480e4ffd..dbfbfcf0f2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1010,7 +1010,6 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; int32_t numOfRetensions; SArray* pRetensions; // SRetention - int8_t schemaless; // for tsma int8_t isTsma; diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 38a6bafe9a..4e2f9b074a 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -89,7 +89,9 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, + {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update }; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index cc884fb18b..35dc24dd9b 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1540,8 +1540,12 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)statusB, false); + } } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index a00841f911..e05b38a7c0 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -218,7 +218,6 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.hashMethod = pDb->cfg.hashMethod; createReq.numOfRetensions = pDb->cfg.numOfRetensions; createReq.pRetensions = pDb->cfg.pRetensions; - createReq.schemaless = pDb->cfg.schemaless; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; From c06316537e45ec2c8f1c0565f31a9ae3d72b6979 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 31 May 2022 15:04:52 +0800 Subject: [PATCH 08/47] enh(query): the results of count and hyperloglog is configurable. --- include/common/tglobal.h | 3 ++- source/common/src/tglobal.c | 8 +++++-- source/libs/function/src/builtins.c | 2 +- source/libs/function/src/builtinsimpl.c | 30 ++++++++++++++++--------- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 2a4ef565dd..30ae6c2adb 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -45,6 +45,8 @@ extern bool tsEnableSlaveQuery; extern bool tsPrintAuth; extern int64_t tsTickPerMin[3]; +extern int32_t tsCountAlwaysReturnValue; + // multi-process extern int32_t tsMultiProcess; extern int32_t tsMnodeShmSize; @@ -102,7 +104,6 @@ extern int32_t tsMaxStreamComputDelay; extern int32_t tsStreamCompStartDelay; extern int32_t tsRetryStreamCompDelay; extern float tsStreamComputDelayRatio; // the delayed computing ration of the whole time window -extern int32_t tsProjectExecInterval; extern int64_t tsMaxRetentWindow; // build info diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 141ec4f03b..e77c462e5a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -109,8 +109,11 @@ int32_t tsCompressColData = -1; */ int32_t tsCompatibleModel = 1; +// count/hyperloglog function always return values in case of all NULL data or Empty data set. +int32_t tsCountAlwaysReturnValue = 1; + // 10 ms for sliding time, the value will changed in case of time precision changed -int32_t tsMinSlidingTime = 10; +int32_t tsMinSlidingTime = 10; // the maxinum number of distict query result int32_t tsMaxNumOfDistinctResults = 1000 * 10000; @@ -130,7 +133,6 @@ int32_t tsRetryStreamCompDelay = 10 * 1000; // The delayed computing ration. 10% of the whole computing time window by default. float tsStreamComputDelayRatio = 0.1f; -int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance // the maximum allowed query buffer size during query processing for each data node. @@ -374,6 +376,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 10, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, 0) != 0) return -1; + if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxStreamCompDelay", tsMaxStreamComputDelay, 10, 1000000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "maxFirstStreamCompDelay", tsStreamCompStartDelay, 1000, 1000000000, 0) != 0) return -1; if (cfgAddInt32(pCfg, "retryStreamCompDelay", tsRetryStreamCompDelay, 10, 1000000000, 0) != 0) return -1; @@ -567,6 +570,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; + tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; tsMaxStreamComputDelay = cfgGetItem(pCfg, "maxStreamCompDelay")->i32; tsStreamCompStartDelay = cfgGetItem(pCfg, "maxFirstStreamCompDelay")->i32; tsRetryStreamCompDelay = cfgGetItem(pCfg, "retryStreamCompDelay")->i32; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6046450f52..b6369c7952 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -433,7 +433,7 @@ static int32_t translateHLL(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { "The input parameter of HYPERLOGLOG function can only be column"); } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_UBIGINT].bytes, .type = TSDB_DATA_TYPE_UBIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4cfb3b5252..c6c707a91c 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,6 +14,7 @@ */ #include "builtinsimpl.h" +#include "tglobal.h" #include "cJSON.h" #include "function.h" #include "querynodes.h" @@ -356,7 +357,7 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) { +static FORCE_INLINE int32_t getNumOfElems(SqlFunctionCtx* pCtx) { int32_t numOfElem = 0; /* @@ -391,11 +392,12 @@ static FORCE_INLINE int32_t getNumofElem(SqlFunctionCtx* pCtx) { * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ int32_t countFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumofElem(pCtx); - SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + int32_t numOfElem = getNumOfElems(pCtx); + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; - int32_t type = pInput->pData[0]->info.type; + + int32_t type = pInput->pData[0]->info.type; char* buf = GET_ROWCELL_INTERBUF(pResInfo); if (IS_NULL_TYPE(type)) { @@ -406,12 +408,17 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { *((int64_t*)buf) += numOfElem; } - SET_VAL(pResInfo, numOfElem, 1); + if (tsCountAlwaysReturnValue) { + pResInfo->numOfRes = 1; + } else { + SET_VAL(pResInfo, 1, 1); + } + return TSDB_CODE_SUCCESS; } int32_t countInvertFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumofElem(pCtx); + int32_t numOfElem = getNumOfElems(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); char* buf = GET_ROWCELL_INTERBUF(pResInfo); @@ -3236,13 +3243,13 @@ static uint64_t hllCountCnt(uint8_t *buckets) { z += buckethisto[j]; z *= 0.5; } + z += m * hllSigma(buckethisto[0]/(double)m); double E = (double)llroundl(HLL_ALPHA_INF*m*m/z); return (uint64_t) E; } - int32_t hllFunction(SqlFunctionCtx *pCtx) { SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -3275,7 +3282,6 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) { if (count > oldcount) { pInfo->buckets[index] = count; } - } SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1); @@ -3283,9 +3289,13 @@ int32_t hllFunction(SqlFunctionCtx *pCtx) { } int32_t hllFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { - SHLLInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + SResultRowEntryInfo *pInfo = GET_RES_INFO(pCtx); - pInfo->result = hllCountCnt(pInfo->buckets); + SHLLInfo* pHllInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + pHllInfo->result = hllCountCnt(pHllInfo->buckets); + if (tsCountAlwaysReturnValue && pHllInfo->result == 0) { + pInfo->numOfRes = 1; + } return functionFinalize(pCtx, pBlock); } From d358212a14fe2b5e8f4f476094225e3191806d8c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 31 May 2022 17:02:44 +0800 Subject: [PATCH 09/47] fix(query): move the bitmap when drop first rows. --- source/common/src/tdatablock.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index d1d5a9fcb2..17b7a5ea88 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1277,25 +1277,39 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { memmove(nullBitmap, nullBitmap + n / 8, newLen); } else { int32_t tail = n % 8; - int32_t i = 0; - + int32_t i = 0; uint8_t* p = (uint8_t*)nullBitmap; - while (i < len) { - uint8_t v = p[i]; - p[i] = 0; - p[i] = (v << tail); + if (n < 8) { + while (i < len) { + uint8_t v = p[i]; // source bitmap value + p[i] = (v << tail); - if (i < len - 1) { - uint8_t next = p[i + 1]; - p[i] |= (next >> (8 - tail)); + if (i < len - 1) { + uint8_t next = p[i + 1]; + p[i] |= (next >> (8 - tail)); + } + + i += 1; } + } else if (n > 8) { + int32_t gap = len - newLen; + while(i < newLen) { + uint8_t v = p[i + gap]; + p[i] = (v << tail); - i += 1; + if (i < newLen - 1) { + uint8_t next = p[i + gap + 1]; + p[i] |= (next >> (8 - tail)); + } + + i += 1; + } } } } + static void colDataTrimFirstNRows(SColumnInfoData* pColInfoData, size_t n, size_t total) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { memmove(pColInfoData->varmeta.offset, &pColInfoData->varmeta.offset[n], (total - n) * sizeof(int32_t)); From cca327190d3f7ea844617cf5ff60051ec7a97f74 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 31 May 2022 17:46:35 +0800 Subject: [PATCH 10/47] fix(query): restore some remove attributes. --- source/libs/function/src/builtinsimpl.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index c6c707a91c..6f7ace45b3 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -226,6 +226,7 @@ typedef struct SUniqueInfo { int32_t numOfPoints; uint8_t colType; int16_t colBytes; + bool hasNull; //null is not hashable, handle separately SHashObj *pHash; char pItems[]; } SUniqueInfo; @@ -3878,10 +3879,11 @@ static void doUniqueAdd(SUniqueInfo* pInfo, char *data, TSKEY ts, bool isNull) { if (isNull == true) { int32_t size = sizeof(SUniqueItem) + pInfo->colBytes; SUniqueItem *pItem = (SUniqueItem *)(pInfo->pItems + pInfo->numOfPoints * size); - if (pItem->isNull == false) { + if (pInfo->hasNull == false && pItem->isNull == false) { pItem->timestamp = ts; pItem->isNull = true; pInfo->numOfPoints++; + pInfo->hasNull = true; } else if (pItem->timestamp > ts && pItem->isNull == true) { pItem->timestamp = ts; } From b445afb5efdffcbe2c013c4ddcddda63dcade5b4 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 31 May 2022 18:01:19 +0800 Subject: [PATCH 11/47] feat: stable interval split --- source/libs/parser/test/mockCatalog.cpp | 4 +- .../libs/parser/test/mockCatalogService.cpp | 29 +++- source/libs/parser/test/mockCatalogService.h | 1 + source/libs/planner/src/planSpliter.c | 124 +++++++++--------- source/libs/planner/test/planIntervalTest.cpp | 8 +- 5 files changed, 101 insertions(+), 65 deletions(-) diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 154f13ea68..8fb28ce395 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -188,8 +188,8 @@ int32_t __catalogGetDBVgVersion(SCatalog* pCtg, const char* dbFName, int32_t* ve } int32_t __catalogGetDBVgInfo(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* dbFName, - SArray** vgroupList) { - return 0; + SArray** pVgList) { + return g_mockCatalogService->catalogGetDBVgInfo(dbFName, pVgList); } int32_t __catalogGetDBCfg(SCatalog* pCtg, void* pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 566c4d8b04..4834d2d377 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include "tdatablock.h" #include "tname.h" @@ -120,6 +121,25 @@ class MockCatalogServiceImpl { return copyTableVgroup(db, tNameGetTableName(pTableName), vgList); } + int32_t catalogGetDBVgInfo(const char* pDbFName, SArray** pVgList) const { + std::string dbFName(pDbFName); + DbMetaCache::const_iterator it = meta_.find(dbFName.substr(std::string(pDbFName).find_last_of('.') + 1)); + if (meta_.end() == it) { + return TSDB_CODE_FAILED; + } + std::set vgSet; + *pVgList = taosArrayInit(it->second.size(), sizeof(SVgroupInfo)); + for (const auto& vgs : it->second) { + for (const auto& vg : vgs.second->vgs) { + if (0 == vgSet.count(vg.vgId)) { + taosArrayPush(*pVgList, &vg); + vgSet.insert(vg.vgId); + } + } + } + return TSDB_CODE_SUCCESS; + } + int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const { auto it = udf_.find(funcName); if (udf_.end() == it) { @@ -187,8 +207,9 @@ class MockCatalogServiceImpl { // number of backward fills #define NOB(n) ((n) % 2 ? (n) / 2 + 1 : (n) / 2) // center aligned -#define CA(n, s) std::setw(NOF((n) - int((s).length()))) << "" << (s) \ - << std::setw(NOB((n) - int((s).length()))) << "" << "|" +#define CA(n, s) \ + std::setw(NOF((n) - int((s).length()))) << "" << (s) << std::setw(NOB((n) - int((s).length()))) << "" \ + << "|" // string field length #define SFL 20 // string field header @@ -490,6 +511,10 @@ int32_t MockCatalogService::catalogGetTableDistVgInfo(const SName* pTableName, S return impl_->catalogGetTableDistVgInfo(pTableName, pVgList); } +int32_t MockCatalogService::catalogGetDBVgInfo(const char* pDbFName, SArray** pVgList) const { + return impl_->catalogGetDBVgInfo(pDbFName, pVgList); +} + int32_t MockCatalogService::catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const { return impl_->catalogGetUdfInfo(funcName, pInfo); } diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index cb0f10e95b..133a355c59 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -61,6 +61,7 @@ class MockCatalogService { int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const; int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const; int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const; + int32_t catalogGetDBVgInfo(const char* pDbFName, SArray** pVgList) const; int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const; int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index ea149f8363..cfa265b722 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -17,7 +17,7 @@ #define SPLIT_FLAG_MASK(n) (1 << n) -#define SPLIT_FLAG_STS SPLIT_FLAG_MASK(0) +#define SPLIT_FLAG_STABLE_SPLIT SPLIT_FLAG_MASK(0) #define SPLIT_FLAG_SET_MASK(val, mask) (val) |= (mask) #define SPLIT_FLAG_TEST_MASK(val, mask) (((val) & (mask)) != 0) @@ -35,27 +35,6 @@ typedef struct SSplitRule { FSplit splitFunc; } SSplitRule; -typedef struct SStsInfo { - SScanLogicNode* pScan; - SLogicSubplan* pSubplan; -} SStsInfo; - -typedef struct SCtjInfo { - SJoinLogicNode* pJoin; - SLogicNode* pSplitNode; - SLogicSubplan* pSubplan; -} SCtjInfo; - -typedef struct SUaInfo { - SProjectLogicNode* pProject; - SLogicSubplan* pSubplan; -} SUaInfo; - -typedef struct SUnInfo { - SAggLogicNode* pAgg; - SLogicSubplan* pSubplan; -} SUnInfo; - typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, void* pInfo); static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, int32_t flag) { @@ -121,14 +100,19 @@ static bool splMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, int32_t flag, return false; } -static SLogicNode* stsMatchByNode(SLogicNode* pNode) { +typedef struct SStableSplitInfo { + SScanLogicNode* pScan; + SLogicSubplan* pSubplan; +} SStableSplitInfo; + +static SLogicNode* stbSplMatchByNode(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != ((SScanLogicNode*)pNode)->pVgroupList && ((SScanLogicNode*)pNode)->pVgroupList->numOfVgroups > 1) { return pNode; } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = stsMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = stbSplMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -136,8 +120,8 @@ static SLogicNode* stsMatchByNode(SLogicNode* pNode) { return NULL; } -static bool stsFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { - SLogicNode* pSplitNode = stsMatchByNode(pSubplan->pNode); +static bool stbSplFindSplitNode(SLogicSubplan* pSubplan, SStableSplitInfo* pInfo) { + SLogicNode* pSplitNode = stbSplMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pScan = (SScanLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; @@ -145,13 +129,13 @@ static bool stsFindSplitNode(SLogicSubplan* pSubplan, SStsInfo* pInfo) { return NULL != pSplitNode; } -static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { - SStsInfo info = {0}; - if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STS, (FSplFindSplitNode)stsFindSplitNode, &info)) { +static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SStableSplitInfo info = {0}; + if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STABLE_SPLIT, (FSplFindSplitNode)stbSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, - splCreateSubplan(pCxt, (SLogicNode*)info.pScan, SPLIT_FLAG_STS)); + splCreateSubplan(pCxt, (SLogicNode*)info.pScan, SPLIT_FLAG_STABLE_SPLIT)); if (TSDB_CODE_SUCCESS == code) { code = splCreateExchangeNode(pCxt, info.pSubplan, (SLogicNode*)info.pScan, SUBPLAN_TYPE_MERGE); } @@ -160,7 +144,13 @@ static int32_t stsSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { return code; } -static bool needSplit(SJoinLogicNode* pJoin) { +typedef struct SSigTbJoinSplitInfo { + SJoinLogicNode* pJoin; + SLogicNode* pSplitNode; + SLogicSubplan* pSubplan; +} SSigTbJoinSplitInfo; + +static bool sigTbJoinSplNeedSplit(SJoinLogicNode* pJoin) { if (!pJoin->isSingleTableJoin) { return false; } @@ -168,13 +158,13 @@ static bool needSplit(SJoinLogicNode* pJoin) { QUERY_NODE_LOGIC_PLAN_EXCHANGE != nodeType(nodesListGetNode(pJoin->node.pChildren, 1)); } -static SJoinLogicNode* ctjMatchByNode(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pNode) && needSplit((SJoinLogicNode*)pNode)) { +static SJoinLogicNode* sigTbJoinSplMatchByNode(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_JOIN == nodeType(pNode) && sigTbJoinSplNeedSplit((SJoinLogicNode*)pNode)) { return (SJoinLogicNode*)pNode; } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SJoinLogicNode* pSplitNode = ctjMatchByNode((SLogicNode*)pChild); + SJoinLogicNode* pSplitNode = sigTbJoinSplMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -182,8 +172,8 @@ static SJoinLogicNode* ctjMatchByNode(SLogicNode* pNode) { return NULL; } -static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SCtjInfo* pInfo) { - SJoinLogicNode* pJoin = ctjMatchByNode(pSubplan->pNode); +static bool sigTbJoinSplFindSplitNode(SLogicSubplan* pSubplan, SSigTbJoinSplitInfo* pInfo) { + SJoinLogicNode* pJoin = sigTbJoinSplMatchByNode(pSubplan->pNode); if (NULL != pJoin) { pInfo->pJoin = pJoin; pInfo->pSplitNode = nodesListGetNode(pJoin->node.pChildren, 1); @@ -192,9 +182,9 @@ static bool ctjFindSplitNode(SLogicSubplan* pSubplan, SCtjInfo* pInfo) { return NULL != pJoin; } -static int32_t ctjSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { - SCtjInfo info = {0}; - if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)ctjFindSplitNode, &info)) { +static int32_t singleTableJoinSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SSigTbJoinSplitInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)sigTbJoinSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateSubplan(pCxt, info.pSplitNode, 0)); @@ -277,13 +267,18 @@ static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubpl return code; } -static SLogicNode* uaMatchByNode(SLogicNode* pNode) { +typedef struct SUnionAllSplitInfo { + SProjectLogicNode* pProject; + SLogicSubplan* pSubplan; +} SUnionAllSplitInfo; + +static SLogicNode* unionAllMatchByNode(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { return pNode; } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = uaMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = unionAllMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -291,8 +286,8 @@ static SLogicNode* uaMatchByNode(SLogicNode* pNode) { return NULL; } -static bool uaFindSplitNode(SLogicSubplan* pSubplan, SUaInfo* pInfo) { - SLogicNode* pSplitNode = uaMatchByNode(pSubplan->pNode); +static bool unionAllFindSplitNode(SLogicSubplan* pSubplan, SUnionAllSplitInfo* pInfo) { + SLogicNode* pSplitNode = unionAllMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pProject = (SProjectLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; @@ -300,13 +295,13 @@ static bool uaFindSplitNode(SLogicSubplan* pSubplan, SUaInfo* pInfo) { return NULL != pSplitNode; } -static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { +static int32_t unionAllCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pCxt->groupId; - // pExchange->precision = pScan->pMeta->tableInfo.precision; + pExchange->precision = pProject->node.precision; pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); if (NULL == pExchange->node.pTargets) { return TSDB_CODE_OUT_OF_MEMORY; @@ -332,28 +327,33 @@ static int32_t uaCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan return TSDB_CODE_FAILED; } -static int32_t uaSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { - SUaInfo info = {0}; - if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)uaFindSplitNode, &info)) { +static int32_t unionAllSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SUnionAllSplitInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unionAllFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pProject); if (TSDB_CODE_SUCCESS == code) { - code = uaCreateExchangeNode(pCxt, info.pSubplan, info.pProject); + code = unionAllCreateExchangeNode(pCxt, info.pSubplan, info.pProject); } ++(pCxt->groupId); pCxt->split = true; return code; } -static SLogicNode* unMatchByNode(SLogicNode* pNode) { +typedef struct SUnionDistinctSplitInfo { + SAggLogicNode* pAgg; + SLogicSubplan* pSubplan; +} SUnionDistinctSplitInfo; + +static SLogicNode* unionDistinctMatchByNode(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { return pNode; } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = unMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = unionDistinctMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -378,8 +378,8 @@ static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan return nodesListMakeAppend(&pAgg->node.pChildren, pExchange); } -static bool unFindSplitNode(SLogicSubplan* pSubplan, SUnInfo* pInfo) { - SLogicNode* pSplitNode = unMatchByNode(pSubplan->pNode); +static bool unionDistinctFindSplitNode(SLogicSubplan* pSubplan, SUnionDistinctSplitInfo* pInfo) { + SLogicNode* pSplitNode = unionDistinctMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pAgg = (SAggLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; @@ -387,9 +387,9 @@ static bool unFindSplitNode(SLogicSubplan* pSubplan, SUnInfo* pInfo) { return NULL != pSplitNode; } -static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { - SUnInfo info = {0}; - if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unFindSplitNode, &info)) { +static int32_t unionDistinctSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { + SUnionDistinctSplitInfo info = {0}; + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unionDistinctFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } @@ -402,10 +402,14 @@ static int32_t unSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { return code; } -static const SSplitRule splitRuleSet[] = {{.pName = "SuperTableScan", .splitFunc = stsSplit}, - {.pName = "ChildTableJoin", .splitFunc = ctjSplit}, - {.pName = "UnionAll", .splitFunc = uaSplit}, - {.pName = "Union", .splitFunc = unSplit}}; +// clang-format off +static const SSplitRule splitRuleSet[] = { + {.pName = "SuperTableSplit", .splitFunc = stableSplit}, + {.pName = "SingleTableJoinSplit", .splitFunc = singleTableJoinSplit}, + {.pName = "UnionAllSplit", .splitFunc = unionAllSplit}, + {.pName = "UnionDistinctSplit", .splitFunc = unionDistinctSplit} +}; +// clang-format on static const int32_t splitRuleNum = (sizeof(splitRuleSet) / sizeof(SSplitRule)); diff --git a/source/libs/planner/test/planIntervalTest.cpp b/source/libs/planner/test/planIntervalTest.cpp index c9bae46ca9..a04f47741e 100644 --- a/source/libs/planner/test/planIntervalTest.cpp +++ b/source/libs/planner/test/planIntervalTest.cpp @@ -50,4 +50,10 @@ TEST_F(PlanIntervalTest, selectFunc) { run("SELECT MAX(c1), MIN(c1) FROM t1 INTERVAL(10s)"); // select function along with the columns of select row, and with INTERVAL clause run("SELECT MAX(c1), c2 FROM t1 INTERVAL(10s)"); -} \ No newline at end of file +} + +TEST_F(PlanIntervalTest, stable) { + useDb("root", "test"); + + run("SELECT COUNT(*) FROM st1 INTERVAL(10s)"); +} From f2163399f5a8e36548396334f6d169203c570bbd Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 31 May 2022 19:10:19 +0800 Subject: [PATCH 12/47] update case for all --- tests/system-test/2-query/mavg.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index c8cbd269f9..d877b0f821 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -288,15 +288,17 @@ class TDTestCase: else: tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0 + # print(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] - pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k - tdSql.query(self.mavg_query_form( - sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, - table_expr=table_expr, condition=condition - )) - for i in range(tdSql.queryRows): - print(f"case in {line}: ", end='') - tdSql.checkData(i, 0, pre_mavg[i]) + if pre_result is pre_result.any: + pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k + tdSql.query(self.mavg_query_form( + sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, + table_expr=table_expr, condition=condition + )) + for i in range(tdSql.queryRows): + print(f"case in {line}: ", end='') + tdSql.checkData(i, 0, pre_mavg[i]) pass From d296ec9baf220e6e91548d6e9b79da8ea214cdee Mon Sep 17 00:00:00 2001 From: "wenzhouwww@live.cn" Date: Tue, 31 May 2022 19:16:09 +0800 Subject: [PATCH 13/47] update case --- tests/system-test/2-query/mavg.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/mavg.py b/tests/system-test/2-query/mavg.py index d877b0f821..1d92964615 100644 --- a/tests/system-test/2-query/mavg.py +++ b/tests/system-test/2-query/mavg.py @@ -289,8 +289,9 @@ class TDTestCase: tdSql.query(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") offset_val = condition.split("offset")[1].split(" ")[1] if "offset" in condition else 0 # print(f"select {col} from {table_expr} {re.sub('limit [0-9]*|offset [0-9]*','',condition)}") - pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] - if pre_result is pre_result.any: + if not tdSql.queryResult: + pre_result = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + pre_mavg = pre_mavg = np.convolve(pre_result, np.ones(k), "valid")[offset_val:]/k tdSql.query(self.mavg_query_form( sel=sel, func=func, col=col, m_comm=m_comm, k=k, r_comm=r_comm, alias=alias, fr=fr, From 41d412fba061ea9be3683b80c7202f00926b9532 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 1 Jun 2022 14:21:08 +0800 Subject: [PATCH 14/47] test: modify consume processor --- tests/test/c/tmqSim.c | 99 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 8 deletions(-) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index accd1dd080..13c64b68f9 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -61,6 +61,10 @@ typedef struct { tmq_t* tmq; tmq_list_t* topicList; + + int32_t numOfVgroups; + int32_t rowsOfPerVgroups[32][2]; // [i][0]: vgroup id, [i][1]: rows of consume + int64_t ts; } SThreadInfo; @@ -69,7 +73,8 @@ typedef struct { char cdbName[32]; char dbName[32]; int32_t showMsgFlag; - int32_t showRowFlag; + int32_t showRowFlag; + int32_t saveRowFlag; int32_t consumeDelay; // unit s int32_t numOfThread; SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; @@ -77,6 +82,7 @@ typedef struct { static SConfInfo g_stConfInfo; TdFilePtr g_fp = NULL; +static int running = 1; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -93,6 +99,8 @@ static void printHelp() { printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); printf("%s%s\n", indent, "-r"); printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%d\n", indent, indent, "saveRowFlag, default is ", g_stConfInfo.saveRowFlag); printf("%s%s\n", indent, "-y"); printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); exit(EXIT_SUCCESS); @@ -135,6 +143,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# saveRowFlag: %d\n", g_stConfInfo.saveRowFlag); taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); @@ -165,6 +174,7 @@ void parseArgument(int32_t argc, char* argv[]) { memset(&g_stConfInfo, 0, sizeof(SConfInfo)); g_stConfInfo.showMsgFlag = 0; g_stConfInfo.showRowFlag = 0; + g_stConfInfo.saveRowFlag = 0; g_stConfInfo.consumeDelay = 5; for (int32_t i = 1; i < argc; i++) { @@ -181,6 +191,8 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { g_stConfInfo.showRowFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + g_stConfInfo.saveRowFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else { @@ -200,6 +212,7 @@ void parseArgument(int32_t argc, char* argv[]) { pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); + pPrint("%s saveRowFlag:%d %s", GREEN, g_stConfInfo.saveRowFlag, NC); #endif } @@ -225,15 +238,52 @@ void ltrim(char* str) { // return str; } -static int running = 1; -static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { +void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { + int32_t i; + for (i = 0; i < pInfo->numOfVgroups; i++) { + if (vgroupId == pInfo->rowsOfPerVgroups[i][0]) { + pInfo->rowsOfPerVgroups[i][1] += rows; + } + } + + if (i == pInfo->numOfVgroups) { + pInfo->rowsOfPerVgroups[i][0] = vgroupId; + pInfo->rowsOfPerVgroups[i][1] += rows; + pInfo->numOfVgroups++; + } +} + + +int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + sprintf(sqlStr, "insert into %s.content_%d values (%"PRId64", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->ts++, buf); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + pError("error in insert consume result, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error in insert consume result, reason:%s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); + taos_free_result(pRes); + exit(-1); + } + + taos_free_result(pRes); + + return 0; +} + +static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t threadLable) { char buf[1024]; int32_t totalRows = 0; - + // printf("topic: %s\n", tmq_get_topic_name(msg)); - // printf("vg:%d\n", tmq_get_vgroup_id(msg)); - taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + int32_t vgroupId = tmq_get_vgroup_id(msg); + + taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", threadLable, pInfo->consumerId); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d, tableName: %s\n", tmq_get_topic_name(msg), vgroupId, tmq_get_table_name(msg)); while (1) { TAOS_ROW row = taos_fetch_row(msg); @@ -247,11 +297,16 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + if (0 != g_stConfInfo.saveRowFlag) { + saveConsumeContentToTbl(pInfo, buf); + } } totalRows++; } + addRowsToVgroupId(pInfo, vgroupId, totalRows); + return totalRows; } @@ -344,6 +399,32 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { taos_free_result(pRes); + #if 0 + // vgroups + for (i = 0; i < pInfo->numOfVgroups; i++) { + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.vgroup_%d values (%"PRId64", %d, %" PRId64 ", %" PRId64 ", %d)", + g_stConfInfo.cdbName, + now, + pInfo->consumerId, + pInfo->consumeMsgCnt, + pInfo->consumeRowCnt, + pInfo->checkresult); + + char tmpString[128]; + taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId ,sqlStr); + + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + exit(-1); + } + + taos_free_result(pRes); + } + #endif + return 0; } @@ -356,11 +437,13 @@ void loop_consume(SThreadInfo* pInfo) { char tmpString[128]; taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), pInfo->consumerId); + pInfo->ts = taosGetTimestampMs(); + while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { - totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); + totalRows += msg_process(tmqMsg, pInfo, totalMsgs); } taos_free_result(tmqMsg); From 36465ca29b3a0d02caf761a08b802583213a1829 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 17:35:27 +0800 Subject: [PATCH 15/47] feat: increase the number of database replicas --- source/dnode/mnode/impl/src/mndVgroup.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 219e0fa3dc..9262aa167b 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -504,7 +504,7 @@ _OVER: taosArrayDestroy(pArray); return code; } -//---> + int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { @@ -543,7 +543,7 @@ int32_t mndAddVnodeToVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; return -1; } -//---> + int32_t mndRemoveVnodeFromVgroup(SMnode *pMnode, SVgObj *pVgroup, SArray *pArray, SVnodeGid *del1, SVnodeGid *del2) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { From 6418caa4304b620b763d21c669877e1af17163f8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 18:53:23 +0800 Subject: [PATCH 16/47] feat:add schemaless param from db --- source/client/src/clientSml.c | 2 ++ source/client/test/smlTest.cpp | 24 +++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 29c7608126..8d1924b1c9 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2381,6 +2381,8 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr } if(isSchemalessDb(info) != TSDB_CODE_SUCCESS){ + request->code = TSDB_CODE_SML_INVALID_DB_CONF; + smlBuildInvalidDataMsg(&info->msgBuf, "Cannot write data to a non schemaless database", NULL); goto end; } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index 217699e360..4684df8418 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1258,4 +1258,26 @@ TEST(testCase, sml_TD15742_Test) { destroyRequest(request); smlDestroyInfo(info); -} \ No newline at end of file +} + +TEST(testCase, sml_params_Test) { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(taos, nullptr); + + TAOS_RES* pRes = taos_query(taos, "create database if not exists param"); + taos_free_result(pRes); + + const char *sql[] = { + "test_ms,t0=t c0=f 1626006833641", + }; + TAOS_RES* res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + ASSERT_EQ(taos_errno(res), TSDB_CODE_PAR_DB_NOT_SPECIFIED); + taos_free_result(pRes); + + pRes = taos_query(taos, "use param"); + taos_free_result(pRes); + + res = taos_schemaless_insert(taos, (char**)sql, 1, TSDB_SML_LINE_PROTOCOL, TSDB_SML_TIMESTAMP_MILLI_SECONDS); + ASSERT_EQ(taos_errno(res), TSDB_CODE_SML_INVALID_DB_CONF); + taos_free_result(pRes); +} From e88abb16c28aaaf87521b02119a539110eb82150 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 1 Jun 2022 19:06:58 +0800 Subject: [PATCH 17/47] feat: tsma days refactor --- include/common/tmsg.h | 10 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 9 + source/dnode/vnode/CMakeLists.txt | 2 +- source/dnode/vnode/inc/vnode.h | 3 + source/dnode/vnode/src/inc/sma.h | 2 + source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/sma/sma.c | 8 + source/dnode/vnode/src/sma/smaEnv.c | 2 +- source/dnode/vnode/src/sma/smaTimeRange.c | 12 +- source/dnode/vnode/src/sma/smaTimeRange2.c | 1084 +++++++++++++++++++ source/dnode/vnode/src/tq/tq.c | 2 +- source/dnode/vnode/src/tsdb/tsdbSma.c | 16 +- source/dnode/vnode/test/tsdbSmaTest.cpp | 14 +- 13 files changed, 1136 insertions(+), 30 deletions(-) create mode 100644 source/dnode/vnode/src/sma/smaTimeRange2.c diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 63898b9163..e417a6d4ed 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2352,19 +2352,19 @@ typedef struct { STSma* tSma; } STSmaWrapper; -static FORCE_INLINE void tdDestroyTSma(STSma* pSma) { +static FORCE_INLINE void tDestroyTSma(STSma* pSma) { if (pSma) { taosMemoryFreeClear(pSma->expr); taosMemoryFreeClear(pSma->tagsFilter); } } -static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { +static FORCE_INLINE void tDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { if (pSW) { if (pSW->tSma) { if (deepCopy) { for (uint32_t i = 0; i < pSW->number; ++i) { - tdDestroyTSma(pSW->tSma + i); + tDestroyTSma(pSW->tSma + i); } } taosMemoryFreeClear(pSW->tSma); @@ -2372,8 +2372,8 @@ static FORCE_INLINE void tdDestroyTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) } } -static FORCE_INLINE void* tdFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { - tdDestroyTSmaWrapper(pSW, deepCopy); +static FORCE_INLINE void* tFreeTSmaWrapper(STSmaWrapper* pSW, bool deepCopy) { + tDestroyTSmaWrapper(pSW, deepCopy); taosMemoryFreeClear(pSW); return NULL; } diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index dda59b2396..b97a07d04d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -194,6 +194,15 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SVnodeCfg vnodeCfg = {0}; vmGenerateVnodeCfg(&createReq, &vnodeCfg); + if (createReq.isTsma) { + SMsgHead *smaMsg = createReq.pTsma; + uint32_t contLen = (uint32_t)(htonl(smaMsg->contLen) - sizeof(SMsgHead)); + if (smaGetTSmaDays(&vnodeCfg, POINTER_SHIFT(smaMsg, sizeof(SMsgHead)), contLen, &vnodeCfg.tsdbCfg.days) < 0) { + dError("vgId:%d, failed to get tsma days since %s", createReq.vgId, terrstr()); + return -1; + } + } + SWrapperCfg wrapperCfg = {0}; vmGenerateWrapperCfg(pMgmt, &createReq, &wrapperCfg); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index ea2a256663..baa171451c 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -32,7 +32,7 @@ target_sources( "src/sma/smaEnv.c" "src/sma/smaOpen.c" "src/sma/smaRollup.c" - "src/sma/smaTimeRange.c" + "src/sma/smaTimeRange2.c" # tsdb # "src/tsdb/tsdbTDBImpl.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index f8f65ce74c..6bdf505a91 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -146,6 +146,9 @@ bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, int32_t *pNumOfRows, int16_t *pNumOfCols); +// sma +int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); + // need to reposition // structs diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 0601df61e7..80fa48a92a 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -223,6 +223,8 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) // TODO: This is the basic params, and should wrap the params to a queryHandle. int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); +int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a57eb4e899..4232c725d4 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -464,7 +464,7 @@ STSmaWrapper *metaGetSmaInfoByTable(SMeta *pMeta, tb_uid_t uid, bool deepCopy) { _err: metaReaderClear(&mr); taosArrayDestroy(pSmaIds); - tdFreeTSmaWrapper(pSW, deepCopy); + tFreeTSmaWrapper(pSW, deepCopy); return NULL; } diff --git a/source/dnode/vnode/src/sma/sma.c b/source/dnode/vnode/src/sma/sma.c index 0e7ce385a1..bf82fbd1f7 100644 --- a/source/dnode/vnode/src/sma/sma.c +++ b/source/dnode/vnode/src/sma/sma.c @@ -52,3 +52,11 @@ int32_t tdGetTSmaData(SSma* pSma, char* pData, int64_t indexUid, TSKEY querySKey } return code; } + +int32_t smaGetTSmaDays(SVnodeCfg* pCfg, void* pCont, uint32_t contLen, int32_t *days) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdGetTSmaDaysImpl(pCfg, pCont, contLen, days)) < 0) { + smaWarn("vgId:%d get tSma days failed since %s", pCfg->vgId, tstrerror(terrno)); + } + return code; +} diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 8285b74e50..531feaf861 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -278,7 +278,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { if (pSmaStatItem) { - tdDestroyTSma(pSmaStatItem->pTSma); + tDestroyTSma(pSmaStatItem->pTSma); taosMemoryFreeClear(pSmaStatItem->pTSma); taosHashCleanup(pSmaStatItem->expiredWindows); taosMemoryFreeClear(pSmaStatItem); diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index f771e73c8a..ebbe30ab5f 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -828,7 +828,7 @@ int32_t tdDropTSma(SSma *pSma, char *pMsg) { // TODO: send msg to stream computing to drop tSma // if ((send msg to stream computing) < 0) { - // tdDestroyTSma(&vCreateSmaReq); + // tDestroyTSma(&vCreateSmaReq); // return -1; // } // @@ -982,25 +982,25 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) SSubmitBlkIter blkIter = {0}; if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); break; } while (true) { STSRow *row = tGetSubmitBlkNext(&blkIter); if (!row) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); break; } if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { if (pSW) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); } if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { break; } if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); break; } @@ -1020,7 +1020,7 @@ int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) if (lastWinSKey != winSKey) { lastWinSKey = winSKey; if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { - pSW = tdFreeTSmaWrapper(pSW, false); + pSW = tFreeTSmaWrapper(pSW, false); tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaTimeRange2.c b/source/dnode/vnode/src/sma/smaTimeRange2.c new file mode 100644 index 0000000000..ba0b2676e0 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaTimeRange2.c @@ -0,0 +1,1084 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" +#include "tsdb.h" + +typedef STsdbCfg STSmaKeepCfg; + +#undef _TEST_SMA_PRINT_DEBUG_LOG_ +#define SMA_STORAGE_MINUTES_MAX 86400 +#define SMA_STORAGE_MINUTES_DAY 1440 +#define SMA_STORAGE_MINUTES_MIN 1440 +#define SMA_STORAGE_TSDB_MINUTES 86400 +#define SMA_STORAGE_TSDB_TIMES 10 +#define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file TODO: the feasible value? +#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 +#define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds + +#define SMA_STATE_ITEM_HASH_SLOT 32 + +typedef struct { + SSma *pSma; + SDBFile dFile; + const SArray *pDataBlocks; // sma data + int64_t interval; // interval with the precision of DB +} STSmaWriteH; + +typedef struct { + int32_t iter; + int32_t fid; +} SmaFsIter; + +typedef struct { + STsdb *pTsdb; + SSma *pSma; + SDBFile dFile; + int64_t interval; // interval with the precision of DB + int32_t blockSize; // size of SMA block item + int32_t days; + int8_t storageLevel; + SmaFsIter smaFsIter; +} STSmaReadH; + +typedef enum { + SMA_STORAGE_LEVEL_TSDB = 0, // use days of self-defined e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f200.tsma + SMA_STORAGE_LEVEL_DFILESET = 1 // use days of TS data e.g. vnode${N}/tsdb/tsma/sma_index_uid/v2f1906.tsma +} ESmaStorageLevel; + +// static func + +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); +static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval); +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit); +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit); +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH); +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel); +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn); +// expired window + +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, int64_t version); +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey); +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid); + +/** + * @brief Judge the tsma file split days + * + * @param pCfg + * @param pCont + * @param contLen + * @param days unit is minute + * @return int32_t + */ +int32_t tdGetTSmaDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { + SDecoder coder = {0}; + tDecoderInit(&coder, pCont, contLen); + + STSma tsma = {0}; + if (tDecodeSVCreateTSmaReq(&coder, &tsma) < 0) { + terrno = TSDB_CODE_MSG_DECODE_ERROR; + goto _err; + } + STsdbCfg *pTsdbCfg = &pCfg->tsdbCfg; + int64_t mInterval = convertTimeFromPrecisionToUnit(tsma.interval, pTsdbCfg->precision, TIME_UNIT_MINUTE); + int64_t records = pTsdbCfg->days / mInterval; + + if (records >= SMA_STORAGE_SPLIT_FACTOR) { + *days = pTsdbCfg->days; + } else { + int64_t daysPerFile = mInterval * SMA_STORAGE_MINUTES_DAY * 2; + + if (daysPerFile > SMA_STORAGE_MINUTES_MAX) { + *days = SMA_STORAGE_MINUTES_MAX; + } else { + *days = (int32_t)daysPerFile; + } + + if(*days < pTsdbCfg->days) { + *days = pTsdbCfg->days; + } + } + tDecoderClear(&coder); + return 0; +_err: + tDecoderClear(&coder); + return -1; +} + +// read data + +// implementation + +/** + * @brief + * + * @param pSmaH + * @param pSma + * @param interval + * @param intervalUnit + * @return int32_t + */ +static int32_t tdInitTSmaReadH(STSmaReadH *pSmaH, SSma *pSma, int64_t interval, int8_t intervalUnit) { + STSmaKeepCfg *pCfg = SMA_TSDB_CFG(pSma); + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->storageLevel = tdGetSmaStorageLevel(pCfg, interval); + pSmaH->days = tdGetTSmaDays(pSma, pSmaH->interval, pSmaH->storageLevel); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Init of tSma FS + * + * @param pReadH + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey) { + SSma *pSma = pSmaH->pSma; + + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, pSmaH->days, SMA_TSDB_CFG(pSma)->precision)); + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + pSmaH->smaFsIter.iter = 0; + pSmaH->smaFsIter.fid = fid; + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Set and open tSma file if it has key locates in queryWin. + * + * @param pReadH + * @param param + * @param queryWin + * @return true + * @return false + */ +static bool tdSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { + // SArray *smaFs = pReadH->pTsdb->fs->cstatus->sf; + // int32_t nSmaFs = taosArrayGetSize(smaFs); + + smaCloseDBF(&pReadH->dFile); + +#if 0 + while (pReadH->smaFsIter.iter < nSmaFs) { + void *pSmaFile = taosArrayGet(smaFs, pReadH->smaFsIter.iter); + if (pSmaFile) { // match(indexName, queryWindow) + // TODO: select the file by index_name ... + pReadH->dFile = pSmaFile; + ++pReadH->smaFsIter.iter; + break; + } + ++pReadH->smaFsIter.iter; + } + + if (pReadH->pDFile) { + tdDebug("vg%d: smaFile %s matched", REPO_ID(pReadH->pTsdb), "[pSmaFile dir]"); + return true; + } +#endif + + return false; +} + +/** + * @brief Approximate value for week/month/year. + * + * @param interval + * @param intervalUnit + * @param precision + * @param adjusted Interval already adjusted according to DB precision + * @return int64_t + */ +static int64_t tdGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { + if (adjusted) { + return interval; + } + + switch (intervalUnit) { + case TIME_UNIT_YEAR: // approximate value + interval *= 365 * 86400 * 1e3; + break; + case TIME_UNIT_MONTH: // approximate value + interval *= 30 * 86400 * 1e3; + break; + case TIME_UNIT_WEEK: // approximate value + interval *= 7 * 86400 * 1e3; + break; + case TIME_UNIT_DAY: // the interval for tSma calculation must <= day + interval *= 86400 * 1e3; + break; + case TIME_UNIT_HOUR: + interval *= 3600 * 1e3; + break; + case TIME_UNIT_MINUTE: + interval *= 60 * 1e3; + break; + case TIME_UNIT_SECOND: + interval *= 1e3; + break; + default: + break; + } + + switch (precision) { + case TSDB_TIME_PRECISION_MILLI: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // nano second + return interval / 1e6; + } else { // ms + return interval; + } + break; + case TSDB_TIME_PRECISION_MICRO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e3; + } else { // ms + return interval * 1e3; + } + break; + case TSDB_TIME_PRECISION_NANO: + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval * 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval; + } else { // ms + return interval * 1e6; + } + break; + default: // ms + if (TIME_UNIT_MICROSECOND == intervalUnit) { // us + return interval / 1e3; + } else if (TIME_UNIT_NANOSECOND == intervalUnit) { // ns + return interval / 1e6; + } else { // ms + return interval; + } + break; + } + return interval; +} + +static int32_t tdInitTSmaWriteH(STSmaWriteH *pSmaH, SSma *pSma, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit) { + pSmaH->pSma = pSma; + pSmaH->interval = tdGetIntervalByPrecision(interval, intervalUnit, SMA_TSDB_CFG(pSma)->precision, true); + pSmaH->pDataBlocks = pDataBlocks; + pSmaH->dFile.fid = SMA_IVLD_FID; + return TSDB_CODE_SUCCESS; +} + +static void tdDestroyTSmaWriteH(STSmaWriteH *pSmaH) { + if (pSmaH) { + smaCloseDBF(&pSmaH->dFile); + } +} + +static int32_t tdSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { + SSma *pSma = pSmaH->pSma; + ASSERT(!pSmaH->dFile.path && !pSmaH->dFile.pDB); + + pSmaH->dFile.fid = fid; + char tSmaFile[TSDB_FILENAME_LEN] = {0}; + snprintf(tSmaFile, TSDB_FILENAME_LEN, "%" PRIi64 "%sv%df%d.tsma", indexUid, TD_DIRSEP, SMA_VID(pSma), fid); + pSmaH->dFile.path = strdup(tSmaFile); + + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma + * @param interval Interval calculated by DB's precision + * @param storageLevel + * @return int32_t + */ +static int32_t tdGetTSmaDays(SSma *pSma, int64_t interval, int32_t storageLevel) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + int32_t daysPerFile = pCfg->days; // unit is minute + + if (storageLevel == SMA_STORAGE_LEVEL_TSDB) { + int32_t minutes = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]); + if (minutes > SMA_STORAGE_TSDB_MINUTES) { + daysPerFile = SMA_STORAGE_TSDB_MINUTES; + } + } + + return daysPerFile; +} + +/** + * @brief Judge the tSma storage level + * + * @param pCfg + * @param interval + * @return int32_t + */ +static int32_t tdGetSmaStorageLevel(STSmaKeepCfg *pCfg, int64_t interval) { + int64_t mInterval = convertTimeFromPrecisionToUnit(interval, pCfg->precision, TIME_UNIT_MINUTE); + if (pCfg->days / mInterval >= SMA_STORAGE_SPLIT_FACTOR) { + return SMA_STORAGE_LEVEL_DFILESET; + } + return SMA_STORAGE_LEVEL_TSDB; +} + +/** + * @brief Insert/Update Time-range-wise SMA data. + * - If interval < SMA_STORAGE_SPLIT_HOURS(e.g. 24), save the SMA data as a part of DFileSet to e.g. + * v3f1900.tsma.${sma_index_name}. The days is the same with that for TS data files. + * - If interval >= SMA_STORAGE_SPLIT_HOURS, save the SMA data to e.g. vnode3/tsma/v3f632.tsma.${sma_index_name}. The + * days is 30 times of the interval, and the minimum days is SMA_STORAGE_TSDB_DAYS(30d). + * - The destination file of one data block for some interval is determined by its start TS key. + * + * @param pSma + * @param msg + * @return int32_t + */ +int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { + STsdbCfg *pCfg = SMA_TSDB_CFG(pSma); + const SArray *pDataBlocks = (const SArray *)msg; + int64_t testSkey = TSKEY_INITIAL_VAL; + + // TODO: destroy SSDataBlocks(msg) + + // For super table aggregation, the sma data is stored in vgroup calculated from the hash value of stable name. Thus + // the sma data would arrive ahead of the update-expired-window msg. + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + if (!pDataBlocks) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", SMA_VID(pSma)); + return terrno; + } + + if (taosArrayGetSize(pDataBlocks) <= 0) { + terrno = TSDB_CODE_INVALID_PARA; + smaWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + + if (!pItem || !(pItem = *(SSmaStatItem **)pItem) || tdSmaStatIsDropped(pItem)) { + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + + STSma *pTSma = pItem->pTSma; + STSmaWriteH tSmaH = {0}; + + if (tdInitTSmaWriteH(&tSmaH, pSma, pDataBlocks, pTSma->interval, pTSma->intervalUnit) != 0) { + return TSDB_CODE_FAILED; + } + + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; + snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); + tfsAbsoluteName(SMA_TFS(pSma), SMA_ENV_DID(pEnv), rPath, aPath); + if (!taosCheckExistFile(aPath)) { + if (tfsMkdirRecurAt(SMA_TFS(pSma), rPath, SMA_ENV_DID(pEnv)) != TSDB_CODE_SUCCESS) { + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } + + // Step 1: Judge the storage level and days + int32_t storageLevel = tdGetSmaStorageLevel(pCfg, tSmaH.interval); + int32_t minutePerFile = tdGetTSmaDays(pSma, tSmaH.interval, storageLevel); + + char smaKey[SMA_KEY_LEN] = {0}; // key: skey + groupId + char dataBuf[512] = {0}; // val: aggr data // TODO: handle 512 buffer? + void *pDataBuf = NULL; + int32_t sz = taosArrayGetSize(pDataBlocks); + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + for (int32_t j = 0; j < rows; ++j) { + printf("|"); + TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval + void *pSmaKey = &smaKey; + bool isStartKey = false; + + int32_t tlen = 0; // reset the len + pDataBuf = &dataBuf; // reset the buf + for (int32_t k = 0; k < colNum; ++k) { + SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + if (!isStartKey) { + isStartKey = true; + skey = *(TSKEY *)var; + testSkey = skey; + printf("= skey %" PRIi64 " groupId = %" PRIi64 "|", skey, groupId); + tdEncodeTSmaKey(groupId, skey, &pSmaKey); + } else { + printf(" %" PRIi64 " |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + } + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15d |", *(uint8_t *)var); + tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15d |", *(int8_t *)var); + tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15d |", *(int16_t *)var); + tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15d |", *(uint16_t *)var); + tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); + break; + case TSDB_DATA_TYPE_INT: + printf(" %15d |", *(int32_t *)var); + tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); + break; + case TSDB_DATA_TYPE_FLOAT: + printf(" %15f |", *(float *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(float)); + break; + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t *)var); + tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); + break; + case TSDB_DATA_TYPE_BIGINT: + printf(" %15ld |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf(" %15lf |", *(double *)var); + tlen += taosEncodeBinary(&pDataBuf, var, sizeof(double)); + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15lu |", *(uint64_t *)var); + tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); + break; + case TSDB_DATA_TYPE_NCHAR: { + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + // TODO: add binary/varbinary + TASSERT(0); + default: + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + break; + } + } + printf("\n"); + // if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { + if (tlen > 0) { + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, minutePerFile, pCfg->precision)); + + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index + // file + // - Set and open the DFile or the B+Tree file + // TODO: tsdbStartTSmaCommit(); + if (fid != tSmaH.dFile.fid) { + if (tSmaH.dFile.fid != SMA_IVLD_FID) { + tdSmaEndCommit(pEnv); + smaCloseDBF(&tSmaH.dFile); + } + tdSetTSmaDataFile(&tSmaH, indexUid, fid); + smaDebug("@@@ vgId:%d write to DBF %s, days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi32 + " queryKey:%" PRIi64, + SMA_VID(pSma), tSmaH.dFile.path, minutePerFile, tSmaH.interval, storageLevel, testSkey); + if (smaOpenDBF(pEnv->dbEnv, &tSmaH.dFile) != 0) { + smaWarn("vgId:%d open DB file %s failed since %s", SMA_VID(pSma), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + tdSmaBeginCommit(pEnv); + } + + if (tdInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, dataBuf, tlen, &pEnv->txn) != 0) { + smaWarn("vgId:%d insert tsma data blocks fail for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 + " since %s", + SMA_VID(pSma), indexUid, skey, groupId, tstrerror(terrno)); + tdSmaEndCommit(pEnv); + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + + smaDebug("vgId:%d insert tsma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, + SMA_VID(pSma), indexUid, skey, groupId); + // TODO:tsdbEndTSmaCommit(); + + // Step 3: reset the SSmaStat + tdResetExpiredWindow(pSma, pStat, indexUid, skey); + } else { + smaWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, + SMA_VID(pSma), skey, tlen, indexUid); + } + } + } + tdSmaEndCommit(pEnv); // TODO: not commit for every insert + tdDestroyTSmaWriteH(&tSmaH); + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} + +int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid) { + int32_t code = TSDB_CODE_SUCCESS; + if ((code = tdDropTSmaDataImpl(pSma, indexUid)) < 0) { + smaWarn("vgId:%d drop tSma data failed since %s", SMA_VID(pSma), tstrerror(terrno)); + } + return code; +} + +/** + * @brief Insert TSma data blocks to DB File build by B+Tree + * + * @param pSmaH + * @param smaKey tableUid-colId-skeyOfWindow(8-2-8) + * @param keyLen + * @param pData + * @param dataLen + * @return int32_t + */ +static int32_t tdInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, int32_t keyLen, void *pData, int32_t dataLen, + TXN *txn) { + SDBFile *pDBFile = &pSmaH->dFile; + + // TODO: insert tsma data blocks into B+Tree(TTB) + if (smaSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen, txn) != 0) { + smaWarn("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " fail", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d insert tsma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 " succeed", + SMA_VID(pSmaH->pSma), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + uint32_t valueSize = 0; + void *data = tdGetSmaDataByKey(pDBFile, smaKey, keyLen, &valueSize); + ASSERT(data != NULL); + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d insert sma data val[%d] %" PRIi64, REPO_ID(pSmaH->pTsdb), v, *(int64_t *)POINTER_SHIFT(data, v)); + } +#endif + return TSDB_CODE_SUCCESS; +} + +/** + * @brief When sma data received from stream computing, make the relative expired window valid. + * + * @param pSma + * @param pStat + * @param indexUid + * @param skey + * @return int32_t + */ +static int32_t tdResetExpiredWindow(SSma *pSma, SSmaStat *pStat, int64_t indexUid, TSKEY skey) { + SSmaStatItem *pItem = NULL; + + tdRefSmaStat(pSma, pStat); + + if (pStat && SMA_STAT_ITEMS(pStat)) { + pItem = taosHashGet(SMA_STAT_ITEMS(pStat), &indexUid, sizeof(indexUid)); + } + if ((pItem) && ((pItem = *(SSmaStatItem **)pItem))) { + // pItem resides in hash buffer all the time unless drop sma index + // TODO: multithread protect + if (taosHashRemove(pItem->expiredWindows, &skey, sizeof(TSKEY)) != 0) { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " fail", SMA_VID(pSma), skey, + indexUid); + return TSDB_CODE_FAILED; + } + smaDebug("vgId:%d remove skey %" PRIi64 " from expired window for sma index %" PRIi64 " succeed", SMA_VID(pSma), + skey, indexUid); + // TODO: use a standalone interface to received state upate notification from stream computing module. + /** + * @brief state + * - When SMA env init in TSDB, its status is TSDB_SMA_STAT_OK. + * - In startup phase of stream computing module, it should notify the SMA env in TSDB to expired if needed(e.g. + * when batch data caculation not finised) + * - When TSDB_SMA_STAT_OK, the stream computing module should also notify that to the SMA env in TSDB. + */ + pItem->state = TSDB_SMA_STAT_OK; + } else { + // error handling + tdUnRefSmaStat(pSma, pStat); + smaWarn("vgId:%d expired window %" PRIi64 " not exists for sma index %" PRIi64, SMA_VID(pSma), skey, indexUid); + return TSDB_CODE_FAILED; + } + + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Drop tSma data and local cache + * - insert/query reference + * @param pSma + * @param msg + * @return int32_t + */ +static int32_t tdDropTSmaDataImpl(SSma *pSma, int64_t indexUid) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + + // clear local cache + if (pEnv) { + smaDebug("vgId:%d drop tSma local cache for %" PRIi64, SMA_VID(pSma), indexUid); + + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if ((pItem) || ((pItem = *(SSmaStatItem **)pItem))) { + if (tdSmaStatIsDropped(pItem)) { + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + + tdWLockSmaEnv(pEnv); + if (tdSmaStatIsDropped(pItem)) { + tdUnLockSmaEnv(pEnv); + smaDebug("vgId:%d tSma stat is already dropped for %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_TDB_INVALID_ACTION; // TODO: duplicate drop msg would be intercepted by mnode + } + tdSmaStatSetDropped(pItem); + tdUnLockSmaEnv(pEnv); + + int32_t nSleep = 0; + int32_t refVal = INT32_MAX; + while (true) { + if ((refVal = T_REF_VAL_GET(SMA_ENV_STAT(pEnv))) <= 0) { + smaDebug("vgId:%d drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + break; + } + smaDebug("vgId:%d wait 1s to drop index %" PRIi64 " since refVal=%d", SMA_VID(pSma), indexUid, refVal); + taosSsleep(1); + if (++nSleep > SMA_DROP_EXPIRED_TIME) { + smaDebug("vgId:%d drop index %" PRIi64 " after wait %d (refVal=%d)", SMA_VID(pSma), indexUid, nSleep, refVal); + break; + }; + } + + tdFreeSmaStatItem(pItem); + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64 " in local cache", SMA_VID(pSma), indexUid); + } + } + // clear sma data files + // TODO: + return TSDB_CODE_SUCCESS; +} + +/** + * @brief + * + * @param pSma Return the data between queryWin and fill the pData. + * @param pData + * @param indexUid + * @param pQuerySKey + * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. + * @return int32_t + */ +int32_t tdGetTSmaDataImpl(SSma *pSma, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { + SSmaEnv *pEnv = atomic_load_ptr(&SMA_TSMA_ENV(pSma)); + SSmaStat *pStat = NULL; + + if (!pEnv) { + terrno = TSDB_CODE_INVALID_PTR; + smaWarn("vgId:%d getTSmaDataImpl failed since pTSmaEnv is NULL", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + pStat = SMA_ENV_STAT(pEnv); + + tdRefSmaStat(pSma, pStat); + SSmaStatItem *pItem = taosHashGet(SMA_ENV_STAT_ITEMS(pEnv), &indexUid, sizeof(indexUid)); + if (!pItem || !(pItem = *(SSmaStatItem **)pItem)) { + // Normally pItem should not be NULL, mark all windows as expired and notify query module to fetch raw TS data if + // it's NULL. + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + smaDebug("vgId:%d getTSmaDataImpl failed since no index %" PRIi64, SMA_VID(pSma), indexUid); + return TSDB_CODE_FAILED; + } + +#if 0 + int32_t nQueryWin = taosArrayGetSize(pQuerySKey); + for (int32_t n = 0; n < nQueryWin; ++n) { + TSKEY skey = taosArrayGet(pQuerySKey, n); + if (taosHashGet(pItem->expiredWindows, &skey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + } + } +#endif + +#if 1 + int8_t smaStat = 0; + if (!tdSmaStatIsOK(pItem, &smaStat)) { // TODO: multiple check for large scale sma query + tdUnRefSmaStat(pSma, pStat); + terrno = TSDB_CODE_TDB_INVALID_SMA_STAT; + smaWarn("vgId:%d getTSmaDataImpl failed from index %" PRIi64 " since %s %" PRIi8, SMA_VID(pSma), indexUid, + tstrerror(terrno), smaStat); + return TSDB_CODE_FAILED; + } + + if (taosHashGet(pItem->expiredWindows, &querySKey, sizeof(TSKEY))) { + // TODO: mark this window as expired. + smaDebug("vgId:%d skey %" PRIi64 " of window exists in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); + } else { + smaDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, SMA_VID(pSma), querySKey, + indexUid); + } + + STSma *pTSma = pItem->pTSma; +#endif + +#if 1 + STSmaReadH tReadH = {0}; + tdInitTSmaReadH(&tReadH, pSma, pTSma->interval, pTSma->intervalUnit); + smaCloseDBF(&tReadH.dFile); + + tdUnRefSmaStat(pSma, pStat); + + tdInitTSmaFile(&tReadH, indexUid, querySKey); + smaDebug("### vgId:%d read from DBF %s days:%d, interval:%" PRIi64 ", storageLevel:%" PRIi8 " queryKey:%" PRIi64, + SMA_VID(pSma), tReadH.dFile.path, tReadH.days, tReadH.interval, tReadH.storageLevel, querySKey); + if (smaOpenDBF(pEnv->dbEnv, &tReadH.dFile) != 0) { + smaWarn("vgId:%d open DBF %s failed since %s", SMA_VID(pSma), tReadH.dFile.path, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + int64_t queryGroupId = 0; + tdEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); + + smaDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", SMA_VID(pSma), tReadH.dFile.path, + *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); + + void *result = NULL; + int32_t valueSize = 0; + if (!(result = smaGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize))) { + smaWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", + SMA_VID(pSma), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_FAILED; + } +#endif + +#ifdef _TEST_SMA_PRINT_DEBUG_LOG_ + for (uint32_t v = 0; v < valueSize; v += 8) { + smaWarn("vgId:%d get sma data v[%d]=%" PRIi64, SMA_VID(pSma), v, *(int64_t *)POINTER_SHIFT(result, v)); + } +#endif + taosMemoryFreeClear(result); // TODO: fill the result to output + +#if 0 + int32_t nResult = 0; + int64_t lastKey = 0; + + while (true) { + if (nResult >= nMaxResult) { + break; + } + + // set and open the file according to the STSma param + if (tdSetAndOpenTSmaFile(&tReadH, queryWin)) { + char bTree[100] = "\0"; + while (strncmp(bTree, "has more nodes", 100) == 0) { + if (nResult >= nMaxResult) { + break; + } + // tdGetDataFromBTree(bTree, queryWin, lastKey) + // fill the pData + ++nResult; + } + } + } +#endif + // read data from file and fill the result + smaCloseDBF(&tReadH.dFile); + return TSDB_CODE_SUCCESS; +} + +int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { + SSmaCfg *pCfg = (SSmaCfg *)pMsg; + + if (metaCreateTSma(SMA_META(pSma), version, pCfg) < 0) { + return -1; + } + + tdTSmaAdd(pSma, 1); + return 0; +} + +int32_t tdDropTSma(SSma *pSma, char *pMsg) { +#if 0 + SVDropTSmaReq vDropSmaReq = {0}; + if (!tDeserializeSVDropTSmaReq(pMsg, &vDropSmaReq)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // TODO: send msg to stream computing to drop tSma + // if ((send msg to stream computing) < 0) { + // tDestroyTSma(&vCreateSmaReq); + // return -1; + // } + // + + if (metaDropTSma(SMA_META(pSma), vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + if (tdDropTSmaData(pSma, vDropSmaReq.indexUid) < 0) { + // TODO: handle error + return -1; + } + + tdTSmaSub(pSma, 1); +#endif + + // TODO: return directly or go on follow steps? + return TSDB_CODE_SUCCESS; +} + +static SSmaStatItem *tdNewSmaStatItem(int8_t state) { + SSmaStatItem *pItem = NULL; + + pItem = (SSmaStatItem *)taosMemoryCalloc(1, sizeof(SSmaStatItem)); + if (!pItem) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + pItem->state = state; + pItem->expiredWindows = taosHashInit(SMA_STATE_ITEM_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TIMESTAMP), + true, HASH_ENTRY_LOCK); + if (!pItem->expiredWindows) { + taosMemoryFreeClear(pItem); + return NULL; + } + + return pItem; +} + +static int32_t tdSetExpiredWindow(SSma *pSma, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey, + int64_t version) { + SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); + if (!pItem) { + // TODO: use TSDB_SMA_STAT_EXPIRED and update by stream computing later + pItem = tdNewSmaStatItem(TSDB_SMA_STAT_OK); // TODO use the real state + if (!pItem) { + // Response to stream computing: OOM + // For query, if the indexUid not found, the TSDB should tell query module to query raw TS data. + return TSDB_CODE_FAILED; + } + + // cache smaMeta + STSma *pTSma = metaGetSmaInfoByIndex(SMA_META(pSma), indexUid); + if (!pTSma) { + terrno = TSDB_CODE_TDB_NO_SMA_INDEX_IN_META; + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + smaWarn("vgId:%d set expire window, get tsma meta failed for smaIndex %" PRIi64 " since %s", SMA_VID(pSma), + indexUid, tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pItem->pTSma = pTSma; + + if (taosHashPut(pItemsHash, &indexUid, sizeof(indexUid), &pItem, sizeof(pItem)) != 0) { + // If error occurs during put smaStatItem, free the resources of pItem + taosHashCleanup(pItem->expiredWindows); + taosMemoryFree(pItem); + return TSDB_CODE_FAILED; + } + } else if (!(pItem = *(SSmaStatItem **)pItem)) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + if (taosHashPut(pItem->expiredWindows, &winSKey, sizeof(TSKEY), &version, sizeof(version)) != 0) { + // If error occurs during taosHashPut expired windows, remove the smaIndex from pSma->pSmaStat, thus TSDB would + // tell query module to query raw TS data. + // N.B. + // 1) It is assumed to be extemely little probability event of fail to taosHashPut. + // 2) This would solve the inconsistency to some extent, but not completely, unless we record all expired + // windows failed to put into hash table. + taosHashCleanup(pItem->expiredWindows); + taosMemoryFreeClear(pItem->pTSma); + taosHashRemove(pItemsHash, &indexUid, sizeof(indexUid)); + smaWarn("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window fail", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_FAILED; + } + + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window succeed", SMA_VID(pSma), indexUid, + winSKey); + return TSDB_CODE_SUCCESS; +} + +/** + * @brief Update expired window according to msg from stream computing module. + * + * @param pSma + * @param msg SSubmitReq + * @return int32_t + */ +int32_t tdUpdateExpiredWindowImpl(SSma *pSma, SSubmitReq *pMsg, int64_t version) { + // no time-range-sma, just return success + if (atomic_load_16(&SMA_TSMA_NUM(pSma)) <= 0) { + smaTrace("vgId:%d not update expire window since no tSma", SMA_VID(pSma)); + return TSDB_CODE_SUCCESS; + } + + if (!SMA_META(pSma)) { + terrno = TSDB_CODE_INVALID_PTR; + smaError("vgId:%d update expire window failed since no meta ptr", SMA_VID(pSma)); + return TSDB_CODE_FAILED; + } + + if (tdCheckAndInitSmaEnv(pSma, TSDB_SMA_TYPE_TIME_RANGE) < 0) { + smaError("vgId:%d init sma env failed since %s", SMA_VID(pSma), terrstr(terrno)); + terrno = TSDB_CODE_TDB_INIT_FAILED; + return TSDB_CODE_FAILED; + } + + // Firstly, assume that tSma can only be created on super table/normal table. + // getActiveTimeWindow + + SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); + SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); + + TASSERT(pEnv && pStat && pItemsHash); + + // basic procedure + // TODO: optimization + tdRefSmaStat(pSma, pStat); + + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock = NULL; + SInterval interval = {0}; + TSKEY lastWinSKey = INT64_MIN; + + if (tInitSubmitMsgIter(pMsg, &msgIter) < 0) { + return TSDB_CODE_FAILED; + } + + while (true) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (!pBlock) break; + + STSmaWrapper *pSW = NULL; + STSma *pTSma = NULL; + + SSubmitBlkIter blkIter = {0}; + if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) < 0) { + pSW = tFreeTSmaWrapper(pSW, false); + break; + } + + while (true) { + STSRow *row = tGetSubmitBlkNext(&blkIter); + if (!row) { + pSW = tFreeTSmaWrapper(pSW, false); + break; + } + if (!pSW || (pTSma && (pTSma->tableUid != msgIter.suid))) { + if (pSW) { + pSW = tFreeTSmaWrapper(pSW, false); + } + if (!(pSW = metaGetSmaInfoByTable(SMA_META(pSma), msgIter.suid, false))) { + break; + } + if ((pSW->number) <= 0 || !pSW->tSma) { + pSW = tFreeTSmaWrapper(pSW, false); + break; + } + + pTSma = pSW->tSma; + + interval.interval = pTSma->interval; + interval.intervalUnit = pTSma->intervalUnit; + interval.offset = pTSma->offset; + interval.precision = SMA_TSDB_CFG(pSma)->precision; + interval.sliding = pTSma->sliding; + interval.slidingUnit = pTSma->slidingUnit; + } + + // TODO: process multiple tsma for one table uid + TSKEY winSKey = taosTimeTruncate(TD_ROW_KEY(row), &interval, interval.precision); + + if (lastWinSKey != winSKey) { + lastWinSKey = winSKey; + if (tdSetExpiredWindow(pSma, pItemsHash, pTSma->indexUid, winSKey, version) < 0) { + pSW = tFreeTSmaWrapper(pSW, false); + tdUnRefSmaStat(pSma, pStat); + return TSDB_CODE_FAILED; + } + } else { + smaDebug("vgId:%d smaIndex %" PRIi64 ", put skey %" PRIi64 " to expire window ignore as duplicated", + SMA_VID(pSma), pTSma->indexUid, winSKey); + } + } + } + + tdUnRefSmaStat(pSma, pStat); + + return TSDB_CODE_SUCCESS; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 93f305ba77..55ebc03d64 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -229,7 +229,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) } memcpy(data, msg, msgLen); - tqProcessStreamTrigger(pTq, data); + // tqProcessStreamTrigger(pTq, data); } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 45b17a0180..e77cc156d4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -515,7 +515,7 @@ static SSmaStatItem *tsdbNewSmaStatItem(int8_t state) { static void *tsdbFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { if (pSmaStatItem) { - tdDestroyTSma(pSmaStatItem->pSma); + tDestroyTSma(pSmaStatItem->pSma); taosMemoryFreeClear(pSmaStatItem->pSma); taosHashCleanup(pSmaStatItem->expiredWindows); taosMemoryFreeClear(pSmaStatItem); @@ -718,25 +718,25 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, SSubmitReq *pMsg, int64_t vers SSubmitBlkIter blkIter = {0}; if (tInitSubmitBlkIter(&msgIter, pBlock, &blkIter) != TSDB_CODE_SUCCESS) { - pSW = tdFreeTSmaWrapper(pSW); + pSW = tFreeTSmaWrapper(pSW); break; } while (true) { STSRow *row = tGetSubmitBlkNext(&blkIter); if (!row) { - tdFreeTSmaWrapper(pSW); + tFreeTSmaWrapper(pSW); break; } if (!pSW || (pTSma->tableUid != pBlock->suid)) { if (pSW) { - pSW = tdFreeTSmaWrapper(pSW); + pSW = tFreeTSmaWrapper(pSW); } if (!(pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid))) { break; } if ((pSW->number) <= 0 || !pSW->tSma) { - pSW = tdFreeTSmaWrapper(pSW); + pSW = tFreeTSmaWrapper(pSW); break; } @@ -1649,13 +1649,13 @@ int32_t tsdbCreateTSma(STsdb *pTsdb, char *pMsg) { // TODO: handle error tsdbWarn("vgId:%d tsma %s:%" PRIi64 " create failed for table %" PRIi64 " since %s", REPO_ID(pTsdb), vCreateSmaReq.tSma.indexName, vCreateSmaReq.tSma.indexUid, vCreateSmaReq.tSma.tableUid, terrstr(terrno)); - tdDestroyTSma(&vCreateSmaReq.tSma); + tDestroyTSma(&vCreateSmaReq.tSma); return -1; } tsdbTSmaAdd(pTsdb, 1); - tdDestroyTSma(&vCreateSmaReq.tSma); + tDestroyTSma(&vCreateSmaReq.tSma); // TODO: return directly or go on follow steps? return TSDB_CODE_SUCCESS; } @@ -1669,7 +1669,7 @@ int32_t tsdbDropTSma(STsdb *pTsdb, char *pMsg) { // TODO: send msg to stream computing to drop tSma // if ((send msg to stream computing) < 0) { - // tdDestroyTSma(&vCreateSmaReq); + // tDestroyTSma(&vCreateSmaReq); // return -1; // } // diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 4d2741f751..3b8c94e413 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -147,8 +147,8 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { // resource release taosMemoryFreeClear(pSW); - tdDestroyTSma(&tSma); - tdDestroyTSmaWrapper(&dstTSmaWrapper); + tDestroyTSma(&tSma); + tDestroyTSmaWrapper(&dstTSmaWrapper); } #endif @@ -218,7 +218,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); - tdDestroyTSma(qSmaCfg); + tDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); qSmaCfg = metaGetSmaInfoByIndex(pMeta, indexUid2, true); @@ -229,7 +229,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); EXPECT_EQ(qSmaCfg->interval, tSma.interval); - tdDestroyTSma(qSmaCfg); + tDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); // get index name by table uid @@ -265,7 +265,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2); EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid); - tdDestroyTSmaWrapper(pSW); + tDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); // get all sma table uids @@ -282,7 +282,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { metaRemoveSmaFromDb(pMeta, indexUid1); metaRemoveSmaFromDb(pMeta, indexUid2); - tdDestroyTSma(&tSma); + tDestroyTSma(&tSma); metaClose(pMeta); } #endif @@ -576,7 +576,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { taosArrayDestroy(pDataBlocks); // release meta - tdDestroyTSma(&tSma); + tDestroyTSma(&tSma); tfsClose(pTsdb->pTfs); tsdbClose(pTsdb); metaClose(pMeta); From ea51d91bc15a01fa01ab89bfba815b445fb5e3e7 Mon Sep 17 00:00:00 2001 From: jiajingbin Date: Wed, 1 Jun 2022 20:19:11 +0800 Subject: [PATCH 18/47] test: update schemaless-testcases for schemaless-database --- .../system-test/1-insert/influxdb_line_taosc_insert.py | 4 ++-- .../system-test/1-insert/opentsdb_json_taosc_insert.py | 10 +++++----- .../1-insert/opentsdb_telnet_line_taosc_insert.py | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/system-test/1-insert/influxdb_line_taosc_insert.py b/tests/system-test/1-insert/influxdb_line_taosc_insert.py index 75e0c3d59c..0ddeba4665 100644 --- a/tests/system-test/1-insert/influxdb_line_taosc_insert.py +++ b/tests/system-test/1-insert/influxdb_line_taosc_insert.py @@ -33,10 +33,10 @@ class TDTestCase: def createDb(self, name="test", db_update_tag=0): if db_update_tag == 0: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms'") + tdSql.execute(f"create database if not exists {name} precision 'ms' schemaless 1") else: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms' update 1") + tdSql.execute(f"create database if not exists {name} precision 'ms' update 1 schemaless 1") tdSql.execute(f'use {name}') def timeTrans(self, time_value, ts_type): diff --git a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py index 360d878dfe..f9bc5bbaf4 100644 --- a/tests/system-test/1-insert/opentsdb_json_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_json_taosc_insert.py @@ -36,10 +36,10 @@ class TDTestCase: if db_update_tag == 0: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms'") + tdSql.execute(f"create database if not exists {name} precision 'ms' schemaless 1") else: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ms' update 1") + tdSql.execute(f"create database if not exists {name} precision 'ms' update 1 schemaless 1") tdSql.execute(f'use {name}') def timeTrans(self, time_value): @@ -535,7 +535,7 @@ class TDTestCase: # check result #! bug tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'ms'") + tdSql.execute(f"create database if not exists test_ts precision 'ms' schemaless 1") tdSql.execute("use test_ts") input_json = [{"metric": "test_ms", "timestamp": {"value": 1626006833640, "type": "ms"}, "value": True, "tags": {"t0": True}}, {"metric": "test_ms", "timestamp": {"value": 1626006833641, "type": "ms"}, "value": False, "tags": {"t0": True}}] @@ -545,7 +545,7 @@ class TDTestCase: tdSql.checkEqual(str(res[1][0]), "2021-07-11 20:33:53.641000") tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'us'") + tdSql.execute(f"create database if not exists test_ts precision 'us' schemaless 1") tdSql.execute("use test_ts") input_json = [{"metric": "test_us", "timestamp": {"value": 1626006833639000, "type": "us"}, "value": True, "tags": {"t0": True}}, {"metric": "test_us", "timestamp": {"value": 1626006833639001, "type": "us"}, "value": False, "tags": {"t0": True}}] @@ -555,7 +555,7 @@ class TDTestCase: tdSql.checkEqual(str(res[1][0]), "2021-07-11 20:33:53.639001") tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'ns'") + tdSql.execute(f"create database if not exists test_ts precision 'ns' schemaless 1") tdSql.execute("use test_ts") input_json = [{"metric": "test_ns", "timestamp": {"value": 1626006833639000000, "type": "ns"}, "value": True, "tags": {"t0": True}}, {"metric": "test_ns", "timestamp": {"value": 1626006833639000001, "type": "ns"}, "value": False, "tags": {"t0": True}}] diff --git a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py index 6d50dd4fda..23404330ed 100644 --- a/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py +++ b/tests/system-test/1-insert/opentsdb_telnet_line_taosc_insert.py @@ -36,10 +36,10 @@ class TDTestCase: if db_update_tag == 0: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'us'") + tdSql.execute(f"create database if not exists {name} precision 'us' schemaless 1") else: tdSql.execute(f"drop database if exists {name}") - tdSql.execute(f"create database if not exists {name} precision 'ns' update 1") + tdSql.execute(f"create database if not exists {name} precision 'ns' update 1 schemaless 1") tdSql.execute(f'use {name}') def timeTrans(self, time_value, ts_type): @@ -393,7 +393,7 @@ class TDTestCase: self.resCmp(input_sql, stb_name, ts_type=TDSmlTimestampType.SECOND.value) tdSql.execute(f"drop database if exists test_ts") - tdSql.execute(f"create database if not exists test_ts precision 'ms'") + tdSql.execute(f"create database if not exists test_ts precision 'ms' schemaless 1") tdSql.execute("use test_ts") input_sql = ['test_ms 1626006833640 t t0=t', 'test_ms 1626006833641 f t0=t'] self._conn.schemaless_insert(input_sql, TDSmlProtocolType.TELNET.value, None) From d8093c91c227e775419a364d88ea1e82dda2f91a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 1 Jun 2022 20:28:29 +0800 Subject: [PATCH 19/47] alter table return meta --- include/common/tmsg.h | 7 +- include/libs/catalog/catalog.h | 2 +- source/client/inc/clientInt.h | 3 +- source/client/src/clientEnv.c | 27 ++++ source/client/src/clientHb.c | 2 +- source/client/src/clientImpl.c | 169 ++++++++++++---------- source/client/src/clientMsgHandler.c | 22 +++ source/common/src/tmsg.c | 84 +++++++++++ source/dnode/mnode/impl/src/mndStb.c | 50 +++++++ source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/meta/metaTable.c | 26 +++- source/dnode/vnode/src/vnd/vnodeSvr.c | 27 +++- source/libs/catalog/inc/catalogInt.h | 2 +- source/libs/catalog/src/catalog.c | 57 +++++--- source/libs/catalog/src/ctgCache.c | 24 +-- source/libs/catalog/test/catalogTests.cpp | 2 +- source/libs/qcom/src/querymsg.c | 10 +- source/libs/scheduler/src/schRemote.c | 2 + 18 files changed, 387 insertions(+), 131 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a679cc11cd..9800c74dd1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1131,6 +1131,10 @@ typedef struct { SSchema* pSchemas; } STableMetaRsp; +typedef struct { + STableMetaRsp* pMeta; +} SMAlterStbRsp; + int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); void tFreeSTableMetaRsp(STableMetaRsp* pRsp); @@ -1875,7 +1879,8 @@ int32_t tEncodeSVAlterTbReq(SEncoder* pEncoder, const SVAlterTbReq* pReq); int32_t tDecodeSVAlterTbReq(SDecoder* pDecoder, SVAlterTbReq* pReq); typedef struct { - int32_t code; + int32_t code; + STableMetaRsp* pMeta; } SVAlterTbRsp; int32_t tEncodeSVAlterTbRsp(SEncoder* pEncoder, const SVAlterTbRsp* pRsp); diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 8027b9394e..f0e642bc9a 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -183,7 +183,7 @@ int32_t catalogGetTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSe */ int32_t catalogGetSTableMeta(SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); -int32_t catalogUpdateSTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); +int32_t catalogUpdateTableMeta(SCatalog* pCatalog, STableMetaRsp *rspMsg); /** diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c5fa377fea..91f9874b23 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -162,6 +162,7 @@ typedef struct SResultColumn { } SResultColumn; typedef struct SReqResultInfo { + void* pExecRes; const char* pRspMsg; const char* pData; TAOS_FIELD* fields; // todo, column names are not needed. @@ -317,7 +318,7 @@ void hbMgrInitMqHbRspHandle(); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res); int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList); -int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList, void** res); +int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); int32_t updateQnodeList(SAppInstInfo*pInfo, SArray* pNodeList); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 669b2bc97e..777f27d29d 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -212,6 +212,31 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) { } } +static void destroyExecRes(SRequestObj* pRequest) { + if (NULL == pRequest || NULL == pRequest->body.resInfo.pExecRes) { + return; + } + + switch (pRequest->type) { + case TDMT_VND_ALTER_TABLE: + case TDMT_MND_ALTER_STB: { + tFreeSTableMetaRsp((STableMetaRsp *)pRequest->body.resInfo.pExecRes); + taosMemoryFree(pRequest->body.resInfo.pExecRes); + break; + } + case TDMT_VND_SUBMIT: { + tFreeSSubmitRsp((SSubmitRsp*)pRequest->body.resInfo.pExecRes); + break; + } + case TDMT_VND_QUERY: { + taosArrayDestroy((SArray*)pRequest->body.resInfo.pExecRes); + break; + } + default: + tscError("invalid exec result for request type %d", pRequest->type); + } +} + static void doDestroyRequest(void *p) { assert(p != NULL); SRequestObj *pRequest = (SRequestObj *)p; @@ -234,6 +259,8 @@ static void doDestroyRequest(void *p) { taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); + destroyExecRes(pRequest); + deregisterRequest(pRequest); taosMemoryFreeClear(pRequest); } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 70b60195d2..09c3d269c7 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -120,7 +120,7 @@ static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalo return TSDB_CODE_TSC_INVALID_VALUE; } - catalogUpdateSTableMeta(pCatalog, rsp); + catalogUpdateTableMeta(pCatalog, rsp); } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 53ee592945..123afa0f7c 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -386,18 +386,20 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod } -int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList, void** pRes) { +int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; SQueryResult res = {.code = 0, .numOfRows = 0}; int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, &res); + + pRequest->body.resInfo.pExecRes = res.res; + if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); } - *pRes = res.res; pRequest->code = code; terrno = code; @@ -412,8 +414,6 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList } } - *pRes = res.res; - pRequest->code = res.code; terrno = res.code; return pRequest->code; @@ -423,80 +423,104 @@ int32_t getQueryPlan(SRequestObj* pRequest, SQuery* pQuery, SArray** pNodeList) return getPlan(pRequest, pQuery, &pRequest->body.pDag, pNodeList); } -int32_t validateSversion(SRequestObj* pRequest, void* res) { - SArray* pArray = NULL; +int32_t handleSubmitExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { int32_t code = 0; - - if (TDMT_VND_SUBMIT == pRequest->type) { - SSubmitRsp* pRsp = (SSubmitRsp*)res; - if (pRsp->nBlocks <= 0) { - return TSDB_CODE_SUCCESS; - } - - pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); - if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < pRsp->nBlocks; ++i) { - SSubmitBlkRsp* blk = pRsp->pBlocks + i; - if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { - continue; - } - - STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; - taosArrayPush(pArray, &tbSver); - } - } else if (TDMT_VND_QUERY == pRequest->type) { - SArray* pTbArray = (SArray*)res; - int32_t tbNum = taosArrayGetSize(pTbArray); - if (tbNum <= 0) { - return TSDB_CODE_SUCCESS; - } - - pArray = taosArrayInit(tbNum, sizeof(STbSVersion)); - if (NULL == pArray) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < tbNum; ++i) { - STbVerInfo* tbInfo = taosArrayGet(pTbArray, i); - STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion, .tver = tbInfo->tversion}; - taosArrayPush(pArray, &tbSver); + SArray* pArray = NULL; + SSubmitRsp* pRsp = (SSubmitRsp*)res; + if (pRsp->nBlocks <= 0) { + return TSDB_CODE_SUCCESS; + } + + pArray = taosArrayInit(pRsp->nBlocks, sizeof(STbSVersion)); + if (NULL == pArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < pRsp->nBlocks; ++i) { + SSubmitBlkRsp* blk = pRsp->pBlocks + i; + if (NULL == blk->tblFName || 0 == blk->tblFName[0]) { + continue; } + + STbSVersion tbSver = {.tbFName = blk->tblFName, .sver = blk->sver}; + taosArrayPush(pArray, &tbSver); } - SCatalog* pCatalog = NULL; - CHECK_CODE_GOTO(catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog), _return); - - SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - - code = catalogChkTbMetaVersion(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, &epset, pArray); + code = catalogChkTbMetaVersion(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, epset, pArray); _return: taosArrayDestroy(pArray); + return code; +} + +int32_t handleQueryExecRes(SRequestObj* pRequest, void* res, SCatalog* pCatalog, SEpSet *epset) { + int32_t code = 0; + SArray* pArray = NULL; + SArray* pTbArray = (SArray*)res; + int32_t tbNum = taosArrayGetSize(pTbArray); + if (tbNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + pArray = taosArrayInit(tbNum, sizeof(STbSVersion)); + if (NULL == pArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int32_t i = 0; i < tbNum; ++i) { + STbVerInfo* tbInfo = taosArrayGet(pTbArray, i); + STbSVersion tbSver = {.tbFName = tbInfo->tbFName, .sver = tbInfo->sversion, .tver = tbInfo->tversion}; + taosArrayPush(pArray, &tbSver); + } + + code = catalogChkTbMetaVersion(pCatalog, pRequest->pTscObj->pAppInfo->pTransporter, epset, pArray); + +_return: + + taosArrayDestroy(pArray); + return code; +} + +int32_t handleAlterTbExecRes(void* res, SCatalog* pCatalog) { + return catalogUpdateTableMeta(pCatalog, (STableMetaRsp*)res); +} + +int32_t handleExecRes(SRequestObj* pRequest) { + int32_t code = 0; + SCatalog* pCatalog = NULL; + code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); + if (code) { + return code; + } + + SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); + + switch (pRequest->type) { + case TDMT_VND_ALTER_TABLE: + case TDMT_MND_ALTER_STB: { + code = handleAlterTbExecRes(pRequest->body.resInfo.pExecRes, pCatalog); + break; + } + case TDMT_VND_SUBMIT: { + code = handleSubmitExecRes(pRequest, pRequest->body.resInfo.pExecRes, pCatalog, &epset); + break; + } + case TDMT_VND_QUERY: { + code = handleQueryExecRes(pRequest, pRequest->body.resInfo.pExecRes, pCatalog, &epset); + break; + } + default: + tscError("invalid exec result for request type %d", pRequest->type); + return TSDB_CODE_APP_ERROR; + } return code; } -void freeRequestRes(SRequestObj* pRequest, void* res) { - if (NULL == pRequest || NULL == res) { - return; - } - - if (TDMT_VND_SUBMIT == pRequest->type) { - tFreeSSubmitRsp((SSubmitRsp*)res); - } else if (TDMT_VND_QUERY == pRequest->type) { - taosArrayDestroy((SArray*)res); - } -} - SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code, bool keepQuery, void** res) { - void* pRes = NULL; - if (TSDB_CODE_SUCCESS == code) { switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: @@ -509,10 +533,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code SArray* pNodeList = NULL; code = getPlan(pRequest, pQuery, &pRequest->body.pDag, &pNodeList); if (TSDB_CODE_SUCCESS == code) { - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList, &pRes); - if (NULL != pRes) { - code = validateSversion(pRequest, pRes); - } + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); } taosArrayDestroy(pNodeList); break; @@ -529,15 +550,17 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code qDestroyQuery(pQuery); } + if (NULL != pRequest->body.resInfo.pExecRes) { + handleExecRes(pRequest); + } + if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } if (res) { - *res = pRes; - } else { - freeRequestRes(pRequest, pRes); - pRes = NULL; + *res = pRequest->body.resInfo.pExecRes; + pRequest->body.resInfo.pExecRes = NULL; } return pRequest; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index f15315fe60..e48770e2a1 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -223,10 +223,32 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } +int32_t processAlterStbRsp(void* param, const SDataBuf* pMsg, int32_t code) { + SRequestObj* pRequest = param; + if (code != TSDB_CODE_SUCCESS) { + setErrno(pRequest, code); + tsem_post(&pRequest->body.rspSem); + return code; + } + + SMAlterStbRsp alterRsp = {0}; + SDecoder coder = {0}; + tDecoderInit(&coder, pMsg->pData, pMsg->len); + tDecodeSMAlterStbRsp(&coder, &alterRsp); + tDecoderClear(&coder); + + pRequest->body.resInfo.pExecRes = alterRsp.pMeta; + + tsem_post(&pRequest->body.rspSem); + return code; +} + + void initMsgHandleFp() { handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp; handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp; + handleRequestRspFp[TMSG_INDEX(TDMT_MND_ALTER_STB)] = processAlterStbRsp; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 233623c616..179631d8c4 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -694,6 +694,7 @@ void tFreeSMAltertbReq(SMAlterStbReq *pReq) { pReq->pFields = NULL; } + int32_t tSerializeSEpSet(void *buf, int32_t bufLen, const SEpSet *pEpset) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -4348,13 +4349,96 @@ int32_t tDecodeSVAlterTbReq(SDecoder *pDecoder, SVAlterTbReq *pReq) { int32_t tEncodeSVAlterTbRsp(SEncoder *pEncoder, const SVAlterTbRsp *pRsp) { if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->code) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->pMeta ? 1 : 0) < 0) return -1; + if (pRsp->pMeta) { + if (tEncodeSTableMetaRsp(pEncoder, pRsp->pMeta) < 0) return -1; + } tEndEncode(pEncoder); return 0; } int32_t tDecodeSVAlterTbRsp(SDecoder *pDecoder, SVAlterTbRsp *pRsp) { + int32_t meta = 0; if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->code) < 0) return -1; + if (tDecodeI32(pDecoder, &meta) < 0) return -1; + if (meta) { + pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + if (NULL == pRsp->pMeta) return -1; + if (tDecodeSTableMetaRsp(pDecoder, pRsp->pMeta) < 0) return -1; + } tEndDecode(pDecoder); return 0; } + +int32_t tDeserializeSVAlterTbRsp(void *buf, int32_t bufLen, SVAlterTbRsp *pRsp) { + int32_t meta = 0; + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->code) < 0) return -1; + if (tDecodeI32(&decoder, &meta) < 0) return -1; + if (meta) { + pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + if (NULL == pRsp->pMeta) return -1; + if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; + } + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->pMeta->pSchemas ? 1 : 0) < 0) return -1; + if (pRsp->pMeta->pSchemas) { + if (tEncodeSTableMetaRsp(pEncoder, pRsp->pMeta) < 0) return -1; + } + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp) { + int32_t meta = 0; + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI32(pDecoder, &meta) < 0) return -1; + if (meta) { + pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + if (NULL == pRsp->pMeta) return -1; + if (tDecodeSTableMetaRsp(pDecoder, pRsp->pMeta) < 0) return -1; + } + tEndDecode(pDecoder); + return 0; +} + +int32_t tDeserializeSMAlterStbRsp(void *buf, int32_t bufLen, SMAlterStbRsp *pRsp) { + int32_t meta = 0; + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &meta) < 0) return -1; + if (meta) { + pRsp->pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + if (NULL == pRsp->pMeta) return -1; + if (tDecodeSTableMetaRsp(&decoder, pRsp->pMeta) < 0) return -1; + } + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp) { + if (NULL == pRsp) { + return; + } + + if (pRsp->pMeta) { + taosMemoryFree(pRsp->pMeta->pSchemas); + taosMemoryFree(pRsp->pMeta); + } +} + + + diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 9ca7613519..e691e0341a 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1207,13 +1207,54 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } +static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { + int ret; + SEncoder ec = {0}; + uint32_t contLen = 0; + SMAlterStbRsp alterRsp = {0}; + SName name = {0}; + tNameFromString(&name, pAlter->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + alterRsp.pMeta = taosMemoryCalloc(1, sizeof(STableMetaRsp)); + if (NULL == alterRsp.pMeta) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, &alterRsp.meta); + if (ret) { + tFreeSMAlterStbRsp(&alterRsp); + return ret; + } + + tEncodeSize(tEncodeSMAlterStbRsp, &alterRsp, contLen, ret); + if (ret) { + tFreeSMAlterStbRsp(&alterRsp); + return ret; + } + + void* cont = taosMemoryMalloc(contLen); + tEncoderInit(&ec, cont, contLen); + tEncodeSMAlterStbRsp(&ec, &alterRsp); + tEncoderClear(&ec); + + tFreeSMAlterStbRsp(&alterRsp); + + *pCont = cont; + *pLen = contLen; + + return 0; +} + static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { + bool needRsp = true; SStbObj stbObj = {0}; taosRLockLatch(&pOld->lock); memcpy(&stbObj, pOld, sizeof(SStbObj)); stbObj.pColumns = NULL; stbObj.pTags = NULL; stbObj.updateTime = taosGetTimestampMs(); + stbObj.lock = 0; taosRUnLockLatch(&pOld->lock); int32_t code = -1; @@ -1247,9 +1288,11 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p code = mndAlterStbColumnBytes(pOld, &stbObj, pField0); break; case TSDB_ALTER_TABLE_UPDATE_OPTIONS: + needRsp = false; code = mndUpdateStbCommentAndTTL(pOld, &stbObj, pAlter->comment, pAlter->commentLen, pAlter->ttl); break; default: + needRsp = false; terrno = TSDB_CODE_OPS_NOT_SUPPORT; break; } @@ -1263,6 +1306,13 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p mDebug("trans:%d, used to alter stb:%s", pTrans->id, pAlter->name); mndTransSetDbInfo(pTrans, pDb); + if (needRsp) { + void* pCont = NULL; + int32_t contLen = 0; + if (mndBuildSMAlterStbRsp(pDb, pAlter, &stbObj, &pCont, &contLen)) goto _OVER; + mndTransSetRpcRsp(pTrans, pCont, contLen); + } + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, &stbObj) != 0) goto _OVER; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index e3a0c94ccc..d2c73316a1 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -87,7 +87,7 @@ int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* p int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq); int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq); int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids); -int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq); +int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp *pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); STSchema* metaGetTbTSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver); int metaGetTableEntryByName(SMetaReader* pReader, const char* name); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 665d571c8f..b588ea3203 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -25,6 +25,24 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); +static int metaUpdateMetaRsp(tb_uid_t uid, char* tbName, SSchemaWrapper *pSchema, STableMetaRsp *pMetaRsp) { + pMetaRsp->pSchemas = taosMemoryMalloc(pSchema->nCols * sizeof(SSchema)); + if (NULL == pMetaRsp->pSchemas) { + terrno = TSDB_CODE_VND_OUT_OF_MEMORY; + return -1; + } + + strcpy(pMetaRsp->tbName, tbName); + pMetaRsp->numOfColumns = pSchema->nCols; + pMetaRsp->tableType = TSDB_NORMAL_TABLE; + pMetaRsp->sversion = pSchema->version; + pMetaRsp->tuid = uid; + + memcpy(pMetaRsp->pSchemas, pSchema->pSchema, pSchema->nCols * sizeof(SSchema)); + + return 0; +} + int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { SMetaEntry me = {0}; int kLen = 0; @@ -323,7 +341,7 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { return 0; } -static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { +static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq, STableMetaRsp *pMetaRsp) { void * pVal = NULL; int nVal = 0; const void * pData = NULL; @@ -463,6 +481,8 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl metaULock(pMeta); + metaUpdateMetaRsp(uid, pAlterTbReq->tbName, pSchema, pMetaRsp); + if (pNewSchema) taosMemoryFree(pNewSchema); tDecoderClear(&dc); tdbTbcClose(pTbDbc); @@ -619,13 +639,13 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p return 0; } -int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq) { +int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: case TSDB_ALTER_TABLE_DROP_COLUMN: case TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES: case TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME: - return metaAlterTableColumn(pMeta, version, pReq); + return metaAlterTableColumn(pMeta, version, pReq, pMetaRsp); case TSDB_ALTER_TABLE_UPDATE_TAG_VAL: return metaUpdateTableTagVal(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_OPTIONS: diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 4b237bc703..3ab5789eae 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -248,6 +248,13 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { tdProcessTSmaInsert(((SVnode *)pVnode)->pSma, smaId, (const char *)data); } +void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { + strcpy(pMetaRsp->dbFName, pVnode->config.dbname); + pMetaRsp->dbId = pVnode->config.dbId; + pMetaRsp->vgId = TD_VID(pVnode); + pMetaRsp->precision = pVnode->config.tsdbCfg.precision; +} + int vnodeProcessSyncReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { int32_t ret = TAOS_SYNC_PROPOSE_OTHER_ERROR; @@ -517,12 +524,13 @@ _exit: } static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SVAlterTbReq vAlterTbReq = {0}; - SVAlterTbRsp vAlterTbRsp = {0}; - SDecoder dc = {0}; - int rcode = 0; - int ret; - SEncoder ec = {0}; + SVAlterTbReq vAlterTbReq = {0}; + SVAlterTbRsp vAlterTbRsp = {0}; + SDecoder dc = {0}; + int rcode = 0; + int ret; + SEncoder ec = {0}; + STableMetaRsp vMetaRsp = {0}; pRsp->msgType = TDMT_VND_ALTER_TABLE_RSP; pRsp->pCont = NULL; @@ -540,7 +548,7 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, i } // process - if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq) < 0) { + if (metaAlterTable(pVnode->pMeta, version, &vAlterTbReq, &vMetaRsp) < 0) { vAlterTbRsp.code = TSDB_CODE_INVALID_MSG; tDecoderClear(&dc); rcode = -1; @@ -548,6 +556,11 @@ static int vnodeProcessAlterTbReq(SVnode *pVnode, int64_t version, void *pReq, i } tDecoderClear(&dc); + if (NULL != vMetaRsp.pSchemas) { + vnodeUpdateMetaRsp(pVnode, &vMetaRsp); + vAlterTbRsp.pMeta = &vMetaRsp; + } + _exit: tEncodeSize(tEncodeSVAlterTbRsp, &vAlterTbRsp, pRsp->contLen, ret); pRsp->pCont = rpcMallocCont(pRsp->contLen); diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 57f651ed69..239d719fa8 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -302,7 +302,7 @@ typedef struct SCtgUpdateEpsetMsg { typedef struct SCtgCacheOperation { int32_t opId; void *data; - bool syncReq; + bool syncOp; uint64_t seqId; } SCtgCacheOperation; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index a874764468..7e0efe22db 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -314,6 +314,36 @@ _return: CTG_RET(code); } +int32_t ctgUpdateTbMeta(SCatalog* pCtg, STableMetaRsp *rspMsg, bool syncOp) { + STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); + if (NULL == output) { + ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + int32_t code = 0; + + strcpy(output->dbFName, rspMsg->dbFName); + strcpy(output->tbName, rspMsg->tbName); + + output->dbId = rspMsg->dbId; + + SET_META_TYPE_TABLE(output->metaType); + + CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, rspMsg->tableType == TSDB_SUPER_TABLE, &output->tbMeta)); + + CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, syncOp)); + + return TSDB_CODE_SUCCESS; + +_return: + + taosMemoryFreeClear(output->tbMeta); + taosMemoryFreeClear(output); + + CTG_RET(code); +} + int32_t ctgChkAuth(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { bool inCache = false; @@ -779,38 +809,17 @@ int32_t catalogGetSTableMeta(SCatalog* pCtg, void * pTrans, const SEpSet* pMgmtE CTG_API_LEAVE(ctgGetTbMeta(CTG_PARAMS_LIST(), &ctx, pTableMeta)); } -int32_t catalogUpdateSTableMeta(SCatalog* pCtg, STableMetaRsp *rspMsg) { +int32_t catalogUpdateTableMeta(SCatalog* pCtg, STableMetaRsp *pMsg) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == rspMsg) { + if (NULL == pCtg || NULL == pMsg) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - STableMetaOutput *output = taosMemoryCalloc(1, sizeof(STableMetaOutput)); - if (NULL == output) { - ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); - CTG_API_LEAVE(TSDB_CODE_CTG_MEM_ERROR); - } - int32_t code = 0; - - strcpy(output->dbFName, rspMsg->dbFName); - strcpy(output->tbName, rspMsg->tbName); - - output->dbId = rspMsg->dbId; - - SET_META_TYPE_TABLE(output->metaType); - - CTG_ERR_JRET(queryCreateTableMetaFromMsg(rspMsg, true, &output->tbMeta)); - - CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, output, false)); - - CTG_API_LEAVE(code); + CTG_ERR_JRET(ctgUpdateTbMeta(pCtg, pMsg, true)); _return: - - taosMemoryFreeClear(output->tbMeta); - taosMemoryFreeClear(output); CTG_API_LEAVE(code); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 2fbb8b499d..0f1344c343 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -552,7 +552,7 @@ int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) { ctgDebug("action [%s] added into queue", gCtgCacheOperation[operation->opId].name); - if (operation->syncReq) { + if (operation->syncOp) { ctgWaitOpDone(operation); } @@ -591,9 +591,9 @@ _return: } -int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncReq) { +int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncOp) { int32_t code = 0; - SCtgCacheOperation action= {.opId = CTG_OP_DROP_STB_META, .syncReq = syncReq}; + SCtgCacheOperation action= {.opId = CTG_OP_DROP_STB_META, .syncOp = syncOp}; SCtgRemoveStbMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveStbMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveStbMsg)); @@ -620,9 +620,9 @@ _return: -int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncReq) { +int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncOp) { int32_t code = 0; - SCtgCacheOperation action= {.opId = CTG_OP_DROP_TB_META, .syncReq = syncReq}; + SCtgCacheOperation action= {.opId = CTG_OP_DROP_TB_META, .syncOp = syncOp}; SCtgRemoveTblMsg *msg = taosMemoryMalloc(sizeof(SCtgRemoveTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgRemoveTblMsg)); @@ -646,9 +646,9 @@ _return: CTG_RET(code); } -int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncReq) { +int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncOp) { int32_t code = 0; - SCtgCacheOperation action= {.opId = CTG_OP_UPDATE_VGROUP, .syncReq = syncReq}; + SCtgCacheOperation action= {.opId = CTG_OP_UPDATE_VGROUP, .syncOp = syncOp}; SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); @@ -679,9 +679,9 @@ _return: CTG_RET(code); } -int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncReq) { +int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncOp) { int32_t code = 0; - SCtgCacheOperation action= {.opId = CTG_OP_UPDATE_TB_META, .syncReq = syncReq}; + SCtgCacheOperation action= {.opId = CTG_OP_UPDATE_TB_META, .syncOp = syncOp}; SCtgUpdateTblMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTblMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTblMsg)); @@ -738,9 +738,9 @@ _return: -int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { +int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncOp) { int32_t code = 0; - SCtgCacheOperation action= {.opId = CTG_OP_UPDATE_USER, .syncReq = syncReq}; + SCtgCacheOperation action= {.opId = CTG_OP_UPDATE_USER, .syncOp = syncOp}; SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); @@ -1574,7 +1574,7 @@ void* ctgUpdateThreadFunc(void* param) { gCtgMgmt.queue.seqDone = operation->seqId; - if (operation->syncReq) { + if (operation->syncOp) { tsem_post(&gCtgMgmt.queue.rspSem); } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 81d206a0f3..5b720006c6 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -1380,7 +1380,7 @@ TEST(tableMeta, updateStbMeta) { STableMetaRsp rsp = {0}; ctgTestBuildSTableMetaRsp(&rsp); - code = catalogUpdateSTableMeta(pCtg, &rsp); + code = catalogUpdateTableMeta(pCtg, &rsp); ASSERT_EQ(code, 0); taosMemoryFreeClear(rsp.pSchemas); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 810c0153b1..e77f2b0ca4 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -273,7 +273,7 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) { return TSDB_CODE_SUCCESS; } -int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STableMeta **pMeta) { +int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isStb, STableMeta **pMeta) { int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; @@ -283,14 +283,14 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STabl return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pTableMeta->vgId = isSuperTable ? 0 : msg->vgId; - pTableMeta->tableType = isSuperTable ? TSDB_SUPER_TABLE : msg->tableType; - pTableMeta->uid = isSuperTable ? msg->suid : msg->tuid; + pTableMeta->vgId = isStb ? 0 : msg->vgId; + pTableMeta->tableType = isStb ? TSDB_SUPER_TABLE : msg->tableType; + pTableMeta->uid = isStb ? msg->suid : msg->tuid; pTableMeta->suid = msg->suid; pTableMeta->sversion = msg->sversion; pTableMeta->tversion = msg->tversion; - if (isSuperTable) { + if (isStb) { qDebug("stable %s meta returned, suid:%" PRIx64, msg->stbName, pTableMeta->suid); } diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index c15649106e..39ba52145c 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -162,6 +162,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch tDecoderClear(&coder); SCH_ERR_JRET(code); SCH_ERR_JRET(rsp.code); + + pJob->queryRes = rsp.pMeta; } SCH_ERR_JRET(rspCode); From f566146cc9c19664f3c3c9ae664beacae5ab65c4 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 1 Jun 2022 12:50:25 +0000 Subject: [PATCH 20/47] fix(query): fix concat_ws separator param can only be constant string TD-16070 --- source/libs/function/src/builtins.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index add94cb83c..8aa3d8fc37 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -817,11 +817,20 @@ static int32_t translateConcatImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t int32_t resultBytes = 0; int32_t sepBytes = 0; + //concat_ws separator should be constant string + if (hasSep) { + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + if (nodeType(pPara) != QUERY_NODE_VALUE) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "The first parameter of CONCAT_WS function can only be constant string"); + } + } + /* For concat/concat_ws function, if params have NCHAR type, promote the final result to NCHAR */ for (int32_t i = 0; i < numOfParams; ++i) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, i); uint8_t paraType = ((SExprNode*)pPara)->resType.type; - if (!IS_VAR_DATA_TYPE(paraType)) { + if (!IS_VAR_DATA_TYPE(paraType) && TSDB_DATA_TYPE_NULL != paraType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } if (TSDB_DATA_TYPE_NCHAR == paraType) { From 61d20798706c93467ca82334ff73abf8c589b909 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 1 Jun 2022 13:09:58 +0000 Subject: [PATCH 21/47] fix(query): concat/concat_ws function null constant handling TD-16071 --- source/libs/scalar/src/sclfunc.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 6ee5f038d6..5478013e7b 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -444,7 +444,8 @@ int32_t concatFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu for (int32_t k = 0; k < numOfRows; ++k) { bool hasNull = false; for (int32_t i = 0; i < inputNum; ++i) { - if (colDataIsNull_s(pInputData[i], k)) { + if (colDataIsNull_s(pInputData[i], k) || + GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_NULL) { colDataAppendNULL(pOutputData, k); hasNull = true; break; @@ -520,7 +521,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p char *output = outputBuf; for (int32_t k = 0; k < numOfRows; ++k) { - if (colDataIsNull_s(pInputData[0], k)) { + if (colDataIsNull_s(pInputData[0], k) || + GET_PARAM_TYPE(&pInput[0]) == TSDB_DATA_TYPE_NULL) { colDataAppendNULL(pOutputData, k); continue; } @@ -528,7 +530,8 @@ int32_t concatWsFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p int16_t dataLen = 0; bool hasNull = false; for (int32_t i = 1; i < inputNum; ++i) { - if (colDataIsNull_s(pInputData[i], k)) { + if (colDataIsNull_s(pInputData[i], k) || + GET_PARAM_TYPE(&pInput[i]) == TSDB_DATA_TYPE_NULL) { hasNull = true; break; } From 1907c0e66c7ba9f4b1e2a7a966b0489b8d76a091 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 21:16:22 +0800 Subject: [PATCH 22/47] enh: add creating and dropping states to mnode --- include/util/taoserror.h | 1 + source/common/src/systable.c | 1 + source/dnode/mnode/impl/src/mndMnode.c | 41 +++-- source/dnode/mnode/impl/src/mndTrans.c | 56 +++---- source/dnode/mnode/sdb/inc/sdb.h | 1 + source/dnode/mnode/sdb/src/sdbHash.c | 28 ++++ source/libs/transport/src/transCli.c | 2 +- source/util/src/terror.c | 8 +- tests/script/tsim/mnode/basic3.sim | 14 +- tests/script/tsim/mnode/basic4.sim | 208 ++++++++++++++++++++++++ tests/script/tsim/stable/column_add.sim | 3 +- tests/script/tsim/stable/tag_add.sim | 4 +- tests/script/tsim/stable/tag_drop.sim | 10 +- 13 files changed, 313 insertions(+), 64 deletions(-) create mode 100644 tests/script/tsim/mnode/basic4.sim diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c3d2788897..4550bccbed 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -85,6 +85,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RPC_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x0102) #define TSDB_CODE_RPC_FQDN_ERROR TAOS_DEF_ERROR_CODE(0, 0x0103) #define TSDB_CODE_RPC_PORT_EADDRINUSE TAOS_DEF_ERROR_CODE(0, 0x0104) +#define TSDB_CODE_RPC_INDIRECT_NETWORK_UNAVAIL TAOS_DEF_ERROR_CODE(0, 0x0105) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 8207ffb22f..b163ac32bf 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -36,6 +36,7 @@ static const SSysDbTableSchema mnodesSchema[] = { {.name = "id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, {.name = "endpoint", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "role", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP}, }; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 4578d81efb..7c94a33ffe 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -392,11 +392,6 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { mDebug("mnode:%d, start to create", createReq.dnodeId); - if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) { - terrno = TSDB_CODE_MND_TOO_MANY_MNODES; - goto _OVER; - } - pObj = mndAcquireMnode(pMnode, createReq.dnodeId); if (pObj != NULL) { terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST; @@ -405,12 +400,22 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { goto _OVER; } + if (sdbGetSize(pMnode->pSdb, SDB_MNODE) >= 3) { + terrno = TSDB_CODE_MND_TOO_MANY_MNODES; + goto _OVER; + } + pDnode = mndAcquireDnode(pMnode, createReq.dnodeId); if (pDnode == NULL) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; goto _OVER; } + if (!mndIsDnodeOnline(pMnode, pDnode, taosGetTimestampMs())) { + terrno = TSDB_CODE_NODE_OFFLINE; + goto _OVER; + } + pUser = mndAcquireUser(pMnode, pReq->conn.user); if (pUser == NULL) { terrno = TSDB_CODE_MND_NO_USER_FROM_CONN; @@ -632,11 +637,12 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t numOfRows = 0; int32_t cols = 0; SMnodeObj *pObj = NULL; + ESdbStatus objStatus; char *pWrite; int64_t curMs = taosGetTimestampMs(); while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj); + pShow->pIter = sdbFetchAll(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj, &objStatus); if (pShow->pIter == NULL) break; cols = 0; @@ -649,23 +655,26 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); - bool online = mndIsDnodeOnline(pMnode, pObj->pDnode, curMs); - const char *roles = NULL; + const char *roles = "OFFLINE"; if (pObj->id == pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_LEADER); - } else { - if (!online) { - roles = "OFFLINE"; - } else { - roles = syncStr(pObj->state); - } } - char *b2 = taosMemoryCalloc(1, 12 + VARSTR_HEADER_SIZE); + if (pObj->pDnode && mndIsDnodeOnline(pMnode, pObj->pDnode, curMs)) { + roles = syncStr(pObj->state); + } + char b2[12 + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(b2, roles, pShow->pMeta->pSchemas[cols].bytes); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)b2, false); + const char *status = "READY"; + if (objStatus == SDB_STATUS_CREATING) status = "CREATING"; + if (objStatus == SDB_STATUS_DROPPING) status = "DROPPING"; + char b3[9 + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(b3, status, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)b3, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pObj->createdTime, false); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index e191bb9b2a..660862b8d8 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -344,7 +344,7 @@ static SSdbRow *mndTransActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &dataLen, _OVER) action.pRaw = taosMemoryMalloc(dataLen); if (action.pRaw == NULL) goto _OVER; - mTrace("raw:%p, is created", pData); + mTrace("raw:%p, is created", action.pRaw); SDB_GET_BINARY(pRaw, dataPos, (void *)action.pRaw, dataLen, _OVER); if (taosArrayPush(pTrans->commitActions, &action) == NULL) goto _OVER; action.pRaw = NULL; @@ -619,9 +619,7 @@ void mndTransSetCb(STrans *pTrans, ETrnFunc startFunc, ETrnFunc stopFunc, void * pTrans->paramLen = paramLen; } -void mndTransSetDbName(STrans *pTrans, const char *dbname) { - memcpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN); -} +void mndTransSetDbName(STrans *pTrans, const char *dbname) { memcpy(pTrans->dbname, dbname, TSDB_DB_FNAME_LEN); } void mndTransSetSerial(STrans *pTrans) { pTrans->exec = TRN_EXEC_SERIAL; } @@ -753,22 +751,30 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { sendRsp = true; } } else { - if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 6) { + if (pTrans->stage == TRN_STAGE_REDO_ACTION && pTrans->failedTimes > 3) { if (code == 0) code = TSDB_CODE_MND_TRANS_UNKNOW_ERROR; sendRsp = true; } } if (sendRsp && pTrans->rpcInfo.handle != NULL) { - void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); - if (rpcCont != NULL) { - memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); - } - taosMemoryFree(pTrans->rpcRsp); - mDebug("trans:%d, send rsp, code:0x%x stage:%s app:%p", pTrans->id, code, mndTransStr(pTrans->stage), pTrans->rpcInfo.ahandle); - SRpcMsg rspMsg = {.code = code, .pCont = rpcCont, .contLen = pTrans->rpcRspLen, .info = pTrans->rpcInfo}; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + code = TSDB_CODE_RPC_INDIRECT_NETWORK_UNAVAIL; + } + SRpcMsg rspMsg = {.code = code, .info = pTrans->rpcInfo}; + + if (pTrans->rpcRspLen != 0) { + void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); + if (rpcCont != NULL) { + memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); + rspMsg.pCont = rpcCont; + rspMsg.contLen = pTrans->rpcRspLen; + } + taosMemoryFree(pTrans->rpcRsp); + } + tmsgSendRsp(&rspMsg); pTrans->rpcInfo.handle = NULL; pTrans->rpcRsp = NULL; @@ -1025,18 +1031,23 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) } if (code == 0) { + pTrans->code = 0; pTrans->redoActionPos++; mDebug("trans:%d, %s:%d is executed and need sync to other mnodes", pTrans->id, mndTransStr(pAction->stage), pAction->id); code = mndTransSync(pMnode, pTrans); if (code != 0) { - mError("trans:%d, failed to sync redoActionPos since %s", pTrans->id, terrstr()); + pTrans->code = terrno; + mError("trans:%d, %s:%d is executed and failed to sync to other mnodes since %s", pTrans->id, + mndTransStr(pAction->stage), pAction->id, terrstr()); break; } } else if (code == TSDB_CODE_ACTION_IN_PROGRESS) { mDebug("trans:%d, %s:%d is in progress and wait it finish", pTrans->id, mndTransStr(pAction->stage), pAction->id); break; } else { + terrno = code; + pTrans->code = code; mError("trans:%d, %s:%d failed to execute since %s", pTrans->id, mndTransStr(pAction->stage), pAction->id, terrstr()); break; @@ -1239,19 +1250,8 @@ int32_t mndKillTrans(SMnode *pMnode, STrans *pTrans) { return -1; } - int32_t size = taosArrayGetSize(pArray); - - for (int32_t i = 0; i < size; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { STransAction *pAction = taosArrayGet(pArray, i); - if (pAction == NULL) continue; - - if (pAction->msgReceived == 0) { - mInfo("trans:%d, %s:%d set processed for kill msg received", pTrans->id, mndTransStr(pAction->stage), i); - pAction->msgSent = 1; - pAction->msgReceived = 1; - pAction->errCode = 0; - } - if (pAction->errCode != 0) { mInfo("trans:%d, %s:%d set processed for kill msg received, errCode from %s to success", pTrans->id, mndTransStr(pAction->stage), i, tstrerror(pAction->errCode)); @@ -1290,9 +1290,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { pTrans = mndAcquireTrans(pMnode, killReq.transId); if (pTrans == NULL) { - terrno = TSDB_CODE_MND_TRANS_NOT_EXIST; - mError("trans:%d, failed to kill since %s", killReq.transId, terrstr()); - return -1; + goto _OVER; } code = mndKillTrans(pMnode, pTrans); @@ -1300,9 +1298,9 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { _OVER: if (code != 0) { mError("trans:%d, failed to kill since %s", killReq.transId, terrstr()); - return -1; } + mndReleaseUser(pMnode, pUser); mndReleaseTrans(pMnode, pTrans); return code; } diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 1fd0260d0d..4a00befa1e 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -301,6 +301,7 @@ void sdbRelease(SSdb *pSdb, void *pObj); * @return void* The next iterator of the table. */ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj); +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) ; /** * @brief Cancel a traversal diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index abf35b71a9..162da2bd0a 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -368,6 +368,34 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) { return ppRow; } +void *sdbFetchAll(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj, ESdbStatus *status) { + *ppObj = NULL; + + SHashObj *hash = sdbGetHash(pSdb, type); + if (hash == NULL) return NULL; + + TdThreadRwlock *pLock = &pSdb->locks[type]; + taosThreadRwlockRdlock(pLock); + + SSdbRow **ppRow = taosHashIterate(hash, pIter); + while (ppRow != NULL) { + SSdbRow *pRow = *ppRow; + if (pRow == NULL) { + ppRow = taosHashIterate(hash, ppRow); + continue; + } + + atomic_add_fetch_32(&pRow->refCount, 1); + sdbPrintOper(pSdb, pRow, "fetch"); + *ppObj = pRow->pObj; + *status = pRow->status; + break; + } + taosThreadRwlockUnlock(pLock); + + return ppRow; +} + void sdbCancelFetch(SSdb *pSdb, void *pIter) { if (pIter == NULL) return; SSdbRow *pRow = *(SSdbRow **)pIter; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a8e79266ac..32d8251b1c 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -954,7 +954,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL */ tmsg_t msgType = pCtx->msgType; - if ((pTransInst->retry != NULL && (pTransInst->retry(pResp->code))) || + if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { pMsg->sent = 0; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index b81d81c736..74fc14ecdd 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -90,6 +90,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RPC_AUTH_FAILURE, "Authentication failur TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_PORT_EADDRINUSE, "Port already in use") +TAOS_DEFINE_ERROR(TSDB_CODE_RPC_INDIRECT_NETWORK_UNAVAIL, "Unable to establish connection") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") @@ -244,7 +245,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_SINGLE_STB_MODE_DB, "Database is single st // mnode-infoSchema TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") - // mnode-func TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_NOT_EXIST, "Func not exists") @@ -325,9 +325,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, "Invalid table ID") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, "Invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, "Invalid table schema version") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, "Table already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST, "Table not exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_NOT_EXIST, "Table not exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_ALREADY_EXIST, "Stable already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_STB_NOT_EXIST, "Stable not exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, "Invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, "Tsdb init failed") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_DISKSPACE, "No diskspace for tsdb") diff --git a/tests/script/tsim/mnode/basic3.sim b/tests/script/tsim/mnode/basic3.sim index 3c69e6ed51..edbf429075 100644 --- a/tests/script/tsim/mnode/basic3.sim +++ b/tests/script/tsim/mnode/basic3.sim @@ -3,6 +3,10 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/deploy.sh -n dnode2 -i 2 system sh/deploy.sh -n dnode3 -i 3 system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode1 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode2 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode3 -c transPullupInterval -v 1 +system sh/cfg.sh -n dnode4 -c transPullupInterval -v 1 system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode3 -s start @@ -18,7 +22,7 @@ $x = 0 step1: $x = $x + 1 sleep 1000 - if $x == 50 then + if $x == 10 then return -1 endi sql show dnodes -x step1 @@ -41,7 +45,7 @@ $x = 0 step2: $x = $x + 1 sleep 1000 - if $x == 50 then + if $x == 10 then return -1 endi sql show mnodes -x step2 @@ -72,7 +76,7 @@ $x = 0 step4: $x = $x + 1 sleep 1000 - if $x == 50 then + if $x == 10 then return -1 endi sql show mnodes -x step4 @@ -102,7 +106,7 @@ $x = 0 step5: $x = $x + 1 sleep 1000 - if $x == 50 then + if $x == 10 then return -1 endi sql show mnodes -x step5 @@ -127,7 +131,7 @@ $x = 0 step6: $x = $x + 1 sleep 1000 - if $x == 50 then + if $x == 10 then return -1 endi sql show mnodes -x step6 diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim new file mode 100644 index 0000000000..70791b009d --- /dev/null +++ b/tests/script/tsim/mnode/basic4.sim @@ -0,0 +1,208 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1: create dnodes +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 5 then + return -1 + endi +sql show dnodes -x step1 +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step2: create mnode 2 +sql create mnode on dnode 2 +sql_error create mnode on dnode 3 +return +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step2: + $x = $x + 1 + sleep 1000 + if $x == 5 then + return -1 + endi +sql show dnodes -x step2 +if $data(1)[4] != ready then + goto step2 +endi +if $data(2)[4] != ready then + goto step2 +endi + +sleep 2000 +system sh/exec.sh -n dnode3 -s stop +sql_error create mnode on dnode 3 + +print =============== step3: show mnodes + +$x = 0 +step3: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show mnodes -x step3 +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] + +if $data(1)[2] != LEADER then + goto step3 +endi +if $data(2)[2] != FOLLOWER then + goto step3 +endi +if $data(3)[2] != OFFLINE then + goto step3 +endi +if $data(1)[3] != READY then + goto step3 +endi +if $data(2)[3] != READY then + goto step3 +endi +if $data(3)[3] != CREATING then + goto step3 +endi + +print =============== step4: start dnode3 +system sh/exec.sh -n dnode3 -s start + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 10 then + return -1 + endi +sql show mnodes -x step4 +if $data(1)[2] != LEADER then + goto step4 +endi +if $data(2)[2] != FOLLOWER then + goto step4 +endi +if $data(3)[2] != FOLLOWER then + goto step4 +endi +if $data(1)[3] != READY then + goto step4 +endi +if $data(2)[3] != READY then + goto step4 +endi +if $data(3)[3] != READY then + goto step4 +endi + +print =============== step5: drop mnode 3 and stop dnode3 + +return + +print =============== step3: create user +sql create user user1 PASS 'user1' +sql show users +if $rows != 2 then + return -1 +endi + +# wait mnode2 mnode3 recv data finish +sleep 10000 + +print =============== step4: stop dnode1 +system sh/exec.sh -n dnode1 -s stop + +$x = 0 +step4: + $x = $x + 1 + sleep 1000 + if $x == 50 then + return -1 + endi +sql show mnodes -x step4 +print $data(1)[0] $data(1)[1] $data(1)[2] +print $data(2)[0] $data(2)[1] $data(2)[2] +print $data(3)[0] $data(3)[1] $data(3)[2] + +sql show users +if $rows != 2 then + return -1 +endi + +sleep 1000 +sql show dnodes +if $data(2)[4] != ready then + return -1 +endi +if $data(3)[4] != ready then + return -1 +endi + +print =============== step5: stop dnode1 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s stop + +$x = 0 +step5: + $x = $x + 1 + sleep 1000 + if $x == 50 then + return -1 + endi +sql show mnodes -x step5 +print $data(1)[0] $data(1)[1] $data(1)[2] +print $data(2)[0] $data(2)[1] $data(2)[2] +print $data(3)[0] $data(3)[1] $data(3)[2] + +if $data(2)[2] != OFFLINE then + goto step5 +endi + +sql show users +if $rows != 2 then + return -1 +endi + +print =============== step6: stop dnode1 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s stop + +$x = 0 +step6: + $x = $x + 1 + sleep 1000 + if $x == 50 then + return -1 + endi +sql show mnodes -x step6 +print $data(1)[0] $data(1)[1] $data(1)[2] +print $data(2)[0] $data(2)[1] $data(2)[2] +print $data(3)[0] $data(3)[1] $data(3)[2] + +sql show users +if $rows != 2 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop +system sh/exec.sh -n dnode2 -s stop +system sh/exec.sh -n dnode3 -s stop +system sh/exec.sh -n dnode4 -s stop \ No newline at end of file diff --git a/tests/script/tsim/stable/column_add.sim b/tests/script/tsim/stable/column_add.sim index a5d9b48508..db592e6c69 100644 --- a/tests/script/tsim/stable/column_add.sim +++ b/tests/script/tsim/stable/column_add.sim @@ -143,7 +143,7 @@ sql insert into db.ctb values(now+2s, 1, 2, 3, 4) sql select * from db.stb print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 3 then return -1 @@ -200,7 +200,6 @@ sql insert into db.ctb values(now+3s, 1, 2, 3, 4, 5) sql select * from db.stb print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 4 then diff --git a/tests/script/tsim/stable/tag_add.sim b/tests/script/tsim/stable/tag_add.sim index 01cc7bc36c..a7615df14c 100644 --- a/tests/script/tsim/stable/tag_add.sim +++ b/tests/script/tsim/stable/tag_add.sim @@ -129,7 +129,7 @@ sql select * from db.stb sql select * from db.stb print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 1 then return -1 @@ -160,7 +160,7 @@ sql insert into db.ctb2 values(now, 1, "2") sql select * from db.stb where tbname = 'ctb2'; print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 1 then return -1 diff --git a/tests/script/tsim/stable/tag_drop.sim b/tests/script/tsim/stable/tag_drop.sim index afac59daff..50907be23e 100644 --- a/tests/script/tsim/stable/tag_drop.sim +++ b/tests/script/tsim/stable/tag_drop.sim @@ -155,7 +155,7 @@ sql select * from db.stb sql select * from db.stb print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 1 then return -1 @@ -186,7 +186,7 @@ sql insert into db.ctb2 values(now, 1, "2") sql select * from db.stb where tbname = 'ctb2'; print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 1 then return -1 @@ -219,7 +219,7 @@ sql select * from db.stb where tbname = 'ctb2'; sql select * from db.stb where tbname = 'ctb2'; print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 1 then return -1 @@ -251,7 +251,7 @@ sql insert into db.ctb3 values(now, 1, "2") sql select * from db.stb where tbname = 'ctb3'; print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 1 then return -1 @@ -313,7 +313,7 @@ endi sql select * from db.stb where tbname = 'ctb3'; print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] -print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] $data[1][5] $data[1][6] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] $data[2][5] $data[2][6] if $rows != 1 then return -1 From d56c574fe47faf13f131d3313e2b8de9aae5d779 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 1 Jun 2022 21:39:04 +0800 Subject: [PATCH 23/47] enh: add creating and dropping states to mnode --- source/dnode/mnode/impl/src/mndTrans.c | 3 + tests/script/jenkins/basic.txt | 2 + tests/script/tsim/mnode/basic4.sim | 118 +++++++++++-------------- 3 files changed, 57 insertions(+), 66 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 660862b8d8..bbee59090d 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1006,6 +1006,9 @@ static int32_t mndTransExecuteRedoActionsSerial(SMnode *pMnode, STrans *pTrans) if (pAction->msgReceived) { if (pAction->errCode != 0 && pAction->errCode != pAction->acceptableCode) { code = pAction->errCode; + pAction->msgSent = 0; + pAction->msgReceived = 0; + mDebug("trans:%d, %s:%d execute status is reset", pTrans->id, mndTransStr(pAction->stage), action); } } else { code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 12b678eeae..e86f07620e 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -57,6 +57,8 @@ # ---- mnode ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim +./test.sh -f tsim/mnode/basic3.sim +./test.sh -f tsim/mnode/basic4.sim # ---- show ./test.sh -f tsim/show/basic.sim diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index 70791b009d..2a4a9d3626 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -28,7 +28,7 @@ endi print =============== step2: create mnode 2 sql create mnode on dnode 2 sql_error create mnode on dnode 3 -return + system sh/exec.sh -n dnode3 -s start $x = 0 @@ -46,7 +46,6 @@ if $data(2)[4] != ready then goto step2 endi -sleep 2000 system sh/exec.sh -n dnode3 -s stop sql_error create mnode on dnode 3 @@ -94,6 +93,10 @@ step4: return -1 endi sql show mnodes -x step4 +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] + if $data(1)[2] != LEADER then goto step4 endi @@ -114,92 +117,75 @@ if $data(3)[3] != READY then endi print =============== step5: drop mnode 3 and stop dnode3 - -return - -print =============== step3: create user -sql create user user1 PASS 'user1' -sql show users -if $rows != 2 then - return -1 -endi - -# wait mnode2 mnode3 recv data finish -sleep 10000 - -print =============== step4: stop dnode1 -system sh/exec.sh -n dnode1 -s stop - -$x = 0 -step4: - $x = $x + 1 - sleep 1000 - if $x == 50 then - return -1 - endi -sql show mnodes -x step4 -print $data(1)[0] $data(1)[1] $data(1)[2] -print $data(2)[0] $data(2)[1] $data(2)[2] -print $data(3)[0] $data(3)[1] $data(3)[2] - -sql show users -if $rows != 2 then - return -1 -endi - -sleep 1000 -sql show dnodes -if $data(2)[4] != ready then - return -1 -endi -if $data(3)[4] != ready then - return -1 -endi - -print =============== step5: stop dnode1 -system sh/exec.sh -n dnode1 -s start -system sh/exec.sh -n dnode2 -s stop +system sh/exec.sh -n dnode3 -s stop +sql_error drop mnode on dnode 3 $x = 0 step5: $x = $x + 1 sleep 1000 - if $x == 50 then + if $x == 10 then return -1 endi sql show mnodes -x step5 -print $data(1)[0] $data(1)[1] $data(1)[2] -print $data(2)[0] $data(2)[1] $data(2)[2] -print $data(3)[0] $data(3)[1] $data(3)[2] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] -if $data(2)[2] != OFFLINE then +if $data(1)[2] != LEADER then + goto step5 +endi +if $data(2)[2] != FOLLOWER then + goto step5 +endi +if $data(3)[2] != OFFLINE then + goto step5 +endi +if $data(1)[3] != READY then + goto step5 +endi +if $data(2)[3] != READY then + goto step5 +endi +if $data(3)[3] != DROPPING then goto step5 endi -sql show users -if $rows != 2 then - return -1 -endi - -print =============== step6: stop dnode1 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s stop +print =============== step6: start dnode3 +system sh/exec.sh -n dnode3 -s start $x = 0 step6: $x = $x + 1 sleep 1000 - if $x == 50 then + if $x == 10 then return -1 endi sql show mnodes -x step6 -print $data(1)[0] $data(1)[1] $data(1)[2] -print $data(2)[0] $data(2)[1] $data(2)[2] -print $data(3)[0] $data(3)[1] $data(3)[2] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +print $data[2][0] $data[2][1] $data[2][2] $data[2][3] $data[2][4] -sql show users if $rows != 2 then - return -1 + goto step6 +endi +if $data(1)[2] != LEADER then + goto step6 +endi +if $data(2)[2] != FOLLOWER then + goto step6 +endi +if $data(3)[2] != null then + goto step6 +endi +if $data(1)[3] != READY then + goto step6 +endi +if $data(2)[3] != READY then + goto step6 +endi +if $data(3)[3] != null then + goto step6 endi system sh/exec.sh -n dnode1 -s stop From 98b40a59a07aaf22a60a5a383f5407d7dd996646 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 1 Jun 2022 13:52:01 +0000 Subject: [PATCH 24/47] feat(query): add log function support natural logrithm TD-15182 --- source/libs/function/src/builtins.c | 24 +++++++++++++++++++++++- source/libs/scalar/src/sclfunc.c | 12 ++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 8aa3d8fc37..d5345877eb 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -103,6 +103,28 @@ static int32_t translateInOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t le return TSDB_CODE_SUCCESS; } +static int32_t translateLogarithm(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (1 != numOfParams && 2 != numOfParams) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_NUMERIC_TYPE(para1Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + if (2 == numOfParams) { + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_NUMERIC_TYPE(para2Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateCount(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -1311,7 +1333,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "log", .type = FUNCTION_TYPE_LOG, .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateIn2NumOutDou, + .translateFunc = translateLogarithm, .getEnvFunc = NULL, .initFunc = NULL, .sprocessFunc = logFunction, diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 5478013e7b..466c477d73 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -15,7 +15,11 @@ typedef void (*_trim_fn)(char *, char*, int32_t, int32_t); typedef int16_t (*_len_fn)(char *, int32_t); /** Math functions **/ -static double tlog(double v, double base) { +static double tlog(double v) { + return log(v); +} + +static double tlog2(double v, double base) { double a = log(v); double b = log(base); if (isnan(a) || isinf(a)) { @@ -1368,7 +1372,11 @@ int32_t powFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu } int32_t logFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - return doScalarFunctionUnique2(pInput, inputNum, pOutput, tlog); + if (inputNum == 1) { + return doScalarFunctionUnique(pInput, inputNum, pOutput, tlog); + } else { + return doScalarFunctionUnique2(pInput, inputNum, pOutput, tlog2); + } } int32_t sqrtFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { From f3b680f6307b9252b8276aac9c3661d1f6734ac4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 1 Jun 2022 22:10:53 +0800 Subject: [PATCH 25/47] refactor(tmq): push mode --- examples/c/tmq.c | 4 +- include/libs/stream/tstream.h | 31 +++++++++++++ source/dnode/vnode/src/inc/tq.h | 15 ++---- source/dnode/vnode/src/tq/tq.c | 60 ++++++++++++++---------- source/dnode/vnode/src/tq/tqPush.c | 73 ++++++++++++++++++++++++++---- 5 files changed, 136 insertions(+), 47 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index a8584bae82..40d72d3af1 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -176,8 +176,8 @@ tmq_t* build_consumer() { tmq_list_t* build_topic_list() { tmq_list_t* topic_list = tmq_list_new(); - /*tmq_list_append(topic_list, "topic_ctb_column");*/ - tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic"); + tmq_list_append(topic_list, "topic_ctb_column"); + /*tmq_list_append(topic_list, "tmq_test_db_multi_insert_topic");*/ return topic_list; } diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 0525cbf367..f7ad7b4ed8 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -80,6 +80,37 @@ typedef struct { int8_t type; } SStreamCheckpoint; +typedef struct { + STaosQueue* queue; + STaosQall* qall; + void* qItem; + int8_t failed; +} SStreamQ; + +static FORCE_INLINE void* streamQCurItem(SStreamQ* queue) { + // + return queue->qItem; +} + +static FORCE_INLINE void* streamQNextItem(SStreamQ* queue) { + int8_t failed = atomic_load_8(&queue->failed); + if (failed) { + ASSERT(queue->qItem != NULL); + return streamQCurItem(queue); + } else { + taosGetQitem(queue->qall, &queue->qItem); + if (queue->qItem == NULL) { + taosReadAllQitems(queue->queue, queue->qall); + taosGetQitem(queue->qall, &queue->qItem); + } + return streamQCurItem(queue); + } +} + +static FORCE_INLINE void streamQSetFail(SStreamQ* queue) { atomic_store_8(&queue->failed, 1); } + +static FORCE_INLINE void streamQSetSuccess(SStreamQ* queue) { atomic_store_8(&queue->failed, 0); } + static FORCE_INLINE SStreamDataSubmit* streamDataSubmitNew(SSubmitReq* pReq) { SStreamDataSubmit* pDataSubmit = (SStreamDataSubmit*)taosAllocateQitem(sizeof(SStreamDataSubmit), DEF_QITEM); if (pDataSubmit == NULL) return NULL; diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 0715ee8066..7cd82b0ac3 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -65,12 +65,6 @@ struct STqReadHandle { // tqPush -typedef struct { - STaosQueue* queue; - STaosQall* qall; - void* qItem; -} STqInputQ; - typedef struct { // msg info int64_t consumerId; @@ -84,10 +78,10 @@ typedef struct { tmr_h timerId; int8_t tmrStopped; // exec - int8_t inputStatus; - int8_t execStatus; - STqInputQ inputQ; - SRWLatch lock; + int8_t inputStatus; + int8_t execStatus; + SStreamQ inputQ; + SRWLatch lock; } STqPushHandle; // tqExec @@ -155,6 +149,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead* // tqExec int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId); +int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp); // tqMeta int32_t tqMetaOpen(STQ* pTq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 67651c2f78..e79de255f3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -81,12 +81,41 @@ void tqClose(STQ* pTq) { // TODO } +int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) { + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = pReq->consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqDataBlkRsp(&abuf, pRsp); + + SRpcMsg resp = { + .info = pMsg->info, + .pCont = buf, + .contLen = tlen, + .code = 0, + }; + tmsgSendRsp(&resp); + + tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, pRsp->reqOffset, pRsp->rspOffset); + + return 0; +} + int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; int64_t timeout = pReq->timeout; int32_t reqEpoch = pReq->epoch; int64_t fetchOffset; + int32_t code = 0; // get offset to fetch message if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { @@ -155,7 +184,9 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; - tqDataExec(pTq, &pHandle->execHandle, pCont, &rsp, workerId); + if (tqDataExec(pTq, &pHandle->execHandle, pCont, &rsp, workerId) < 0) { + /*ASSERT(0);*/ + } } else { // TODO ASSERT(0); @@ -180,31 +211,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { rsp.rspOffset = fetchOffset; - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, &rsp); - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - pMsg->code = -1; - return -1; + if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) { + code = -1; } - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqDataBlkRsp(&abuf, &rsp); - - SRpcMsg resp = { - .info = pMsg->info, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - tmsgSendRsp(&resp); - - tqDebug("vg %d offset %ld from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), fetchOffset, consumerId, pReq->epoch, rsp.blockNum, rsp.reqOffset, rsp.rspOffset); - // TODO wrap in destroy func taosArrayDestroy(rsp.blockData); taosArrayDestroy(rsp.blockDataLen); @@ -217,7 +227,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree); } - return 0; + return code; } int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) { diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 2d9207a0de..26e9dfe2e2 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -21,21 +21,74 @@ void tqTmrRspFunc(void* param, void* tmrId) { } int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { + SMqDataBlkRsp rsp = {0}; // 1. guard and set status executing - // 2. check processedVer - // 2.1. if not missed, get msg from queue - // 2.2. if missed, scan wal - // - // 3. exec, after each success, update processed ver - // first run - // set exec status closing - // second run - // set exec status idle - // + int8_t execStatus = + atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); + if (execStatus == TASK_STATUS__IDLE) { + SStreamDataSubmit* pSubmit = NULL; + // 2. check processedVer + // 2.1. if not missed, get msg from queue + // 2.2. if missed, scan wal + pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + while (pHandle->pushHandle.processedVer <= pSubmit->ver) { + // read from wal + } + while (pHandle->pushHandle.processedVer > pSubmit->ver + 1) { + streamQSetSuccess(&pHandle->pushHandle.inputQ); + streamDataSubmitRefDec(pSubmit); + pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + if (pSubmit == NULL) break; + } + // 3. exec, after each success, update processed ver + // first run + while (pSubmit != NULL) { + ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); + if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, &rsp, 0) < 0) { + /*ASSERT(0);*/ + } + // update processed + atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); + streamQSetSuccess(&pHandle->pushHandle.inputQ); + streamDataSubmitRefDec(pSubmit); + if (rsp.blockNum > 0) { + goto SEND_RSP; + } else { + pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + } + } + // set exec status closing + atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__CLOSING); + // second run + while (pSubmit != NULL) { + ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); + if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, &rsp, 0) < 0) { + /*ASSERT(0);*/ + } + // update processed + atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); + streamQSetSuccess(&pHandle->pushHandle.inputQ); + streamDataSubmitRefDec(pSubmit); + if (rsp.blockNum > 0) { + goto SEND_RSP; + } else { + pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + } + } + // set exec status idle + atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); + } +SEND_RSP: // 4. if get result // 4.1 set exec input status blocked and exec status idle + atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); // 4.2 rpc send + rsp.rspOffset = pHandle->pushHandle.processedVer; + /*if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {*/ + /*return -1;*/ + /*}*/ // 4.3 clear rpc info + memset(&pHandle->pushHandle.rpcInfo, 0, sizeof(SRpcHandleInfo)); return 0; } From 2bb92cdf5e9ce1afb0db6ebe5664a24d30ae96d7 Mon Sep 17 00:00:00 2001 From: Linhe Huo Date: Wed, 1 Jun 2022 22:32:01 +0800 Subject: [PATCH 26/47] docs: update Grafana plugin docs (#13395) --- docs-cn/20-third-party/01-grafana.mdx | 36 ++++++++++----------------- docs-en/20-third-party/01-grafana.mdx | 34 +++++++++++-------------- 2 files changed, 27 insertions(+), 43 deletions(-) diff --git a/docs-cn/20-third-party/01-grafana.mdx b/docs-cn/20-third-party/01-grafana.mdx index 328bd6bb45..40b5c0ff4f 100644 --- a/docs-cn/20-third-party/01-grafana.mdx +++ b/docs-cn/20-third-party/01-grafana.mdx @@ -18,21 +18,22 @@ TDengine 能够与开源数据可视化系统 [Grafana](https://www.grafana.com/ ## 配置 Grafana -TDengine 的 Grafana 插件托管在 GitHub，可从 下载，当前最新版本为 3.1.4。 - -推荐使用 [`grafana-cli` 命令行工具](https://grafana.com/docs/grafana/latest/administration/cli/) 进行插件安装。 +使用 [`grafana-cli` 命令行工具](https://grafana.com/docs/grafana/latest/administration/cli/) 进行插件[安装](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation)。 ```bash -sudo -u grafana grafana-cli \ - --pluginUrl https://github.com/taosdata/grafanaplugin/releases/download/v3.1.7/tdengine-datasource-3.1.7.zip \ - plugins install tdengine-datasource +grafana-cli plugins install tdengine-datasource +# with sudo +sudo -u grafana grafana-cli plugins install tdengine-datasource ``` -或者下载到本地并解压到 Grafana 插件目录。 +或者从 [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) 或 [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) 下载 .zip 文件到本地并解压到 Grafana 插件目录。命令行下载示例如下： ```bash -GF_VERSION=3.1.7 +GF_VERSION=3.2.2 +# from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip +# from Grafana +wget -O tdengine-datasource-$GF_VERSION.zip https://grafana.com/api/plugins/tdengine-datasource/versions/$GF_VERSION/download ``` 以 CentOS 7.2 操作系统为例，将插件包解压到 /var/lib/grafana/plugins 目录下，重新启动 grafana 即可。 @@ -41,28 +42,17 @@ wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/td sudo unzip tdengine-datasource-$GF_VERSION.zip -d /var/lib/grafana/plugins/ ``` -:::note -3.1.6 和更早版本未签名，会在 Grafana 7.3+ / 8.x 版本签名检查时失败导致无法加载插件，需要在 grafana.ini 文件中修改配置如下： - -```ini -[plugins] -allow_loading_unsigned_plugins = tdengine-datasource -``` - -::: - -在 Docker 环境下，可以使用如下的环境变量设置自动安装并设置 TDengine 插件： +如果 Grafana 在 Docker 环境下运行，可以使用如下的环境变量设置自动安装 TDengine 数据源插件： ```bash -GF_INSTALL_PLUGINS=https://github.com/taosdata/grafanaplugin/releases/download/v3.1.4/tdengine-datasource-3.1.4.zip;tdengine-datasource -GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=tdengine-datasource +GF_INSTALL_PLUGINS=tdengine-datasource ``` ## 使用 Grafana ### 配置数据源 -用户可以直接通过 http://localhost:3000 的网址，登录 Grafana 服务器（用户名/密码：admin/admin），通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示： +用户可以直接通过 的网址，登录 Grafana 服务器（用户名/密码：admin/admin），通过左侧 `Configuration -> Data Sources` 可以添加数据源，如下图所示： ![TDengine Database Grafana plugin add data source](./add_datasource1.webp) @@ -74,7 +64,7 @@ GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=tdengine-datasource ![TDengine Database Grafana plugin add data source](./add_datasource3.webp) -- Host： TDengine 集群中提供 REST 服务 （在 2.4 之前由 taosd 提供， 从 2.4 开始由 taosAdapter 提供）的组件所在服务器的 IP 地址与 TDengine REST 服务的端口号(6041)，默认 http://localhost:6041。 +- Host： TDengine 集群中提供 REST 服务 （在 2.4 之前由 taosd 提供， 从 2.4 开始由 taosAdapter 提供）的组件所在服务器的 IP 地址与 TDengine REST 服务的端口号(6041)，默认 。 - User：TDengine 用户名。 - Password：TDengine 用户密码。 diff --git a/docs-en/20-third-party/01-grafana.mdx b/docs-en/20-third-party/01-grafana.mdx index b3cab62710..1a84e02c66 100644 --- a/docs-en/20-third-party/01-grafana.mdx +++ b/docs-en/20-third-party/01-grafana.mdx @@ -9,7 +9,8 @@ You can learn more about using the TDengine plugin on [GitHub](https://github.co ## Prerequisites -In order for Grafana to add the TDengine data source successfully, the following preparation is required: +In order for Grafana to add the TDengine data source successfully, the following preparations are required: + 1. The TDengine cluster is deployed and functioning properly 2. taosAdapter is installed and running properly. Please refer to the taosAdapter manual for details. @@ -19,41 +20,34 @@ TDengine currently supports Grafana versions 7.0 and above. Users can go to the ## Configuring Grafana -You can download The Grafana plugin for TDengine from . The current latest version is 3.1.4. - -Recommend using the [``grafana-cli`` command-line tool](https://grafana.com/docs/grafana/latest/administration/cli/) for plugin installation. +Follow the installation steps in [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) with the [``grafana-cli`` command-line tool](https://grafana.com/docs/grafana/latest/administration/cli/) for plugin installation. ```bash -sudo -u grafana grafana-cli \ - --pluginUrl https://github.com/taosdata/grafanaplugin/releases/download/v3.1.4/tdengine-datasource-3.1.4.zip \ - plugins install tdengine-datasource +grafana-cli plugins install tdengine-datasource +# with sudo +sudo -u grafana grafana-cli plugins install tdengine-datasource ``` -Or download it locally and extract it to the Grafana plugin directory. +Alternatively, you can manually download the .zip file from [GitHub](https://github.com/taosdata/grafanaplugin/releases/tag/latest) or [Grafana](https://grafana.com/grafana/plugins/tdengine-datasource/?tab=installation) and unpack it into your grafana plugins directory. ```bash -GF_VERSION=3.1.4 +GF_VERSION=3.2.2 +# from GitHub wget https://github.com/taosdata/grafanaplugin/releases/download/v$GF_VERSION/tdengine-datasource-$GF_VERSION.zip +# from Grafana +wget -O tdengine-datasource-$GF_VERSION.zip https://grafana.com/api/plugins/tdengine-datasource/versions/$GF_VERSION/download ``` -In CentOS 7.2 for example, extract the plugin package to /var/lib/grafana/plugins directory, and restart grafana. +Take CentOS 7.2 for example, extract the plugin package to /var/lib/grafana/plugins directory, and restart grafana. ```bash sudo unzip tdengine-datasource-$GF_VERSION.zip -d /var/lib/grafana/plugins/ ``` -Grafana versions 7.3+ / 8.x do signature checks on plugins, so you also need to add the following line to the grafana.ini file to use the plugin correctly. - -```ini -[plugins] -allow_loading_unsigned_plugins = tdengine-datasource -``` - -The TDengine plugin can be automatically installed and set up using the following environment variable settings in a Docker environment. +If Grafana is running in a Docker environment, the TDengine plugin can be automatically installed and set up using the following environment variable settings: ```bash -GF_INSTALL_PLUGINS=https://github.com/taosdata/grafanaplugin/releases/download/v3.1.4/tdengine-datasource-3.1.4.zip;tdengine- datasource -GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS=tdengine-datasource +GF_INSTALL_PLUGINS=tdengine-datasource ``` ## Using Grafana From 58a0691abe7162307d0dd08026003005f190f10f Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Wed, 1 Jun 2022 23:08:51 +0800 Subject: [PATCH 27/47] test:add test cases for tmq --- tests/system-test/7-tmq/db.py | 462 ++++++++++++++++++++++++++++++++++ 1 file changed, 462 insertions(+) create mode 100644 tests/system-test/7-tmq/db.py diff --git a/tests/system-test/7-tmq/db.py b/tests/system-test/7-tmq/db.py new file mode 100644 index 0000000000..0115686798 --- /dev/null +++ b/tests/system-test/7-tmq/db.py @@ -0,0 +1,462 @@ + +import taos +import sys +import time +import socket +import os +import threading +from enum import Enum + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * + +class actionType(Enum): + CREATE_DATABASE = 0 + CREATE_STABLE = 1 + CREATE_CTABLE = 2 + INSERT_DATA = 3 + +class TDTestCase: + hostname = socket.gethostname() + #rpcDebugFlagVal = '143' + #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #clientCfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''} + #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal + #print ("===================: ", updatecfgDict) + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def newcur(self,cfg,host,port): + user = "root" + password = "taosdata" + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("drop database if exists %s "%(cdbName)) + tdSql.query("create database %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + + def initConsumeContentTable(self,id=0,cdbName='cdb'): + tdSql.query("drop table if exists %s.content_%d "%(cdbName, id)) + tdSql.query("create table %s.content_%d (ts timestamp, contentOfRow binary(1024))"%cdbName, id) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,buildPath,cfgPath,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + shellCmd = 'nohup ' + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + shellCmd += buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctables(self,tsql, dbName,stbName,ctbPrefix,ctbNum): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + sql += " %s_%d using %s tags(%d)"%(ctbPrefix,i,stbName,i+1) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + ctbDict = {} + for i in range(ctbNum): + ctbDict[i] = 0 + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfCtb = 0 + while rowsOfCtb < rowsPerTbl: + for i in range(ctbNum): + sql += " %s.%s_%d values "%(dbName,ctbPrefix,i) + for k in range(batchNum): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i]) + ctbDict[i] += 1 + if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl): + tsql.execute(sql) + sql = "insert into " + break + rowsOfCtb = ctbDict[0] + + tdLog.debug("insert data ............ [OK]") + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s_%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s_%d values " %(ctbPrefix,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data wiht auto create child table ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'autodata_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareEnv(self, **parameterDict): + # create new connector for my thread + tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030) + + if parameterDict["actionType"] == actionType.CREATE_DATABASE: + self.create_database(tsql, parameterDict["dbName"]) + elif parameterDict["actionType"] == actionType.CREATE_STABLE: + self.create_stable(tsql, parameterDict["dbName"], parameterDict["stbName"]) + elif parameterDict["actionType"] == actionType.CREATE_CTABLE: + self.create_ctables(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["stbName"], parameterDict["ctbNum"]) + elif parameterDict["actionType"] == actionType.INSERT_DATA: + self.insert_data(tsql, parameterDict["dbName"], parameterDict["stbName"], parameterDict["ctbNum"],\ + parameterDict["rowsPerTbl"],parameterDict["batchNum"]) + else: + tdLog.exit("not support's action: ", parameterDict["actionType"]) + + return + + def tmqCase1(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 1: ") + ''' + subscribe one db, multi normal table which have not same schema, and include rows of all tables in one insert sql + ''' + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db1', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + self.create_database(tdSql, parameterDict["dbName"]) + tdSql.execute("create table %s.ntb0 (ts timestamp, c1 int)"%(parameterDict["dbName"])) + tdSql.execute("create table %s.ntb1 (ts timestamp, c1 int, c2 float)"%(parameterDict["dbName"])) + tdSql.execute("create table %s.ntb2 (ts timestamp, c1 int, c2 float, c3 binary(32))"%(parameterDict["dbName"])) + tdSql.execute("create table %s.ntb3 (ts timestamp, c1 int, c2 float, c3 binary(32), c4 timestamp)"%(parameterDict["dbName"])) + + tdSql.execute("insert into %s.ntb0 values(now, 1) %s.ntb1 values(now, 1, 1) %s.ntb2 values(now, 1, 1, '1') %s.ntb3 values(now, 1, 1, '1', now)"%(parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"])) + tdSql.execute("insert into %s.ntb0 values(now, 2)(now+1s, 3) \ + %s.ntb1 values(now, 2, 2)(now+1s, 3, 3) \ + %s.ntb2 values(now, 2, 2, '2')(now+1s, 3, 3, '3') \ + %s.ntb3 values(now, 2, 2, '2', now)(now+1s, 3, 3, '3', now)"\ + %(parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"])) + tdSql.execute("insert into %s.ntb0 values(now, 4)(now+1s, 5) \ + %s.ntb1 values(now, 4, 4)(now+1s, 5, 5) \ + %s.ntb2 values(now, 4, 4, '4')(now+1s, 5, 5, '5') \ + %s.ntb3 values(now, 4, 4, '4', now)(now+1s, 5, 5, '5', now) \ + %s.ntb0 values(now+2s, 6)(now+3s, 7) \ + %s.ntb1 values(now+2s, 6, 6)(now+3s, 7, 7) \ + %s.ntb2 values(now+2s, 6, 6, '6')(now+3s, 7, 7, '7') \ + %s.ntb3 values(now+2s, 6, 6, '6', now)(now+3s, 7, 7, '7', now)"\ + %(parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"],parameterDict["dbName"])) + numOfNtb = 4 + rowsOfPerNtb = 7 + + tdLog.info("create topics from db") + topicFromDb = 'topic_db_mulit_tbl' + + tdSql.execute("create topic %s as database %s" %(topicFromDb, parameterDict['dbName'])) + consumerId = 0 + expectrowcnt = numOfNtb * rowsOfPerNtb + topicList = topicFromDb + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,enable.auto.commit:false,\ + auto.commit.interval.ms:6000,auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 10 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromDb) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 2: ") + ''' + subscribe one stb, multi child talbe and normal table which have not same schema, and include rows of all tables in one insert sql + ''' + self.initConsumerTable() + + # create and start thread + parameterDict = {'cfg': '', \ + 'actionType': 0, \ + 'dbName': 'db2', \ + 'dropFlag': 1, \ + 'vgroups': 4, \ + 'replica': 1, \ + 'stbName': 'stb1', \ + 'ctbNum': 10, \ + 'rowsPerTbl': 10000, \ + 'batchNum': 100, \ + 'startTs': 1640966400000} # 2022-01-01 00:00:00.000 + parameterDict['cfg'] = cfgPath + + dbName = parameterDict["dbName"] + + self.create_database(tdSql, dbName) + + tdSql.execute("create stable %s.stb (ts timestamp, s1 bigint, s2 binary(32), s3 double) tags (t1 int, t2 binary(32))"%(dbName)) + tdSql.execute("create table %s.ctb0 using %s.stb tags(0, 'ctb0')"%(dbName,dbName)) + tdSql.execute("create table %s.ctb1 using %s.stb tags(1, 'ctb1')"%(dbName,dbName)) + + tdSql.execute("create table %s.ntb0 (ts timestamp, c1 binary(32))"%(dbName)) + tdSql.execute("create table %s.ntb1 (ts timestamp, c1 binary(32), c2 float)"%(dbName)) + tdSql.execute("create table %s.ntb2 (ts timestamp, c1 int, c2 float, c3 binary(32))"%(dbName)) + tdSql.execute("create table %s.ntb3 (ts timestamp, c1 int, c2 float, c3 binary(32), c4 timestamp)"%(dbName)) + + tdSql.execute("insert into %s.ntb0 values(now, 'ntb0-11') \ + %s.ntb1 values(now, 'ntb1', 11) \ + %s.ntb2 values(now, 11, 11, 'ntb2') \ + %s.ctb0 values(now, 11, 'ctb0', 11) \ + %s.ntb3 values(now, 11, 11, 'ntb3', now) \ + %s.ctb1 values(now, 11, 'ctb1', 11)"\ + %(dbName,dbName,dbName,dbName,dbName,dbName)) + + tdSql.execute("insert into %s.ntb0 values(now, 'ntb0-12')(now+1s, 'ntb0-13') \ + %s.ntb1 values(now, 'ntb1', 12)(now+1s, 'ntb1', 13) \ + %s.ntb2 values(now, 12, 12, 'ntb2')(now+1s, 13, 13, 'ntb2') \ + %s.ctb0 values(now, 12, 'ctb0', 12)(now+1s, 13, 'ctb0', 13) \ + %s.ntb3 values(now, 12, 12, 'ntb3', now)(now+1s, 13, 13, 'ntb3', now) \ + %s.ctb1 values(now, 12, 'ctb1', 12)(now+1s, 13, 'ctb1', 13)"\ + %(dbName,dbName,dbName,dbName,dbName,dbName)) + tdSql.execute("insert into %s.ntb0 values(now, 'ntb0-14')(now+1s, 'ntb0-15') \ + %s.ntb1 values(now, 'ntb1', 14)(now+1s, 'ntb1', 15) \ + %s.ntb2 values(now, 14, 14, 'ntb2')(now+1s, 15, 15, 'ntb2') \ + %s.ctb0 values(now, 14, 'ctb0', 14)(now+1s, 15, 'ctb0', 15) \ + %s.ntb3 values(now, 14, 14, 'ntb3', now)(now+1s, 15, 15, 'ntb3', now) \ + %s.ctb1 values(now, 14, 'ctb1', 14)(now+1s, 15, 'ctb1', 15) \ + %s.ntb0 values(now+2s, 'ntb0-16')(now+3s, 'ntb0-17') \ + %s.ntb1 values(now+2s, 'ntb1', 16)(now+3s, 'ntb1', 17) \ + %s.ntb2 values(now+2s, 16, 16, 'ntb2')(now+3s, 17, 17, 'ntb2') \ + %s.ctb0 values(now+2s, 16, 'ctb0', 16)(now+3s, 17, 'ctb0', 17) \ + %s.ntb3 values(now+2s, 16, 16, 'ntb3', now)(now+3s, 17, 17, 'ntb3', now) \ + %s.ctb1 values(now+2s, 16, 'ctb1', 16)(now+3s, 17, 'ctb1', 17)"\ + %(dbName,dbName,dbName,dbName,dbName,dbName,dbName,dbName,dbName,dbName,dbName,dbName)) + numOfNtb = 4 + numOfCtb = 2 + rowsOfPerNtb = 7 + + tdLog.info("create topics from db") + topicFromStb = 'topic_stb_mulit_tbl' + + tdSql.execute("create topic %s as stable %s.stb" %(topicFromStb, dbName)) + consumerId = 0 + expectrowcnt = numOfCtb * rowsOfPerNtb + topicList = topicFromStb + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:cgrp1,enable.auto.commit:false,\ + auto.commit.interval.ms:6000,auto.offset.reset:earliest' + self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + pollDelay = 10 + showMsg = 1 + showRow = 1 + self.startTmqSimProcess(buildPath,cfgPath,pollDelay,dbName,showMsg, showRow) + + tdLog.info("insert process end, and start to check consume result") + expectRows = 1 + resultList = self.selectConsumeResult(expectRows) + totalConsumeRows = 0 + for i in range(expectRows): + totalConsumeRows += resultList[i] + + if totalConsumeRows != expectrowcnt: + tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) + tdLog.exit("tmq consume rows error!") + + tdSql.query("drop topic %s"%topicFromStb) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def tmqCase3(self, cfgPath, buildPath): + tdLog.printNoPrefix("======== test case 3: ") + + tdLog.printNoPrefix("======== test case 3 end ...... ") + + + def run(self): + tdSql.prepare() + + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + cfgPath = buildPath + "/../sim/psim/cfg" + tdLog.info("cfgPath: %s" % cfgPath) + + # self.tmqCase1(cfgPath, buildPath) + self.tmqCase2(cfgPath, buildPath) + # self.tmqCase3(cfgPath, buildPath) + # self.tmqCase4(cfgPath, buildPath) + # self.tmqCase5(cfgPath, buildPath) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From 3a4044dabae9d8ee2f8212fac784fadf9704a1cc Mon Sep 17 00:00:00 2001 From: Bo Ding Date: Thu, 2 Jun 2022 00:08:36 +0800 Subject: [PATCH 28/47] docs: fix include error --- docs-cn/14-reference/03-connector/cpp.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs-cn/14-reference/03-connector/cpp.mdx b/docs-cn/14-reference/03-connector/cpp.mdx index aba1d6c717..aecf9fde12 100644 --- a/docs-cn/14-reference/03-connector/cpp.mdx +++ b/docs-cn/14-reference/03-connector/cpp.mdx @@ -114,7 +114,6 @@ TDengine 客户端驱动的安装请参考 [安装指南](/reference/connector# 订阅和消费 ```c -{{#include examples/c/subscribe.c}} ``` From 7b7cbd54704ed73f81a6cc768feb0c09625db887 Mon Sep 17 00:00:00 2001 From: Bo Ding Date: Thu, 2 Jun 2022 00:11:16 +0800 Subject: [PATCH 29/47] docs: fix include error --- docs-en/14-reference/03-connector/cpp.mdx | 1 - 1 file changed, 1 deletion(-) diff --git a/docs-en/14-reference/03-connector/cpp.mdx b/docs-en/14-reference/03-connector/cpp.mdx index d13a74384c..d549413012 100644 --- a/docs-en/14-reference/03-connector/cpp.mdx +++ b/docs-en/14-reference/03-connector/cpp.mdx @@ -114,7 +114,6 @@ This section shows sample code for standard access methods to TDengine clusters Subscribe and consume ```c -{{#include examples/c/subscribe.c}} ``` From e1fd4a66409857af294017562018e47bb26d359a Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 2 Jun 2022 10:18:11 +0800 Subject: [PATCH 30/47] feat(tmq): get_db api --- examples/c/tmq.c | 1 + include/client/taos.h | 5 +++-- include/common/tmsg.h | 22 +++++++++------------- include/util/tdef.h | 2 +- source/client/inc/clientInt.h | 1 + source/client/src/tmq.c | 14 +++++++++++++- source/dnode/mnode/impl/inc/mndDef.h | 5 +++-- source/dnode/mnode/impl/src/mndConsumer.c | 1 + source/dnode/mnode/impl/src/mndSubscribe.c | 3 --- source/dnode/mnode/impl/src/mndTopic.c | 13 +++++++------ source/dnode/vnode/src/tq/tq.c | 12 +++++++++--- 11 files changed, 48 insertions(+), 31 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 40d72d3af1..2e8aa21da7 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -24,6 +24,7 @@ static void msg_process(TAOS_RES* msg) { char buf[1024]; /*memset(buf, 0, 1024);*/ printf("topic: %s\n", tmq_get_topic_name(msg)); + printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); while (1) { TAOS_ROW row = taos_fetch_row(msg); diff --git a/include/client/taos.h b/include/client/taos.h index bab0c18db1..b65091f52b 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -144,8 +144,8 @@ DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *nam DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); DLL_EXPORT int taos_stmt_set_tags(TAOS_STMT *stmt, TAOS_MULTI_BIND *tags); DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields); -DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int* fieldNum, TAOS_FIELD_E** fields); +DLL_EXPORT int taos_stmt_get_tag_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); +DLL_EXPORT int taos_stmt_get_col_fields(TAOS_STMT *stmt, int *fieldNum, TAOS_FIELD_E **fields); DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); @@ -269,6 +269,7 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); +DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index f332300a4f..21fed73f6d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2203,10 +2203,8 @@ typedef struct { int64_t newConsumerId; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int8_t subType; - // int8_t withTbName; - // int8_t withSchema; - // int8_t withTag; - char* qmsg; + char* qmsg; + int64_t suid; } SMqRebVgReq; static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pReq) { @@ -2217,11 +2215,10 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); tlen += taosEncodeString(buf, pReq->subKey); tlen += taosEncodeFixedI8(buf, pReq->subType); - // tlen += taosEncodeFixedI8(buf, pReq->withTbName); - // tlen += taosEncodeFixedI8(buf, pReq->withSchema); - // tlen += taosEncodeFixedI8(buf, pReq->withTag); if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { tlen += taosEncodeString(buf, pReq->qmsg); + } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + tlen += taosEncodeFixedI64(buf, pReq->suid); } return tlen; } @@ -2233,11 +2230,10 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); buf = taosDecodeStringTo(buf, pReq->subKey); buf = taosDecodeFixedI8(buf, &pReq->subType); - // buf = taosDecodeFixedI8(buf, &pReq->withTbName); - // buf = taosDecodeFixedI8(buf, &pReq->withSchema); - // buf = taosDecodeFixedI8(buf, &pReq->withTag); if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { buf = taosDecodeString(buf, &pReq->qmsg); + } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { + buf = taosDecodeFixedI64(buf, &pReq->suid); } return (void*)buf; } @@ -2471,7 +2467,7 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { typedef struct { char topic[TSDB_TOPIC_FNAME_LEN]; - int8_t isSchemaAdaptive; + char db[TSDB_DB_FNAME_LEN]; SArray* vgs; // SArray SSchemaWrapper schema; } SMqSubTopicEp; @@ -2479,7 +2475,7 @@ typedef struct { static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { int32_t tlen = 0; tlen += taosEncodeString(buf, pTopicEp->topic); - tlen += taosEncodeFixedI8(buf, pTopicEp->isSchemaAdaptive); + tlen += taosEncodeString(buf, pTopicEp->db); int32_t sz = taosArrayGetSize(pTopicEp->vgs); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { @@ -2492,7 +2488,7 @@ static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { buf = taosDecodeStringTo(buf, pTopicEp->topic); - buf = taosDecodeFixedI8(buf, &pTopicEp->isSchemaAdaptive); + buf = taosDecodeStringTo(buf, pTopicEp->db); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); diff --git a/include/util/tdef.h b/include/util/tdef.h index de139368c9..0ae22d1953 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -209,7 +209,7 @@ typedef enum ELogicConditionType { #define TSDB_INDEX_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_INDEX_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) -#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN +#define TSDB_TOPIC_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_STREAM_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_SUBSCRIBE_KEY_LEN (TSDB_CGROUP_LEN + TSDB_TOPIC_FNAME_LEN + 2) #define TSDB_PARTITION_KEY_LEN (TSDB_SUBSCRIBE_KEY_LEN + 20) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index d5e07ce676..0732c7890d 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -191,6 +191,7 @@ typedef struct SRequestSendRecvBody { typedef struct { int8_t resType; char topic[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; int32_t vgId; SSchemaWrapper schema; int32_t resIter; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 416d1a6f26..c2170631c2 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -143,6 +143,7 @@ typedef struct { typedef struct { // subscribe info char* topicName; + char db[TSDB_DB_FNAME_LEN]; SArray* vgs; // SArray @@ -1039,6 +1040,7 @@ bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { topic.schema = pTopicEp->schema; taosHashClear(pHash); topic.topicName = strdup(pTopicEp->topic); + tstrncpy(topic.db, pTopicEp->db, TSDB_DB_FNAME_LEN); tscDebug("consumer %ld update topic: %s", tmq->consumerId, topic.topicName); int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); @@ -1283,7 +1285,8 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); pRspObj->resType = RES_TYPE__TMQ; - strncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp)); @@ -1506,6 +1509,15 @@ const char* tmq_get_topic_name(TAOS_RES* res) { } } +const char* tmq_get_db_name(TAOS_RES* res) { + if (TD_RES_TMQ(res)) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + return strchr(pRspObj->db, '.') + 1; + } else { + return NULL; + } +} + int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index d0c737ae5a..4bf7b15593 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -168,7 +168,7 @@ typedef struct { int64_t createdTime; int64_t updateTime; SDnodeObj* pDnode; - SQnodeLoad load; + SQnodeLoad load; } SQnodeObj; typedef struct { @@ -422,6 +422,7 @@ typedef struct { char* ast; char* physicalPlan; SSchemaWrapper schema; + int64_t stbUid; // int32_t refConsumerCnt; } SMqTopicObj; @@ -535,7 +536,7 @@ typedef struct { } SMqRebOutputObj; typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; + char name[TSDB_STREAM_FNAME_LEN]; char sourceDb[TSDB_DB_FNAME_LEN]; char targetDb[TSDB_DB_FNAME_LEN]; char targetSTbName[TSDB_TABLE_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 0314891d59..1f8bf06993 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -306,6 +306,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { SMqTopicObj *pTopic = mndAcquireTopic(pMnode, topic); ASSERT(pTopic); taosRLockLatch(&pTopic->lock); + tstrncpy(topicEp.db, pTopic->db, TSDB_DB_FNAME_LEN); topicEp.schema.nCols = pTopic->schema.nCols; if (topicEp.schema.nCols) { topicEp.schema.pSchema = taosMemoryCalloc(topicEp.schema.nCols, sizeof(SSchema)); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index fc736809fd..3c43998e85 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -121,9 +121,6 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.vgId = pRebVg->pVgEp->vgId; req.qmsg = pRebVg->pVgEp->qmsg; req.subType = pSub->subType; - /*req.withTbName = pSub->withTbName;*/ - /*req.withSchema = pSub->withSchema;*/ - /*req.withTag = pSub->withTag;*/ strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 446992a245..b364f25b9d 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -375,13 +375,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * taosMemoryFree(topicObj.sql); return -1; } - /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ - /*topicObj.ast = NULL;*/ - /*topicObj.astLen = 0;*/ - /*topicObj.physicalPlan = NULL;*/ - /*topicObj.withTbName = 1;*/ - /*topicObj.withSchema = 1;*/ + } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { } + /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ + /*topicObj.ast = NULL;*/ + /*topicObj.astLen = 0;*/ + /*topicObj.physicalPlan = NULL;*/ + /*topicObj.withTbName = 1;*/ + /*topicObj.withSchema = 1;*/ STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index e79de255f3..b8c79608f1 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -264,14 +264,13 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { /*pExec->withSchema = req.withSchema;*/ /*pExec->withTag = req.withTag;*/ - pHandle->execHandle.exec.execCol.qmsg = req.qmsg; - req.qmsg = NULL; - pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); for (int32_t i = 0; i < 5; i++) { pHandle->execHandle.pExecReader[i] = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); } if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + pHandle->execHandle.exec.execCol.qmsg = req.qmsg; + req.qmsg = NULL; for (int32_t i = 0; i < 5; i++) { SReadHandle handle = { .reader = pHandle->execHandle.pExecReader[i], @@ -286,6 +285,13 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->execHandle.exec.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + int64_t suid = 0; + /*pHandle->execHandle.exec.execTb.suid = req.suid;*/ + SArray* tbUidList = taosArrayInit(0, sizeof(int16_t)); + tsdbGetAllTableList(pTq->pVnode->pMeta, suid, tbUidList); + for (int32_t i = 0; i < 5; i++) { + tqReadHandleSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList); + } } taosHashPut(pTq->handles, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); } else { From 7fceb5a7092a980e09063ebcd321954ab1b1323e Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 2 Jun 2022 10:22:34 +0800 Subject: [PATCH 31/47] other: code optimization and do more check --- include/common/taosdef.h | 1 + include/common/tdataformat.h | 3 ++- source/common/src/tdataformat.c | 22 +++++++++++++++++++--- source/dnode/vnode/src/meta/metaEntry.c | 2 ++ source/libs/executor/src/scanoperator.c | 4 ++-- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/include/common/taosdef.h b/include/common/taosdef.h index d39c7a1215..516df71b0b 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -97,6 +97,7 @@ extern char *qtypeStr[]; #undef TD_DEBUG_PRINT_ROW #undef TD_DEBUG_PRINT_TSDB_LOAD_DCOLS +#undef TD_DEBUG_PRINT_TAG #ifdef __cplusplus } diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 867115d16d..10bc6a6176 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -70,7 +70,8 @@ char* tTagValToData(const STagVal *pTagVal, bool isJson); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); -void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); +void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove +void debugCheckTags(STag *pTag); // TODO: remove // STRUCT ================= struct STColumn { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 77fc156492..636eeb6d61 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1008,6 +1008,21 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { printf("\n"); } +void debugCheckTags(STag *pTag) { + switch (pTag->flags) { + case 0x0: + case 0x20: + case 0x40: + case 0x60: + break; + default: + ASSERT(0); + } + + ASSERT(pTag->nTag <= 128 && pTag->nTag >= 0); + ASSERT(pTag->ver <= 512 && pTag->ver >= 0); // temp condition for pTag->ver +} + static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { int32_t n = 0; @@ -1114,9 +1129,11 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { } n += tPutTagVal(p + n, (STagVal *)taosArrayGet(pArray, iTag), isJson); } - +#ifdef TD_DEBUG_PRINT_TAG debugPrintSTag(*ppTag, __func__, __LINE__); +#endif + debugCheckTags(*ppTag); // TODO: remove this line after debug return code; _err: @@ -1199,8 +1216,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag) { } int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { - uint32_t len = 0; - return tDecodeBinary(pDecoder, (uint8_t **)ppTag, &len); + return tDecodeBinary(pDecoder, (uint8_t **)ppTag, NULL); } int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index a003494457..db99257ea7 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -30,6 +30,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; + debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; } else if (pME->type == TSDB_NORMAL_TABLE) { if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; @@ -62,6 +63,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) + debugCheckTags((STag*)pME->ctbEntry.pTags); // TODO: remove after debug } else if (pME->type == TSDB_NORMAL_TABLE) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c5d19981cf..0e682682a1 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -327,8 +327,8 @@ void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_ for (int32_t i = 0; i < pBlock->info.rows; ++i) { colDataAppend(pColInfoData, i, data, (data == NULL)); } - if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL && - IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ + if (data && (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) && p != NULL && + IS_VAR_DATA_TYPE(((const STagVal*)p)->type)) { taosMemoryFree(data); } } From edd0295a00dba5b145576230199814cf3c12e8f7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 2 Jun 2022 10:45:27 +0800 Subject: [PATCH 32/47] feat: interval distributed split --- include/libs/function/functionMgt.h | 3 + include/libs/nodes/nodes.h | 2 + include/libs/nodes/plannodes.h | 15 +- source/libs/function/inc/builtins.h | 26 +- source/libs/function/src/builtins.c | 33 +-- source/libs/function/src/functionMgt.c | 78 ++++++ source/libs/nodes/src/nodesCloneFuncs.c | 11 +- source/libs/nodes/src/nodesCodeFuncs.c | 196 ++++++++++++++- source/libs/nodes/src/nodesUtilFuncs.c | 4 + source/libs/planner/inc/planInt.h | 1 + source/libs/planner/src/planLogicCreater.c | 106 ++++---- source/libs/planner/src/planPhysiCreater.c | 68 +++++- source/libs/planner/src/planSpliter.c | 271 ++++++++++++++++++--- source/libs/planner/src/planUtil.c | 51 ++++ source/libs/planner/src/planner.c | 21 +- 15 files changed, 749 insertions(+), 137 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 922136b590..f3e28936af 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -156,6 +156,9 @@ bool fmIsDynamicScanOptimizedFunc(int32_t funcId); bool fmIsMultiResFunc(int32_t funcId); bool fmIsRepeatScanFunc(int32_t funcId); bool fmIsUserDefinedFunc(int32_t funcId); +bool fmIsDistExecFunc(int32_t funcId); + +int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); typedef enum EFuncDataRequired { FUNC_DATA_REQUIRED_DATA_LOAD = 1, diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 3860266725..9a974f1b0d 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -190,6 +190,7 @@ typedef enum ENodeType { QUERY_NODE_LOGIC_PLAN_PROJECT, QUERY_NODE_LOGIC_PLAN_VNODE_MODIF, QUERY_NODE_LOGIC_PLAN_EXCHANGE, + QUERY_NODE_LOGIC_PLAN_MERGE, QUERY_NODE_LOGIC_PLAN_WINDOW, QUERY_NODE_LOGIC_PLAN_FILL, QUERY_NODE_LOGIC_PLAN_SORT, @@ -207,6 +208,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_JOIN, QUERY_NODE_PHYSICAL_PLAN_AGG, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, + QUERY_NODE_PHYSICAL_PLAN_MERGE, QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 2648a468dd..26d0f93ca1 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -94,9 +94,15 @@ typedef struct SVnodeModifLogicNode { typedef struct SExchangeLogicNode { SLogicNode node; int32_t srcGroupId; - uint8_t precision; } SExchangeLogicNode; +typedef struct SMergeLogicNode { + SLogicNode node; + SNodeList* pMergeKeys; + int32_t numOfChannels; + int32_t srcGroupId; +} SMergeLogicNode; + typedef enum EWindowType { WINDOW_TYPE_INTERVAL = 1, WINDOW_TYPE_SESSION, WINDOW_TYPE_STATE } EWindowType; typedef struct SWindowLogicNode { @@ -265,6 +271,13 @@ typedef struct SExchangePhysiNode { SNodeList* pSrcEndPoints; // element is SDownstreamSource, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhysiNode; +typedef struct SMergePhysiNode { + SPhysiNode node; + SNodeList* pMergeKeys; + int32_t numOfChannels; + int32_t srcGroupId; +} SMergePhysiNode; + typedef struct SWinodwPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of parameter expression of function diff --git a/source/libs/function/inc/builtins.h b/source/libs/function/inc/builtins.h index 3bd0f35bf5..bc91875006 100644 --- a/source/libs/function/inc/builtins.h +++ b/source/libs/function/inc/builtins.h @@ -26,22 +26,24 @@ typedef int32_t (*FTranslateFunc)(SFunctionNode* pFunc, char* pErrBuf, int32_t l typedef EFuncDataRequired (*FFuncDataRequired)(SFunctionNode* pFunc, STimeWindow* pTimeWindow); typedef struct SBuiltinFuncDefinition { - char name[FUNCTION_NAME_MAX_LENGTH]; - EFunctionType type; - uint64_t classification; - FTranslateFunc translateFunc; - FFuncDataRequired dataRequiredFunc; - FExecGetEnv getEnvFunc; - FExecInit initFunc; - FExecProcess processFunc; + const char* name; + EFunctionType type; + uint64_t classification; + FTranslateFunc translateFunc; + FFuncDataRequired dataRequiredFunc; + FExecGetEnv getEnvFunc; + FExecInit initFunc; + FExecProcess processFunc; FScalarExecProcess sprocessFunc; - FExecFinalize finalizeFunc; - FExecProcess invertFunc; - FExecCombine combineFunc; + FExecFinalize finalizeFunc; + FExecProcess invertFunc; + FExecCombine combineFunc; + const char* pPartialFunc; + const char* pMergeFunc; } SBuiltinFuncDefinition; extern const SBuiltinFuncDefinition funcMgtBuiltins[]; -extern const int funcMgtBuiltinsNum; +extern const int funcMgtBuiltinsNum; #ifdef __cplusplus } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 4725f11715..dc812b3a83 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -156,14 +156,14 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - //param0 + // param0 SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The first parameter of PERCENTILE function can only be column"); } - //param1 + // param1 SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); if (pValue->datum.i < 0 || pValue->datum.i > 100) { @@ -178,7 +178,7 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - //set result type + // set result type pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; return TSDB_CODE_SUCCESS; } @@ -197,14 +197,14 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - //param0 + // param0 SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The first parameter of APERCENTILE function can only be column"); } - //param1 + // param1 SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1); if (nodeType(pParamNode1) != QUERY_NODE_VALUE) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -223,7 +223,7 @@ static int32_t translateApercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - //param2 + // param2 if (3 == numOfParams) { uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type; if (!IS_VAR_DATA_TYPE(para3Type)) { @@ -263,14 +263,14 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - //param0 + // param0 SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The first parameter of TOP/BOTTOM function can only be column"); } - //param1 + // param1 SNode* pParamNode1 = nodesListGetNode(pFunc->pParameterList, 1); if (nodeType(pParamNode1) != QUERY_NODE_VALUE) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -287,7 +287,7 @@ static int32_t translateTop(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { pValue->notReserved = true; - //set result type + // set result type SDataType* pType = &((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; pFunc->node.resType = (SDataType){.bytes = pType->bytes, .type = pType->type}; return TSDB_CODE_SUCCESS; @@ -659,8 +659,8 @@ static int32_t translateTail(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (pValue->datum.i < ((i > 1) ? 0 : 1) || pValue->datum.i > 100) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "TAIL function second parameter should be in range [1, 100], " - "third parameter should be in range [0, 100]"); + "TAIL function second parameter should be in range [1, 100], " + "third parameter should be in range [0, 100]"); } pValue->notReserved = true; @@ -721,7 +721,7 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - //param0 + // param0 SNode* pParamNode0 = nodesListGetNode(pFunc->pParameterList, 0); if (nodeType(pParamNode0) != QUERY_NODE_COLUMN) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, @@ -729,12 +729,11 @@ static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && - TSDB_DATA_TYPE_BOOL != colType) { + if (!IS_SIGNED_NUMERIC_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - //param1 + // param1 if (numOfParams == 2) { uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; if (!IS_INTEGER_TYPE(paraType)) { @@ -852,7 +851,7 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) if (3 == numOfParams) { SExprNode* p2 = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 2); - uint8_t para2Type = p2->resType.type; + uint8_t para2Type = p2->resType.type; if (!IS_INTEGER_TYPE(para2Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -993,6 +992,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = functionFinalize, .invertFunc = countInvertFunction, .combineFunc = combineFunction, + // .pPartialFunc = "count", + // .pMergeFunc = "sum" }, { .name = "sum", diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index c2b325bc92..611ae8d81f 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -199,3 +199,81 @@ bool fmIsInvertible(int32_t funcId) { } return res; } + +static SFunctionNode* createFunction(const char* pName, SNodeList* pParameterList) { + SFunctionNode* pFunc = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + return NULL; + } + strcpy(pFunc->functionName, pName); + pFunc->pParameterList = pParameterList; + char msg[64] = {0}; + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc, msg, sizeof(msg))) { + nodesDestroyNode(pFunc); + return NULL; + } + return pFunc; +} + +static SColumnNode* createColumnByFunc(const SFunctionNode* pFunc) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return NULL; + } + strcpy(pCol->colName, pFunc->node.aliasName); + pCol->node.resType = pFunc->node.resType; + return pCol; +} + +bool fmIsDistExecFunc(int32_t funcId) { + if (!fmIsVectorFunc(funcId)) { + return true; + } + return (NULL != funcMgtBuiltins[funcId].pPartialFunc && NULL != funcMgtBuiltins[funcId].pMergeFunc); +} + +static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNode** pPartialFunc) { + SNodeList* pParameterList = nodesCloneList(pSrcFunc->pParameterList); + if (NULL == pParameterList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + *pPartialFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pPartialFunc, pParameterList); + if (NULL == *pPartialFunc) { + nodesDestroyList(pParameterList); + return TSDB_CODE_OUT_OF_MEMORY; + } + snprintf((*pPartialFunc)->node.aliasName, sizeof((*pPartialFunc)->node.aliasName), "%s.%p", + (*pPartialFunc)->functionName, pSrcFunc); + return TSDB_CODE_SUCCESS; +} + +static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctionNode* pPartialFunc, + SFunctionNode** pMergeFunc) { + SNodeList* pParameterList = NULL; + nodesListMakeStrictAppend(&pParameterList, createColumnByFunc(pPartialFunc)); + *pMergeFunc = createFunction(funcMgtBuiltins[pSrcFunc->funcId].pMergeFunc, pParameterList); + if (NULL == *pMergeFunc) { + nodesDestroyList(pParameterList); + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy((*pMergeFunc)->node.aliasName, pSrcFunc->node.aliasName); + return TSDB_CODE_SUCCESS; +} + +int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc) { + if (!fmIsDistExecFunc(pFunc->funcId)) { + return TSDB_CODE_FAILED; + } + + int32_t code = createPartialFunction(pFunc, pPartialFunc); + if (TSDB_CODE_SUCCESS == code) { + code = createMergeFunction(pFunc, *pPartialFunc, pMergeFunc); + } + + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(*pPartialFunc); + nodesDestroyNode(*pMergeFunc); + } + + return code; +} diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 68d3741b48..0973435e89 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -366,7 +366,14 @@ static SNode* logicVnodeModifCopy(const SVnodeModifLogicNode* pSrc, SVnodeModifL static SNode* logicExchangeCopy(const SExchangeLogicNode* pSrc, SExchangeLogicNode* pDst) { COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); COPY_SCALAR_FIELD(srcGroupId); - COPY_SCALAR_FIELD(precision); + return (SNode*)pDst; +} + +static SNode* logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, logicNodeCopy); + CLONE_NODE_LIST_FIELD(pMergeKeys); + COPY_SCALAR_FIELD(numOfChannels); + COPY_SCALAR_FIELD(srcGroupId); return (SNode*)pDst; } @@ -529,6 +536,8 @@ SNodeptr nodesCloneNode(const SNodeptr pNode) { return logicVnodeModifCopy((const SVnodeModifLogicNode*)pNode, (SVnodeModifLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return logicExchangeCopy((const SExchangeLogicNode*)pNode, (SExchangeLogicNode*)pDst); + case QUERY_NODE_LOGIC_PLAN_MERGE: + return logicMergeCopy((const SMergeLogicNode*)pNode, (SMergeLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_WINDOW: return logicWindowCopy((const SWindowLogicNode*)pNode, (SWindowLogicNode*)pDst); case QUERY_NODE_LOGIC_PLAN_FILL: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 78710569cb..72e2a54a12 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -190,6 +190,8 @@ const char* nodesNodeName(ENodeType type) { return "LogicVnodeModif"; case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return "LogicExchange"; + case QUERY_NODE_LOGIC_PLAN_MERGE: + return "LogicMerge"; case QUERY_NODE_LOGIC_PLAN_WINDOW: return "LogicWindow"; case QUERY_NODE_LOGIC_PLAN_FILL: @@ -220,6 +222,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiAgg"; case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return "PhysiExchange"; + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + return "PhysiMerge"; case QUERY_NODE_PHYSICAL_PLAN_SORT: return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: @@ -596,7 +600,6 @@ static int32_t jsonToLogicProjectNode(const SJson* pJson, void* pObj) { } static const char* jkExchangeLogicPlanSrcGroupId = "SrcGroupId"; -static const char* jkExchangeLogicPlanSrcPrecision = "Precision"; static int32_t logicExchangeNodeToJson(const void* pObj, SJson* pJson) { const SExchangeLogicNode* pNode = (const SExchangeLogicNode*)pObj; @@ -605,9 +608,6 @@ static int32_t logicExchangeNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcGroupId, pNode->srcGroupId); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkExchangeLogicPlanSrcPrecision, pNode->precision); - } return code; } @@ -619,8 +619,144 @@ static int32_t jsonToLogicExchangeNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetIntValue(pJson, jkExchangeLogicPlanSrcGroupId, &pNode->srcGroupId); } + + return code; +} + +static const char* jkMergeLogicPlanMergeKeys = "MergeKeys"; +static const char* jkMergeLogicPlanNumOfChannels = "NumOfChannels"; +static const char* jkMergeLogicPlanSrcGroupId = "SrcGroupId"; + +static int32_t logicMergeNodeToJson(const void* pObj, SJson* pJson) { + const SMergeLogicNode* pNode = (const SMergeLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUTinyIntValue(pJson, jkExchangeLogicPlanSrcPrecision, &pNode->precision); + code = nodeListToJson(pJson, jkMergeLogicPlanMergeKeys, pNode->pMergeKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkMergeLogicPlanNumOfChannels, pNode->numOfChannels); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkMergeLogicPlanSrcGroupId, pNode->srcGroupId); + } + + return code; +} + +static int32_t jsonToLogicMergeNode(const SJson* pJson, void* pObj) { + SMergeLogicNode* pNode = (SMergeLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkMergeLogicPlanMergeKeys, &pNode->pMergeKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkMergeLogicPlanNumOfChannels, &pNode->numOfChannels); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkMergeLogicPlanSrcGroupId, &pNode->srcGroupId); + } + + return code; +} + +static const char* jkWindowLogicPlanWinType = "WinType"; +static const char* jkWindowLogicPlanFuncs = "Funcs"; +static const char* jkWindowLogicPlanInterval = "Interval"; +static const char* jkWindowLogicPlanOffset = "Offset"; +static const char* jkWindowLogicPlanSliding = "Sliding"; +static const char* jkWindowLogicPlanIntervalUnit = "IntervalUnit"; +static const char* jkWindowLogicPlanSlidingUnit = "SlidingUnit"; +static const char* jkWindowLogicPlanSessionGap = "SessionGap"; +static const char* jkWindowLogicPlanTspk = "Tspk"; +static const char* jkWindowLogicPlanStateExpr = "StateExpr"; +static const char* jkWindowLogicPlanTriggerType = "TriggerType"; +static const char* jkWindowLogicPlanWatermark = "Watermark"; + +static int32_t logicWindowNodeToJson(const void* pObj, SJson* pJson) { + const SWindowLogicNode* pNode = (const SWindowLogicNode*)pObj; + + int32_t code = logicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanWinType, pNode->winType); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkWindowLogicPlanFuncs, pNode->pFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanInterval, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanOffset, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanSliding, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanIntervalUnit, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanSlidingUnit, pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanSessionGap, pNode->sessionGap); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkWindowLogicPlanTspk, nodeToJson, pNode->pTspk); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkWindowLogicPlanStateExpr, nodeToJson, pNode->pStateExpr); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanTriggerType, pNode->triggerType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkWindowLogicPlanWatermark, pNode->watermark); + } + + return code; +} + +static int32_t jsonToLogicWindowNode(const SJson* pJson, void* pObj) { + SWindowLogicNode* pNode = (SWindowLogicNode*)pObj; + + int32_t code = jsonToLogicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + tjsonGetNumberValue(pJson, jkWindowLogicPlanWinType, pNode->winType, code); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkWindowLogicPlanFuncs, &pNode->pFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanInterval, &pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanOffset, &pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanSliding, &pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkWindowLogicPlanIntervalUnit, &pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkWindowLogicPlanSlidingUnit, &pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanSessionGap, &pNode->sessionGap); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkWindowLogicPlanTspk, &pNode->pTspk); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkWindowLogicPlanStateExpr, &pNode->pStateExpr); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkWindowLogicPlanTriggerType, &pNode->triggerType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkWindowLogicPlanWatermark, &pNode->watermark); } return code; @@ -1453,6 +1589,44 @@ static int32_t jsonToPhysiExchangeNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkMergePhysiPlanMergeKeys = "MergeKeys"; +static const char* jkMergePhysiPlanNumOfChannels = "NumOfChannels"; +static const char* jkMergePhysiPlanSrcGroupId = "SrcGroupId"; + +static int32_t physiMergeNodeToJson(const void* pObj, SJson* pJson) { + const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj; + + int32_t code = physicPlanNodeToJson(pObj, pJson); + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkMergePhysiPlanMergeKeys, pNode->pMergeKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanNumOfChannels, pNode->numOfChannels); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkMergePhysiPlanSrcGroupId, pNode->srcGroupId); + } + + return code; +} + +static int32_t jsonToPhysiMergeNode(const SJson* pJson, void* pObj) { + SMergePhysiNode* pNode = (SMergePhysiNode*)pObj; + + int32_t code = jsonToPhysicPlanNode(pJson, pObj); + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkMergePhysiPlanMergeKeys, &pNode->pMergeKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkMergePhysiPlanNumOfChannels, &pNode->numOfChannels); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkMergePhysiPlanSrcGroupId, &pNode->srcGroupId); + } + + return code; +} + static const char* jkSortPhysiPlanExprs = "Exprs"; static const char* jkSortPhysiPlanSortKeys = "SortKeys"; static const char* jkSortPhysiPlanTargets = "Targets"; @@ -3388,6 +3562,10 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { break; case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return logicExchangeNodeToJson(pObj, pJson); + case QUERY_NODE_LOGIC_PLAN_MERGE: + return logicMergeNodeToJson(pObj, pJson); + case QUERY_NODE_LOGIC_PLAN_WINDOW: + return logicWindowNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_FILL: return logicFillNodeToJson(pObj, pJson); case QUERY_NODE_LOGIC_PLAN_SORT: @@ -3414,6 +3592,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return physiAggNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return physiExchangeNodeToJson(pObj, pJson); + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + return physiMergeNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: @@ -3499,6 +3679,10 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicProjectNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return jsonToLogicExchangeNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_MERGE: + return jsonToLogicMergeNode(pJson, pObj); + case QUERY_NODE_LOGIC_PLAN_WINDOW: + return jsonToLogicWindowNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_FILL: return jsonToLogicFillNode(pJson, pObj); case QUERY_NODE_LOGIC_PLAN_SORT: @@ -3525,6 +3709,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToPhysiAggNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return jsonToPhysiExchangeNode(pJson, pObj); + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + return jsonToPhysiMergeNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index e28844f2e1..fa9cb40d17 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -222,6 +222,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SVnodeModifLogicNode)); case QUERY_NODE_LOGIC_PLAN_EXCHANGE: return makeNode(type, sizeof(SExchangeLogicNode)); + case QUERY_NODE_LOGIC_PLAN_MERGE: + return makeNode(type, sizeof(SMergeLogicNode)); case QUERY_NODE_LOGIC_PLAN_WINDOW: return makeNode(type, sizeof(SWindowLogicNode)); case QUERY_NODE_LOGIC_PLAN_FILL: @@ -252,6 +254,8 @@ SNodeptr nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SAggPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: return makeNode(type, sizeof(SExchangePhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + return makeNode(type, sizeof(SMergePhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_SORT: return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERVAL: diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index 6a18a267e2..1a8c7657df 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -36,6 +36,7 @@ extern "C" { #define planTrace(param, ...) qTrace("PLAN: " param, __VA_ARGS__) int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...); +int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList); int32_t createLogicPlan(SPlanContext* pCxt, SLogicNode** pLogicNode); int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicNode* pLogicNode); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 467b26b7c4..52393c5707 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -132,56 +132,56 @@ static int32_t createChildLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelec return code; } -typedef struct SCreateColumnCxt { - int32_t errCode; - SNodeList* pList; -} SCreateColumnCxt; +// typedef struct SCreateColumnCxt { +// int32_t errCode; +// SNodeList* pList; +// } SCreateColumnCxt; -static EDealRes doCreateColumn(SNode* pNode, void* pContext) { - SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; - switch (nodeType(pNode)) { - case QUERY_NODE_COLUMN: { - SNode* pCol = nodesCloneNode(pNode); - if (NULL == pCol) { - return DEAL_RES_ERROR; - } - return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); - } - case QUERY_NODE_OPERATOR: - case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_FUNCTION: { - SExprNode* pExpr = (SExprNode*)pNode; - SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); - if (NULL == pCol) { - return DEAL_RES_ERROR; - } - pCol->node.resType = pExpr->resType; - strcpy(pCol->colName, pExpr->aliasName); - return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); - } - default: - break; - } +// static EDealRes doCreateColumn(SNode* pNode, void* pContext) { +// SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; +// switch (nodeType(pNode)) { +// case QUERY_NODE_COLUMN: { +// SNode* pCol = nodesCloneNode(pNode); +// if (NULL == pCol) { +// return DEAL_RES_ERROR; +// } +// return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); +// } +// case QUERY_NODE_OPERATOR: +// case QUERY_NODE_LOGIC_CONDITION: +// case QUERY_NODE_FUNCTION: { +// SExprNode* pExpr = (SExprNode*)pNode; +// SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); +// if (NULL == pCol) { +// return DEAL_RES_ERROR; +// } +// pCol->node.resType = pExpr->resType; +// strcpy(pCol->colName, pExpr->aliasName); +// return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); +// } +// default: +// break; +// } - return DEAL_RES_CONTINUE; -} +// return DEAL_RES_CONTINUE; +// } -static int32_t createColumnByRewriteExps(SLogicPlanContext* pCxt, SNodeList* pExprs, SNodeList** pList) { - SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; - if (NULL == cxt.pList) { - return TSDB_CODE_OUT_OF_MEMORY; - } +// static int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList) { +// SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; +// if (NULL == cxt.pList) { +// return TSDB_CODE_OUT_OF_MEMORY; +// } - nodesWalkExprs(pExprs, doCreateColumn, &cxt); - if (TSDB_CODE_SUCCESS != cxt.errCode) { - nodesDestroyList(cxt.pList); - return cxt.errCode; - } - if (NULL == *pList) { - *pList = cxt.pList; - } - return cxt.errCode; -} +// nodesWalkExprs(pExprs, doCreateColumn, &cxt); +// if (TSDB_CODE_SUCCESS != cxt.errCode) { +// nodesDestroyList(cxt.pList); +// return cxt.errCode; +// } +// if (NULL == *pList) { +// *pList = cxt.pList; +// } +// return cxt.errCode; +// } static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols, STableMeta* pMeta) { @@ -293,10 +293,10 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect // set output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pCxt, pScan->pScanCols, &pScan->node.pTargets); + code = createColumnByRewriteExps(pScan->pScanCols, &pScan->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pCxt, pScan->pScanPseudoCols, &pScan->node.pTargets); + code = createColumnByRewriteExps(pScan->pScanPseudoCols, &pScan->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -461,10 +461,10 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, // set the output if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pGroupKeys) { - code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); + code = createColumnByRewriteExps(pAgg->pGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code && NULL != pAgg->pAggFuncs) { - code = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs, &pAgg->node.pTargets); + code = createColumnByRewriteExps(pAgg->pAggFuncs, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -490,7 +490,7 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm } if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pCxt, pWindow->pFuncs, &pWindow->node.pTargets); + code = createColumnByRewriteExps(pWindow->pFuncs, &pWindow->node.pTargets); } pSelect->hasAggFuncs = false; @@ -760,7 +760,7 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe // set the output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); + code = createColumnByRewriteExps(pAgg->pGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { @@ -907,7 +907,7 @@ static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pS // set the output if (TSDB_CODE_SUCCESS == code) { - code = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys, &pAgg->node.pTargets); + code = createColumnByRewriteExps(pAgg->pGroupKeys, &pAgg->node.pTargets); } if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index a45eabefb9..52f289601c 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -835,7 +835,7 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)makePhysiNode( - pCxt, pExchangeLogicNode->precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -845,10 +845,11 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic return TSDB_CODE_SUCCESS; } + static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, SPhysiNode** pPhyNode) { SScanPhysiNode* pScan = (SScanPhysiNode*)makePhysiNode( - pCxt, pExchangeLogicNode->precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + pCxt, pExchangeLogicNode->node.precision, (SLogicNode*)pExchangeLogicNode, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -949,7 +950,8 @@ static int32_t createSessionWindowPhysiNode(SPhysiPlanContext* pCxt, SNodeList* SWindowLogicNode* pWindowLogicNode, SPhysiNode** pPhyNode) { SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)makePhysiNode( pCxt, getPrecision(pChildren), (SLogicNode*)pWindowLogicNode, - (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW : QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW)); + (pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION_WINDOW + : QUERY_NODE_PHYSICAL_PLAN_SESSION_WINDOW)); if (NULL == pSession) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1132,6 +1134,54 @@ static int32_t createFillPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren return code; } +static int32_t createExchangePhysiNodeByMerge(SMergePhysiNode* pMerge) { + SExchangePhysiNode* pExchange = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_EXCHANGE); + if (NULL == pExchange) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pExchange->srcGroupId = pMerge->srcGroupId; + pExchange->node.pParent = (SPhysiNode*)pMerge; + pExchange->node.pOutputDataBlockDesc = nodesCloneNode(pMerge->node.pOutputDataBlockDesc); + if (NULL == pExchange->node.pOutputDataBlockDesc) { + nodesDestroyNode(pExchange); + return TSDB_CODE_OUT_OF_MEMORY; + } + return nodesListMakeStrictAppend(&pMerge->node.pChildren, pExchange); +} + +static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pMergeLogicNode, SPhysiNode** pPhyNode) { + SMergePhysiNode* pMerge = (SMergePhysiNode*)makePhysiNode( + pCxt, pMergeLogicNode->node.precision, (SLogicNode*)pMergeLogicNode, QUERY_NODE_PHYSICAL_PLAN_MERGE); + if (NULL == pMerge) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pMerge->numOfChannels = pMergeLogicNode->numOfChannels; + pMerge->srcGroupId = pMergeLogicNode->srcGroupId; + + int32_t code = TSDB_CODE_SUCCESS; + + for (int32_t i = 0; i < pMerge->numOfChannels; ++i) { + code = createExchangePhysiNodeByMerge(pMerge); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + + if (TSDB_CODE_SUCCESS == code) { + code = setListSlotId(pCxt, pMerge->node.pOutputDataBlockDesc->dataBlockId, -1, pMergeLogicNode->pMergeKeys, + &pMerge->pMergeKeys); + } + + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pMerge; + } else { + nodesDestroyNode(pMerge); + } + + return code; +} + static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, SSubplan* pSubplan, SNodeList* pChildren, SPhysiNode** pPhyNode) { switch (nodeType(pLogicNode)) { @@ -1153,6 +1203,8 @@ static int32_t doCreatePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return createPartitionPhysiNode(pCxt, pChildren, (SPartitionLogicNode*)pLogicNode, pPhyNode); case QUERY_NODE_LOGIC_PLAN_FILL: return createFillPhysiNode(pCxt, pChildren, (SFillLogicNode*)pLogicNode, pPhyNode); + case QUERY_NODE_LOGIC_PLAN_MERGE: + return createMergePhysiNode(pCxt, (SMergeLogicNode*)pLogicNode, pPhyNode); default: break; } @@ -1183,9 +1235,13 @@ static int32_t createPhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode, } if (TSDB_CODE_SUCCESS == code) { - (*pPhyNode)->pChildren = pChildren; - SNode* pChild; - FOREACH(pChild, (*pPhyNode)->pChildren) { ((SPhysiNode*)pChild)->pParent = (*pPhyNode); } + if (LIST_LENGTH(pChildren) > 0) { + (*pPhyNode)->pChildren = pChildren; + SNode* pChild; + FOREACH(pChild, (*pPhyNode)->pChildren) { ((SPhysiNode*)pChild)->pParent = (*pPhyNode); } + } else { + nodesDestroyList(pChildren); + } } else { nodesDestroyList(pChildren); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index cfa265b722..2ae749bef5 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "functionMgt.h" #include "planInt.h" #define SPLIT_FLAG_MASK(n) (1 << n) @@ -37,7 +38,17 @@ typedef struct SSplitRule { typedef bool (*FSplFindSplitNode)(SLogicSubplan* pSubplan, void* pInfo); -static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, int32_t flag) { +static void splSetSubplanVgroups(SLogicSubplan* pSubplan, SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pNode)->pVgroupList); + } else { + if (1 == LIST_LENGTH(pNode->pChildren)) { + splSetSubplanVgroups(pSubplan, (SLogicNode*)nodesListGetNode(pNode->pChildren, 0)); + } + } +} + +static SLogicSubplan* splCreateScanSubplan(SSplitContext* pCxt, SLogicNode* pNode, int32_t flag) { SLogicSubplan* pSubplan = nodesMakeNode(QUERY_NODE_LOGIC_SUBPLAN); if (NULL == pSubplan) { return NULL; @@ -45,10 +56,9 @@ static SLogicSubplan* splCreateSubplan(SSplitContext* pCxt, SLogicNode* pNode, i pSubplan->id.queryId = pCxt->queryId; pSubplan->id.groupId = pCxt->groupId; pSubplan->subplanType = SUBPLAN_TYPE_SCAN; - pSubplan->pNode = (SLogicNode*)nodesCloneNode(pNode); - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - TSWAP(pSubplan->pVgroupList, ((SScanLogicNode*)pSubplan->pNode)->pVgroupList); - } + pSubplan->pNode = pNode; + pSubplan->pNode->pParent = NULL; + splSetSubplanVgroups(pSubplan, pNode); SPLIT_FLAG_SET_MASK(pSubplan->splitFlag, flag); return pSubplan; } @@ -60,7 +70,7 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pCxt->groupId; - pExchange->precision = pSplitNode->precision; + pExchange->node.precision = pSplitNode->precision; pExchange->node.pTargets = nodesCloneList(pSplitNode->pTargets); if (NULL == pExchange->node.pTargets) { return TSDB_CODE_OUT_OF_MEMORY; @@ -77,7 +87,7 @@ static int32_t splCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla FOREACH(pNode, pSplitNode->pParent->pChildren) { if (nodesEqualNode(pNode, pSplitNode)) { REPLACE_NODE(pExchange); - nodesDestroyNode(pNode); + pExchange->node.pParent = pSplitNode->pParent; return TSDB_CODE_SUCCESS; } } @@ -101,13 +111,50 @@ static bool splMatch(SSplitContext* pCxt, SLogicSubplan* pSubplan, int32_t flag, } typedef struct SStableSplitInfo { - SScanLogicNode* pScan; - SLogicSubplan* pSubplan; + SLogicNode* pSplitNode; + SLogicSubplan* pSubplan; } SStableSplitInfo; +static bool stbSplHasGatherExecFunc(const SNodeList* pFuncs) { + SNode* pFunc = NULL; + FOREACH(pFunc, pFuncs) { + if (!fmIsDistExecFunc(((SFunctionNode*)pFunc)->funcId)) { + return true; + } + } + return false; +} + +static bool stbSplIsMultiTbScan(SScanLogicNode* pScan) { + return (NULL != pScan->pVgroupList && pScan->pVgroupList->numOfVgroups > 1); +} + +static bool stbSplHasMultiTbScan(SLogicNode* pNode) { + if (1 != LIST_LENGTH(pNode->pChildren)) { + return false; + } + SNode* pChild = nodesListGetNode(pNode->pChildren, 0); + return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan((SScanLogicNode*)pChild)); +} + +static bool stbSplNeedSplit(SLogicNode* pNode) { + switch (nodeType(pNode)) { + // case QUERY_NODE_LOGIC_PLAN_AGG: + // return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(pNode); + case QUERY_NODE_LOGIC_PLAN_WINDOW: + return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(pNode); + // case QUERY_NODE_LOGIC_PLAN_SORT: + // return stbSplHasMultiTbScan(pNode); + case QUERY_NODE_LOGIC_PLAN_SCAN: + return stbSplIsMultiTbScan((SScanLogicNode*)pNode); + default: + break; + } + return false; +} + static SLogicNode* stbSplMatchByNode(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode) && NULL != ((SScanLogicNode*)pNode)->pVgroupList && - ((SScanLogicNode*)pNode)->pVgroupList->numOfVgroups > 1) { + if (stbSplNeedSplit(pNode)) { return pNode; } SNode* pChild; @@ -123,22 +170,178 @@ static SLogicNode* stbSplMatchByNode(SLogicNode* pNode) { static bool stbSplFindSplitNode(SLogicSubplan* pSubplan, SStableSplitInfo* pInfo) { SLogicNode* pSplitNode = stbSplMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { - pInfo->pScan = (SScanLogicNode*)pSplitNode; + pInfo->pSplitNode = pSplitNode; pInfo->pSubplan = pSubplan; } return NULL != pSplitNode; } +static int32_t stbSplRewriteFuns(const SNodeList* pFuncs, SNodeList** pPartialFuncs, SNodeList** pMergeFuncs) { + SNode* pNode = NULL; + FOREACH(pNode, pFuncs) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + SFunctionNode* pPartFunc = NULL; + SFunctionNode* pMergeFunc = NULL; + int32_t code = TSDB_CODE_SUCCESS; + if (fmIsWindowPseudoColumnFunc(pFunc->funcId)) { + pPartFunc = nodesCloneNode(pFunc); + pMergeFunc = nodesCloneNode(pFunc); + if (NULL == pPartFunc || NULL == pMergeFunc) { + nodesDestroyNode(pPartFunc); + nodesDestroyNode(pMergeFunc); + code = TSDB_CODE_OUT_OF_MEMORY; + } + } else { + code = fmGetDistMethod(pFunc, &pPartFunc, &pMergeFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(pPartialFuncs, pPartFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(pMergeFuncs, pMergeFunc); + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyList(*pPartialFuncs); + nodesDestroyList(*pMergeFuncs); + return code; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t stbSplAppendWStart(SNodeList* pFuncs, int32_t* pIndex) { + int32_t index = 0; + SNode* pFunc = NULL; + FOREACH(pFunc, pFuncs) { + if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pFunc)->funcType) { + *pIndex = index; + return TSDB_CODE_SUCCESS; + } + ++index; + } + + SFunctionNode* pWStart = nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pWStart) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pWStart->functionName, "_wstartts"); + snprintf(pWStart->node.aliasName, sizeof(pWStart->node.aliasName), "%s.%p", pWStart->functionName, pWStart); + int32_t code = fmGetFuncInfo(pWStart, NULL, 0); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pFuncs, pWStart); + } + *pIndex = index; + return code; +} + +static int32_t stbSplCreatePartWindowNode(SWindowLogicNode* pMergeWindow, SLogicNode** pPartWindow) { + SNodeList* pFunc = pMergeWindow->pFuncs; + pMergeWindow->pFuncs = NULL; + SNodeList* pTargets = pMergeWindow->node.pTargets; + pMergeWindow->node.pTargets = NULL; + SNodeList* pChildren = pMergeWindow->node.pChildren; + pMergeWindow->node.pChildren = NULL; + + int32_t code = TSDB_CODE_SUCCESS; + SWindowLogicNode* pPartWin = nodesCloneNode(pMergeWindow); + if (NULL == pPartWin) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + + if (TSDB_CODE_SUCCESS == code) { + pMergeWindow->node.pTargets = pTargets; + pPartWin->node.pChildren = pChildren; + code = stbSplRewriteFuns(pFunc, &pPartWin->pFuncs, &pMergeWindow->pFuncs); + } + int32_t index = 0; + if (TSDB_CODE_SUCCESS == code) { + code = stbSplAppendWStart(pPartWin->pFuncs, &index); + } + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExps(pPartWin->pFuncs, &pPartWin->node.pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + nodesDestroyNode(pMergeWindow->pTspk); + pMergeWindow->pTspk = nodesCloneNode(nodesListGetNode(pPartWin->node.pTargets, index)); + if (NULL == pMergeWindow->pTspk) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + + nodesDestroyList(pFunc); + if (TSDB_CODE_SUCCESS == code) { + *pPartWindow = (SLogicNode*)pPartWin; + } else { + nodesDestroyNode(pPartWin); + } + + return code; +} + +static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicNode* pParent, SLogicNode* pPartChild) { + SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); + if (NULL == pMerge) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pMerge->numOfChannels = ((SScanLogicNode*)nodesListGetNode(pPartChild->pChildren, 0))->pVgroupList->numOfVgroups; + pMerge->srcGroupId = pCxt->groupId; + pMerge->node.pParent = pParent; + pMerge->node.precision = pPartChild->precision; + int32_t code = nodesListMakeStrictAppend(&pMerge->pMergeKeys, nodesCloneNode(((SWindowLogicNode*)pParent)->pTspk)); + if (TSDB_CODE_SUCCESS == code) { + pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); + if (NULL == pMerge->node.pTargets) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pParent->pChildren, pMerge); + } + + return code; +} + +static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SLogicNode* pPartWindow = NULL; + int32_t code = stbSplCreatePartWindowNode((SWindowLogicNode*)pInfo->pSplitNode, &pPartWindow); + if (TSDB_CODE_SUCCESS == code) { + code = stbSplCreateMergeNode(pCxt, pInfo->pSplitNode, pPartWindow); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + splCreateScanSubplan(pCxt, pPartWindow, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + return code; +} + +static int32_t stbSplSplitScanNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + int32_t code = splCreateExchangeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, SUBPLAN_TYPE_MERGE); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); + } + return code; +} + static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SStableSplitInfo info = {0}; if (!splMatch(pCxt, pSubplan, SPLIT_FLAG_STABLE_SPLIT, (FSplFindSplitNode)stbSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } - int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, - splCreateSubplan(pCxt, (SLogicNode*)info.pScan, SPLIT_FLAG_STABLE_SPLIT)); - if (TSDB_CODE_SUCCESS == code) { - code = splCreateExchangeNode(pCxt, info.pSubplan, (SLogicNode*)info.pScan, SUBPLAN_TYPE_MERGE); + + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(info.pSplitNode)) { + case QUERY_NODE_LOGIC_PLAN_WINDOW: + code = stbSplSplitWindowNode(pCxt, &info); + break; + case QUERY_NODE_LOGIC_PLAN_SCAN: + code = stbSplSplitScanNode(pCxt, &info); + break; + default: + break; } + ++(pCxt->groupId); pCxt->split = true; return code; @@ -187,9 +390,9 @@ static int32_t singleTableJoinSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)sigTbJoinSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } - int32_t code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateSubplan(pCxt, info.pSplitNode, 0)); + int32_t code = splCreateExchangeNode(pCxt, info.pSubplan, info.pSplitNode, info.pSubplan->subplanType); if (TSDB_CODE_SUCCESS == code) { - code = splCreateExchangeNode(pCxt, info.pSubplan, info.pSplitNode, info.pSubplan->subplanType); + code = nodesListMakeStrictAppend(&info.pSubplan->pChildren, splCreateScanSubplan(pCxt, info.pSplitNode, 0)); } ++(pCxt->groupId); pCxt->split = true; @@ -272,13 +475,13 @@ typedef struct SUnionAllSplitInfo { SLogicSubplan* pSubplan; } SUnionAllSplitInfo; -static SLogicNode* unionAllMatchByNode(SLogicNode* pNode) { +static SLogicNode* unAllSplMatchByNode(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_PROJECT == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { return pNode; } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = unionAllMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = unAllSplMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -286,8 +489,8 @@ static SLogicNode* unionAllMatchByNode(SLogicNode* pNode) { return NULL; } -static bool unionAllFindSplitNode(SLogicSubplan* pSubplan, SUnionAllSplitInfo* pInfo) { - SLogicNode* pSplitNode = unionAllMatchByNode(pSubplan->pNode); +static bool unAllSplFindSplitNode(SLogicSubplan* pSubplan, SUnionAllSplitInfo* pInfo) { + SLogicNode* pSplitNode = unAllSplMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pProject = (SProjectLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; @@ -295,13 +498,13 @@ static bool unionAllFindSplitNode(SLogicSubplan* pSubplan, SUnionAllSplitInfo* p return NULL != pSplitNode; } -static int32_t unionAllCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { +static int32_t unAllSplCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SProjectLogicNode* pProject) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pCxt->groupId; - pExchange->precision = pProject->node.precision; + pExchange->node.precision = pProject->node.precision; pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); if (NULL == pExchange->node.pTargets) { return TSDB_CODE_OUT_OF_MEMORY; @@ -329,13 +532,13 @@ static int32_t unionAllCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pS static int32_t unionAllSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SUnionAllSplitInfo info = {0}; - if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unionAllFindSplitNode, &info)) { + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unAllSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pProject); if (TSDB_CODE_SUCCESS == code) { - code = unionAllCreateExchangeNode(pCxt, info.pSubplan, info.pProject); + code = unAllSplCreateExchangeNode(pCxt, info.pSubplan, info.pProject); } ++(pCxt->groupId); pCxt->split = true; @@ -347,13 +550,13 @@ typedef struct SUnionDistinctSplitInfo { SLogicSubplan* pSubplan; } SUnionDistinctSplitInfo; -static SLogicNode* unionDistinctMatchByNode(SLogicNode* pNode) { +static SLogicNode* unDistSplMatchByNode(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && LIST_LENGTH(pNode->pChildren) > 1) { return pNode; } SNode* pChild; FOREACH(pChild, pNode->pChildren) { - SLogicNode* pSplitNode = unionDistinctMatchByNode((SLogicNode*)pChild); + SLogicNode* pSplitNode = unDistSplMatchByNode((SLogicNode*)pChild); if (NULL != pSplitNode) { return pSplitNode; } @@ -361,13 +564,13 @@ static SLogicNode* unionDistinctMatchByNode(SLogicNode* pNode) { return NULL; } -static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SAggLogicNode* pAgg) { +static int32_t unDistSplCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan, SAggLogicNode* pAgg) { SExchangeLogicNode* pExchange = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_EXCHANGE); if (NULL == pExchange) { return TSDB_CODE_OUT_OF_MEMORY; } pExchange->srcGroupId = pCxt->groupId; - // pExchange->precision = pScan->pMeta->tableInfo.precision; + pExchange->node.precision = pAgg->node.precision; pExchange->node.pTargets = nodesCloneList(pAgg->pGroupKeys); if (NULL == pExchange->node.pTargets) { return TSDB_CODE_OUT_OF_MEMORY; @@ -378,8 +581,8 @@ static int32_t unCreateExchangeNode(SSplitContext* pCxt, SLogicSubplan* pSubplan return nodesListMakeAppend(&pAgg->node.pChildren, pExchange); } -static bool unionDistinctFindSplitNode(SLogicSubplan* pSubplan, SUnionDistinctSplitInfo* pInfo) { - SLogicNode* pSplitNode = unionDistinctMatchByNode(pSubplan->pNode); +static bool unDistSplFindSplitNode(SLogicSubplan* pSubplan, SUnionDistinctSplitInfo* pInfo) { + SLogicNode* pSplitNode = unDistSplMatchByNode(pSubplan->pNode); if (NULL != pSplitNode) { pInfo->pAgg = (SAggLogicNode*)pSplitNode; pInfo->pSubplan = pSubplan; @@ -389,13 +592,13 @@ static bool unionDistinctFindSplitNode(SLogicSubplan* pSubplan, SUnionDistinctSp static int32_t unionDistinctSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { SUnionDistinctSplitInfo info = {0}; - if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unionDistinctFindSplitNode, &info)) { + if (!splMatch(pCxt, pSubplan, 0, (FSplFindSplitNode)unDistSplFindSplitNode, &info)) { return TSDB_CODE_SUCCESS; } int32_t code = unionSplitSubplan(pCxt, info.pSubplan, (SLogicNode*)info.pAgg); if (TSDB_CODE_SUCCESS == code) { - code = unCreateExchangeNode(pCxt, info.pSubplan, info.pAgg); + code = unDistSplCreateExchangeNode(pCxt, info.pSubplan, info.pAgg); } ++(pCxt->groupId); pCxt->split = true; diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c index 3c83d9f53a..63d31912f0 100644 --- a/source/libs/planner/src/planUtil.c +++ b/source/libs/planner/src/planUtil.c @@ -34,3 +34,54 @@ int32_t generateUsageErrMsg(char* pBuf, int32_t len, int32_t errCode, ...) { va_end(vArgList); return errCode; } + +typedef struct SCreateColumnCxt { + int32_t errCode; + SNodeList* pList; +} SCreateColumnCxt; + +static EDealRes doCreateColumn(SNode* pNode, void* pContext) { + SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext; + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: { + SNode* pCol = nodesCloneNode(pNode); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + case QUERY_NODE_OPERATOR: + case QUERY_NODE_LOGIC_CONDITION: + case QUERY_NODE_FUNCTION: { + SExprNode* pExpr = (SExprNode*)pNode; + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return DEAL_RES_ERROR; + } + pCol->node.resType = pExpr->resType; + strcpy(pCol->colName, pExpr->aliasName); + return (TSDB_CODE_SUCCESS == nodesListAppend(pCxt->pList, pCol) ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + default: + break; + } + + return DEAL_RES_CONTINUE; +} + +int32_t createColumnByRewriteExps(SNodeList* pExprs, SNodeList** pList) { + SCreateColumnCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pList = (NULL == *pList ? nodesMakeList() : *pList)}; + if (NULL == cxt.pList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + nodesWalkExprs(pExprs, doCreateColumn, &cxt); + if (TSDB_CODE_SUCCESS != cxt.errCode) { + nodesDestroyList(cxt.pList); + return cxt.errCode; + } + if (NULL == *pList) { + *pList = cxt.pList; + } + return cxt.errCode; +} diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index af62c52a89..f8d240c7b2 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -58,16 +58,19 @@ static int32_t setSubplanExecutionNode(SPhysiNode* pNode, int32_t groupId, SDown if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == nodeType(pNode)) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pNode; if (pExchange->srcGroupId == groupId) { - if (NULL == pExchange->pSrcEndPoints) { - pExchange->pSrcEndPoints = nodesMakeList(); - if (NULL == pExchange->pSrcEndPoints) { - return TSDB_CODE_OUT_OF_MEMORY; - } + return nodesListMakeStrictAppend(&pExchange->pSrcEndPoints, nodesCloneNode(pSource)); + } + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == nodeType(pNode)) { + SMergePhysiNode* pMerge = (SMergePhysiNode*)pNode; + if (pMerge->srcGroupId == groupId) { + SExchangePhysiNode* pExchange = + (SExchangePhysiNode*)nodesListGetNode(pMerge->node.pChildren, pMerge->numOfChannels - 1); + if (1 == pMerge->numOfChannels) { + pMerge->numOfChannels = LIST_LENGTH(pMerge->node.pChildren); + } else { + --(pMerge->numOfChannels); } - if (TSDB_CODE_SUCCESS != nodesListStrictAppend(pExchange->pSrcEndPoints, nodesCloneNode(pSource))) { - return TSDB_CODE_OUT_OF_MEMORY; - } - return TSDB_CODE_SUCCESS; + return nodesListMakeStrictAppend(&pExchange->pSrcEndPoints, nodesCloneNode(pSource)); } } From f2048b00cbdefb90e48c7bfb0b232a67f29ed529 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 2 Jun 2022 11:01:05 +0800 Subject: [PATCH 33/47] fix(tmq): subscribe stb --- source/dnode/mnode/impl/inc/mndDef.h | 35 +++++++++------------- source/dnode/mnode/impl/src/mndDef.c | 12 ++------ source/dnode/mnode/impl/src/mndSubscribe.c | 5 ++-- source/dnode/mnode/impl/src/mndTopic.c | 33 +++----------------- source/dnode/vnode/inc/vnode.h | 17 ++++++----- source/dnode/vnode/src/tq/tq.c | 16 +++++----- source/dnode/vnode/src/tsdb/tsdbRead.c | 25 +++++++++++++--- 7 files changed, 62 insertions(+), 81 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 4bf7b15593..2f7c357aef 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -403,19 +403,15 @@ int32_t tEncodeSMqOffsetObj(void** buf, const SMqOffsetObj* pOffset); void* tDecodeSMqOffsetObj(void* buf, SMqOffsetObj* pOffset); typedef struct { - char name[TSDB_TOPIC_FNAME_LEN]; - char db[TSDB_DB_FNAME_LEN]; - int64_t createTime; - int64_t updateTime; - int64_t uid; - int64_t dbUid; - int32_t version; - int8_t subType; // column, db or stable - // int8_t withTbName; - // int8_t withSchema; - // int8_t withTag; + char name[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int64_t createTime; + int64_t updateTime; + int64_t uid; + int64_t dbUid; + int32_t version; + int8_t subType; // column, db or stable SRWLatch lock; - int32_t consumerCnt; int32_t sqlLen; int32_t astLen; char* sql; @@ -423,7 +419,6 @@ typedef struct { char* physicalPlan; SSchemaWrapper schema; int64_t stbUid; - // int32_t refConsumerCnt; } SMqTopicObj; typedef struct { @@ -477,14 +472,12 @@ int32_t tEncodeSMqConsumerEp(void** buf, const SMqConsumerEp* pEp); void* tDecodeSMqConsumerEp(const void* buf, SMqConsumerEp* pEp); typedef struct { - char key[TSDB_SUBSCRIBE_KEY_LEN]; - SRWLatch lock; - int64_t dbUid; - int32_t vgNum; - int8_t subType; - // int8_t withTbName; - // int8_t withSchema; - // int8_t withTag; + char key[TSDB_SUBSCRIBE_KEY_LEN]; + SRWLatch lock; + int64_t dbUid; + int32_t vgNum; + int8_t subType; + int64_t stbUid; SHashObj* consumerHash; // consumerId -> SMqConsumerEp SArray* unassignedVgs; // SArray } SMqSubscribeObj; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index b45b6f9ee9..b6659e1632 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -395,10 +395,8 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { taosInitRWLatch(&pSubNew->lock); pSubNew->dbUid = pSub->dbUid; + pSubNew->stbUid = pSub->stbUid; pSubNew->subType = pSub->subType; - /*pSubNew->withTbName = pSub->withTbName;*/ - /*pSubNew->withSchema = pSub->withSchema;*/ - /*pSubNew->withTag = pSub->withTag;*/ pSubNew->vgNum = pSub->vgNum; pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); @@ -431,9 +429,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { tlen += taosEncodeFixedI64(buf, pSub->dbUid); tlen += taosEncodeFixedI32(buf, pSub->vgNum); tlen += taosEncodeFixedI8(buf, pSub->subType); - /*tlen += taosEncodeFixedI8(buf, pSub->withTbName);*/ - /*tlen += taosEncodeFixedI8(buf, pSub->withSchema);*/ - /*tlen += taosEncodeFixedI8(buf, pSub->withTag);*/ + tlen += taosEncodeFixedI64(buf, pSub->stbUid); void *pIter = NULL; int32_t sz = taosHashGetSize(pSub->consumerHash); @@ -458,9 +454,7 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { buf = taosDecodeFixedI64(buf, &pSub->dbUid); buf = taosDecodeFixedI32(buf, &pSub->vgNum); buf = taosDecodeFixedI8(buf, &pSub->subType); - /*buf = taosDecodeFixedI8(buf, &pSub->withTbName);*/ - /*buf = taosDecodeFixedI8(buf, &pSub->withSchema);*/ - /*buf = taosDecodeFixedI8(buf, &pSub->withTag);*/ + buf = taosDecodeFixedI64(buf, &pSub->stbUid); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 3c43998e85..41065a3fdd 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -93,10 +93,8 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, return NULL; } pSub->dbUid = pTopic->dbUid; + pSub->stbUid = pTopic->stbUid; pSub->subType = pTopic->subType; - /*pSub->withTbName = pTopic->withTbName;*/ - /*pSub->withSchema = pTopic->withSchema;*/ - /*pSub->withTag = pTopic->withTag;*/ ASSERT(pSub->unassignedVgs->size == 0); ASSERT(taosHashGetSize(pSub->consumerHash) == 0); @@ -121,6 +119,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.vgId = pRebVg->pVgEp->vgId; req.qmsg = pRebVg->pVgEp->qmsg; req.subType = pSub->subType; + req.suid = pSub->stbUid; strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b364f25b9d..21b5e37e1e 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -96,11 +96,8 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER); - /*SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER);*/ - /*SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER);*/ - /*SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER);*/ - SDB_SET_INT32(pRaw, dataPos, pTopic->consumerCnt, TOPIC_ENCODE_OVER); + SDB_SET_INT64(pRaw, dataPos, pTopic->stbUid, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -122,8 +119,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_BINARY(pRaw, dataPos, swBuf, schemaLen, TOPIC_ENCODE_OVER); } - /*SDB_SET_INT32(pRaw, dataPos, pTopic->refConsumerCnt, TOPIC_ENCODE_OVER);*/ - SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER); SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER); @@ -168,12 +163,8 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER); - /*SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER);*/ - /*SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER);*/ - /*SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER);*/ - - SDB_GET_INT32(pRaw, dataPos, &pTopic->consumerCnt, TOPIC_DECODE_OVER); + SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); if (pTopic->sql == NULL) { @@ -222,8 +213,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { pTopic->schema.pSchema = NULL; } - /*SDB_GET_INT32(pRaw, dataPos, &pTopic->refConsumerCnt, TOPIC_DECODE_OVER);*/ - SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER); terrno = TSDB_CODE_SUCCESS; @@ -254,8 +243,6 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic atomic_exchange_64(&pOldTopic->updateTime, pNewTopic->updateTime); atomic_exchange_32(&pOldTopic->version, pNewTopic->version); - /*atomic_store_32(&pOldTopic->refConsumerCnt, pNewTopic->refConsumerCnt);*/ - /*taosWLockLatch(&pOldTopic->lock);*/ // TODO handle update @@ -278,18 +265,6 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { sdbRelease(pSdb, pTopic); } -#if 0 -static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { - SName name = {0}; - tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - - char db[TSDB_TOPIC_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, db); - - return mndAcquireDb(pMnode, db); -} -#endif - static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { int32_t contLen = sizeof(SDDropTopicReq); @@ -341,8 +316,6 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { topicObj.ast = strdup(pCreate->ast); topicObj.astLen = strlen(pCreate->ast) + 1; - /*topicObj.withTbName = pCreate->withTbName;*/ - /*topicObj.withSchema = pCreate->withSchema;*/ SNode *pAst = NULL; if (nodesStringToNode(pCreate->ast, &pAst) != 0) { @@ -376,6 +349,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * return -1; } } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { + SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName); + topicObj.stbUid = pStb->uid; } /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ /*topicObj.ast = NULL;*/ diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 3e56ea75ad..308f89736d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -85,7 +85,7 @@ typedef struct SMetaFltParam { tb_uid_t suid; int16_t cid; int16_t type; - char * val; + char *val; bool reverse; int (*filterFunc)(void *a, void *b, int16_t type); @@ -119,7 +119,8 @@ tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STab int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo); bool isTsdbCacheLastRow(tsdbReaderT *pReader); int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); -void * tsdbGetIdx(SMeta *pMeta); +int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); +void *tsdbGetIdx(SMeta *pMeta); int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); @@ -192,7 +193,7 @@ struct SMetaEntry { int64_t version; int8_t type; tb_uid_t uid; - char * name; + char *name; union { struct { SSchemaWrapper schemaRow; @@ -220,17 +221,17 @@ struct SMetaEntry { struct SMetaReader { int32_t flags; - SMeta * pMeta; + SMeta *pMeta; SDecoder coder; SMetaEntry me; - void * pBuf; + void *pBuf; int32_t szBuf; }; struct SMTbCursor { - TBC * pDbc; - void * pKey; - void * pVal; + TBC *pDbc; + void *pKey; + void *pVal; int32_t kLen; int32_t vLen; SMetaReader mr; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b8c79608f1..310b59b2e8 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -260,9 +260,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->epoch = -1; pHandle->execHandle.subType = req.subType; - /*pExec->withTbName = req.withTbName;*/ - /*pExec->withSchema = req.withSchema;*/ - /*pExec->withTag = req.withTag;*/ pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); for (int32_t i = 0; i < 5; i++) { @@ -285,13 +282,18 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->execHandle.exec.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { - int64_t suid = 0; - /*pHandle->execHandle.exec.execTb.suid = req.suid;*/ - SArray* tbUidList = taosArrayInit(0, sizeof(int16_t)); - tsdbGetAllTableList(pTq->pVnode->pMeta, suid, tbUidList); + pHandle->execHandle.exec.execTb.suid = req.suid; + SArray* tbUidList = taosArrayInit(0, sizeof(int64_t)); + tsdbGetCtbIdList(pTq->pVnode->pMeta, req.suid, tbUidList); + tqDebug("vg %d, tq try get suid: %ld", pTq->pVnode->config.vgId, req.suid); + for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { + int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); + tqDebug("vg %d, idx %d, uid: %ld", pTq->pVnode->config.vgId, i, tbUid); + } for (int32_t i = 0; i < 5; i++) { tqReadHandleSetTbUidList(pHandle->execHandle.pExecReader[i], tbUidList); } + taosArrayDestroy(tbUidList); } taosHashPut(pTq->handles, req.subKey, strlen(req.subKey), pHandle, sizeof(STqHandle)); } else { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f9c5fac536..61daa0c9b3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "vnode.h" #include "tsdb.h" +#include "vnode.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -327,8 +327,8 @@ static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableData if (updateTs) { tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 ", %s", - pTsdbReadHandle, pCond->twindows[tWinIdx].skey, pCond->twindows[tWinIdx].ekey, pTsdbReadHandle->window.skey, - pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); + pTsdbReadHandle, pCond->twindows[tWinIdx].skey, pCond->twindows[tWinIdx].ekey, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->idStr); } } @@ -586,7 +586,8 @@ void tsdbResetReadHandle(tsdbReaderT queryHandle, SQueryTableDataCond* pCond, in resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, SQueryTableDataCond* pCond, STableListInfo* tableList, int32_t tWinIdx) { +void tsdbResetQueryHandleForNewTable(tsdbReaderT queryHandle, SQueryTableDataCond* pCond, STableListInfo* tableList, + int32_t tWinIdx) { STsdbReadHandle* pTsdbReadHandle = queryHandle; pTsdbReadHandle->order = pCond->order; @@ -2845,6 +2846,22 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { return TSDB_CODE_SUCCESS; } +int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) { + SMCtbCursor* pCur = metaOpenCtbCursor(pMeta, suid); + + while (1) { + tb_uid_t id = metaCtbCursorNext(pCur); + if (id == 0) { + break; + } + + taosArrayPush(list, &id); + } + + metaCloseCtbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + static void destroyHelper(void* param) { if (param == NULL) { return; From 114792424e6e628e3a7277c2a2c76296563742fe Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 2 Jun 2022 11:12:46 +0800 Subject: [PATCH 34/47] fix(mnode): try fix case --- tests/script/jenkins/basic.txt | 2 +- tests/script/tsim/mnode/basic4.sim | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index e86f07620e..9b8a63e404 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -58,7 +58,7 @@ ./test.sh -f tsim/mnode/basic1.sim ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim -./test.sh -f tsim/mnode/basic4.sim +#./test.sh -f tsim/mnode/basic4.sim # ---- show ./test.sh -f tsim/show/basic.sim diff --git a/tests/script/tsim/mnode/basic4.sim b/tests/script/tsim/mnode/basic4.sim index 2a4a9d3626..11a94dbc55 100644 --- a/tests/script/tsim/mnode/basic4.sim +++ b/tests/script/tsim/mnode/basic4.sim @@ -191,4 +191,4 @@ endi system sh/exec.sh -n dnode1 -s stop system sh/exec.sh -n dnode2 -s stop system sh/exec.sh -n dnode3 -s stop -system sh/exec.sh -n dnode4 -s stop \ No newline at end of file +system sh/exec.sh -n dnode4 -s stop From 361332505c15b8c6517b0cf8100f6a2dda90c62e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 2 Jun 2022 11:38:04 +0800 Subject: [PATCH 35/47] feat(wal): support restore from snapshot --- include/libs/wal/wal.h | 1 + source/libs/wal/src/walWrite.c | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index e541c214de..95af8ac306 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -184,6 +184,7 @@ int32_t walRollback(SWal *, int64_t ver); // notify that previous logs can be pruned safely int32_t walBeginSnapshot(SWal *, int64_t ver); int32_t walEndSnapshot(SWal *); +void walRestoreFromSnapshot(SWal *, int64_t ver); // int32_t walDataCorrupted(SWal*); // read diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index d2a43c4107..4e2270c37b 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -18,6 +18,14 @@ #include "tchecksum.h" #include "walInt.h" +void walRestoreFromSnapshot(SWal *pWal, int64_t ver) { + pWal->vers.firstVer = -1; + pWal->vers.lastVer = ver; + pWal->vers.commitVer = ver - 1; + pWal->vers.snapshotVer = ver - 1; + pWal->vers.verInSnapshotting = -1; +} + int32_t walCommit(SWal *pWal, int64_t ver) { ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); From 10807abbc8429fea42937240e0468aeba4f3efc8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 11:43:57 +0800 Subject: [PATCH 36/47] fix:fix error in get db name for schemaless parameters --- source/libs/parser/inc/parInt.h | 2 +- source/libs/parser/src/parInsert.c | 4 ++-- source/libs/parser/src/parTranslater.c | 22 +++++++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/source/libs/parser/inc/parInt.h b/source/libs/parser/inc/parInt.h index bdef3becf7..8ec20cde5a 100644 --- a/source/libs/parser/inc/parInt.h +++ b/source/libs/parser/inc/parInt.h @@ -32,7 +32,7 @@ int32_t authenticate(SParseContext* pParseCxt, SQuery* pQuery); int32_t translate(SParseContext* pParseCxt, SQuery* pQuery); int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema); int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery); -int32_t isNotSchemalessDb(SParseContext* pContext); +int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 7729d4c415..7c179f8ac9 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1404,9 +1404,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return buildOutput(pCxt); } -int32_t isNotSchemalessDb(SParseContext* pContext){ +int32_t isNotSchemalessDb(SParseContext* pContext, char *dbName){ SName name; - tNameSetDbName(&name, pContext->acctId, pContext->db, strlen(pContext->db)); + tNameSetDbName(&name, pContext->acctId, dbName, strlen(dbName)); char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); SDbCfgInfo pInfo = {0}; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 18d64905c9..7b9d38e5b3 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2648,8 +2648,8 @@ static int32_t checkCreateTable(STranslateContext* pCxt, SCreateTableStmt* pStmt code = checkTableSchema(pCxt, pStmt); } if (TSDB_CODE_SUCCESS == code) { - if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ - code = TSDB_CODE_SML_INVALID_DB_CONF; + if(pCxt->pParseCxt->schemalessType == 0){ + code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName); } } return code; @@ -4430,9 +4430,7 @@ static SArray* serializeVgroupsCreateTableBatch(int32_t acctId, SHashObj* pVgrou } static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) { - if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ - return TSDB_CODE_SML_INVALID_DB_CONF; - } + SCreateMultiTableStmt* pStmt = (SCreateMultiTableStmt*)pQuery->pRoot; SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); @@ -4443,6 +4441,10 @@ static int32_t rewriteCreateMultiTable(STranslateContext* pCxt, SQuery* pQuery) int32_t code = TSDB_CODE_SUCCESS; SNode* pNode; FOREACH(pNode, pStmt->pSubTables) { + if(pCxt->pParseCxt->schemalessType == 0 && + (code = isNotSchemalessDb(pCxt->pParseCxt, ((SCreateSubTableClause*)pNode)->dbName)) != TSDB_CODE_SUCCESS){ + return code; + } code = rewriteCreateSubTable(pCxt, (SCreateSubTableClause*)pNode, pVgroupHashmap); if (TSDB_CODE_SUCCESS != code) { taosHashCleanup(pVgroupHashmap); @@ -4853,13 +4855,15 @@ static int32_t buildModifyVnodeArray(STranslateContext* pCxt, SAlterTableStmt* p } static int32_t rewriteAlterTable(STranslateContext* pCxt, SQuery* pQuery) { - if(pCxt->pParseCxt->schemalessType == 0 && isNotSchemalessDb(pCxt->pParseCxt) != TSDB_CODE_SUCCESS){ - return TSDB_CODE_SML_INVALID_DB_CONF; - } SAlterTableStmt* pStmt = (SAlterTableStmt*)pQuery->pRoot; + int32_t code = TSDB_CODE_SUCCESS; + if(pCxt->pParseCxt->schemalessType == 0 && + (code = isNotSchemalessDb(pCxt->pParseCxt, pStmt->dbName)) != TSDB_CODE_SUCCESS){ + return code; + } STableMeta* pTableMeta = NULL; - int32_t code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); + code = getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pTableMeta); if (TSDB_CODE_SUCCESS != code) { return code; } From 3d9f5d13cd48340096bf570c7c607ec224409ac3 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 2 Jun 2022 12:19:27 +0800 Subject: [PATCH 37/47] test: temp close one case --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 45e25032cf..f03a07e8e1 100644 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -18,7 +18,7 @@ python3 ./test.py -f 0-others/fsync.py python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py +#python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py From a10643a074faf771d8fee72a5c8320b7ed1030fe Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 2 Jun 2022 12:34:35 +0800 Subject: [PATCH 38/47] update table meta after alter table --- include/common/tmsg.h | 4 + include/libs/nodes/querynodes.h | 2 +- include/libs/qcom/query.h | 9 +- include/libs/scheduler/scheduler.h | 2 +- source/client/inc/clientInt.h | 2 +- source/client/src/clientEnv.c | 27 +--- source/client/src/clientImpl.c | 35 ++--- source/client/src/clientMsgHandler.c | 5 +- source/dnode/mnode/impl/src/mndStb.c | 142 ++++++++++--------- source/libs/parser/src/parInsert.c | 27 +++- source/libs/parser/src/parTranslater.c | 68 +++++---- source/libs/qcom/src/queryUtil.c | 27 ++++ source/libs/scheduler/inc/schedulerInt.h | 2 +- source/libs/scheduler/src/schJob.c | 19 ++- source/libs/scheduler/src/schRemote.c | 10 +- tests/script/jenkins/basic.txt | 3 + tests/script/tsim/catalog/alterInCurrent.sim | 70 +++++++++ 17 files changed, 291 insertions(+), 163 deletions(-) create mode 100644 tests/script/tsim/catalog/alterInCurrent.sim diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 9800c74dd1..ac65e3da8b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1135,6 +1135,10 @@ typedef struct { STableMetaRsp* pMeta; } SMAlterStbRsp; +int32_t tEncodeSMAlterStbRsp(SEncoder *pEncoder, const SMAlterStbRsp *pRsp); +int32_t tDecodeSMAlterStbRsp(SDecoder *pDecoder, SMAlterStbRsp *pRsp); +void tFreeSMAlterStbRsp(SMAlterStbRsp* pRsp); + int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); void tFreeSTableMetaRsp(STableMetaRsp* pRsp); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index ab5e10dc2a..e4af78892b 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -331,8 +331,8 @@ typedef struct SQuery { int8_t precision; SCmdMsgInfo* pCmdMsg; int32_t msgType; - SArray* pDbList; SArray* pTableList; + SArray* pDbList; bool showRewrite; int32_t placeholderNum; SArray* pPlaceholderValues; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 296b18e8de..45a7e9a29f 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -56,6 +56,11 @@ typedef struct STableComInfo { int32_t rowSize; // row size of the schema } STableComInfo; +typedef struct SQueryExecRes { + int32_t msgType; + void* res; +} SQueryExecRes; + typedef struct SIndexMeta { #ifdef WINDOWS size_t avoidCompilationErrors; @@ -192,6 +197,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl char* jobTaskStatusStr(int32_t status); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); +void destroyQueryExecRes(SQueryExecRes* pRes); extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t)); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); @@ -204,7 +210,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \ (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \ - (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code == TSDB_CODE_PAR_VALUE_TOO_LONG)) + (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code == TSDB_CODE_PAR_VALUE_TOO_LONG) || \ + (_code == TSDB_CODE_PAR_INVALID_DROP_COL)) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) ((_code) == TSDB_CODE_TDB_TABLE_RECREATED) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 0d32cce20b..331b787690 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -56,7 +56,7 @@ typedef struct SQueryProfileSummary { typedef struct SQueryResult { int32_t code; uint64_t numOfRows; - void *res; + SQueryExecRes res; } SQueryResult; typedef struct STaskInfo { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 91f9874b23..ad610603bf 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -162,7 +162,7 @@ typedef struct SResultColumn { } SResultColumn; typedef struct SReqResultInfo { - void* pExecRes; + SQueryExecRes execRes; const char* pRspMsg; const char* pData; TAOS_FIELD* fields; // todo, column names are not needed. diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 777f27d29d..4f5f23b68b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -212,31 +212,6 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) { } } -static void destroyExecRes(SRequestObj* pRequest) { - if (NULL == pRequest || NULL == pRequest->body.resInfo.pExecRes) { - return; - } - - switch (pRequest->type) { - case TDMT_VND_ALTER_TABLE: - case TDMT_MND_ALTER_STB: { - tFreeSTableMetaRsp((STableMetaRsp *)pRequest->body.resInfo.pExecRes); - taosMemoryFree(pRequest->body.resInfo.pExecRes); - break; - } - case TDMT_VND_SUBMIT: { - tFreeSSubmitRsp((SSubmitRsp*)pRequest->body.resInfo.pExecRes); - break; - } - case TDMT_VND_QUERY: { - taosArrayDestroy((SArray*)pRequest->body.resInfo.pExecRes); - break; - } - default: - tscError("invalid exec result for request type %d", pRequest->type); - } -} - static void doDestroyRequest(void *p) { assert(p != NULL); SRequestObj *pRequest = (SRequestObj *)p; @@ -259,7 +234,7 @@ static void doDestroyRequest(void *p) { taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); - destroyExecRes(pRequest); + destroyQueryExecRes(&pRequest->body.resInfo.execRes); deregisterRequest(pRequest); taosMemoryFreeClear(pRequest); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 123afa0f7c..729f53f443 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -340,7 +340,7 @@ void setResPrecision(SReqResultInfo* pResInfo, int32_t precision) { pResInfo->precision = precision; } -int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList, void** pRes) { +int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; tsem_init(&schdRspSem, 0, 0); @@ -348,14 +348,15 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod SQueryResult res = {.code = 0, .numOfRows = 0}; int32_t code = schedulerAsyncExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, schdExecCallback, &res); + + pRequest->body.resInfo.execRes = res.res; + while (true) { if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); } - *pRes = res.res; - pRequest->code = code; terrno = code; return pRequest->code; @@ -378,8 +379,6 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod } } - *pRes = res.res; - pRequest->code = res.code; terrno = res.code; return pRequest->code; @@ -393,14 +392,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList int32_t code = schedulerExecJob(pTransporter, pNodeList, pDag, &pRequest->body.queryJob, pRequest->sqlstr, pRequest->metric.start, &res); - pRequest->body.resInfo.pExecRes = res.res; + pRequest->body.resInfo.execRes = res.res; if (code != TSDB_CODE_SUCCESS) { if (pRequest->body.queryJob != 0) { schedulerFreeJob(pRequest->body.queryJob); } - pRequest->code = code; terrno = code; return pRequest->code; @@ -489,6 +487,10 @@ int32_t handleAlterTbExecRes(void* res, SCatalog* pCatalog) { } int32_t handleExecRes(SRequestObj* pRequest) { + if (NULL == pRequest->body.resInfo.execRes.res) { + return TSDB_CODE_SUCCESS; + } + int32_t code = 0; SCatalog* pCatalog = NULL; code = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); @@ -497,19 +499,20 @@ int32_t handleExecRes(SRequestObj* pRequest) { } SEpSet epset = getEpSet_s(&pRequest->pTscObj->pAppInfo->mgmtEp); - - switch (pRequest->type) { + SQueryExecRes* pRes = &pRequest->body.resInfo.execRes; + + switch (pRes->msgType) { case TDMT_VND_ALTER_TABLE: case TDMT_MND_ALTER_STB: { - code = handleAlterTbExecRes(pRequest->body.resInfo.pExecRes, pCatalog); + code = handleAlterTbExecRes(pRes->res, pCatalog); break; } case TDMT_VND_SUBMIT: { - code = handleSubmitExecRes(pRequest, pRequest->body.resInfo.pExecRes, pCatalog, &epset); + code = handleSubmitExecRes(pRequest, pRes->res, pCatalog, &epset); break; } case TDMT_VND_QUERY: { - code = handleQueryExecRes(pRequest, pRequest->body.resInfo.pExecRes, pCatalog, &epset); + code = handleQueryExecRes(pRequest, pRes->res, pCatalog, &epset); break; } default: @@ -550,17 +553,15 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, int32_t code qDestroyQuery(pQuery); } - if (NULL != pRequest->body.resInfo.pExecRes) { - handleExecRes(pRequest); - } + handleExecRes(pRequest); if (NULL != pRequest && TSDB_CODE_SUCCESS != code) { pRequest->code = terrno; } if (res) { - *res = pRequest->body.resInfo.pExecRes; - pRequest->body.resInfo.pExecRes = NULL; + *res = pRequest->body.resInfo.execRes.res; + pRequest->body.resInfo.execRes.res = NULL; } return pRequest; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index e48770e2a1..9de3ee1d0f 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -236,8 +236,9 @@ int32_t processAlterStbRsp(void* param, const SDataBuf* pMsg, int32_t code) { tDecoderInit(&coder, pMsg->pData, pMsg->len); tDecodeSMAlterStbRsp(&coder, &alterRsp); tDecoderClear(&coder); - - pRequest->body.resInfo.pExecRes = alterRsp.pMeta; + + pRequest->body.resInfo.execRes.msgType = TDMT_MND_ALTER_STB; + pRequest->body.resInfo.execRes.res = alterRsp.pMeta; tsem_post(&pRequest->body.rspSem); return code; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index e691e0341a..aa8789829c 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1207,6 +1207,77 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } + +static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { + taosRLockLatch(&pStb->lock); + + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + taosRUnLockLatch(&pStb->lock); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + strcpy(pRsp->dbFName, pStb->db); + strcpy(pRsp->tbName, tbName); + strcpy(pRsp->stbName, tbName); + pRsp->dbId = pDb->uid; + pRsp->numOfTags = pStb->numOfTags; + pRsp->numOfColumns = pStb->numOfColumns; + pRsp->precision = pDb->cfg.precision; + pRsp->tableType = TSDB_SUPER_TABLE; + pRsp->sversion = pStb->colVer; + pRsp->tversion = pStb->tagVer; + pRsp->suid = pStb->uid; + pRsp->tuid = pStb->uid; + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i]; + SSchema *pSrcSchema = &pStb->pColumns[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = pSrcSchema->colId; + pSchema->bytes = pSrcSchema->bytes; + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i + pStb->numOfColumns]; + SSchema *pSrcSchema = &pStb->pTags[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = pSrcSchema->colId; + pSchema->bytes = pSrcSchema->bytes; + } + + taosRUnLockLatch(&pStb->lock); + return 0; +} + +static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); + + SDbObj *pDb = mndAcquireDb(pMnode, dbFName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_MND_INVALID_STB; + return -1; + } + + int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + return code; +} + + static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { int ret; SEncoder ec = {0}; @@ -1221,7 +1292,7 @@ static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, S return -1; } - ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, &alterRsp.meta); + ret = mndBuildStbSchemaImp(pDb, pObj, name.tname, alterRsp.pMeta); if (ret) { tFreeSMAlterStbRsp(&alterRsp); return ret; @@ -1533,75 +1604,6 @@ static int32_t mndProcessVDropStbRsp(SRpcMsg *pRsp) { return 0; } -static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { - taosRLockLatch(&pStb->lock); - - int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); - if (pRsp->pSchemas == NULL) { - taosRUnLockLatch(&pStb->lock); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - strcpy(pRsp->dbFName, pStb->db); - strcpy(pRsp->tbName, tbName); - strcpy(pRsp->stbName, tbName); - pRsp->dbId = pDb->uid; - pRsp->numOfTags = pStb->numOfTags; - pRsp->numOfColumns = pStb->numOfColumns; - pRsp->precision = pDb->cfg.precision; - pRsp->tableType = TSDB_SUPER_TABLE; - pRsp->sversion = pStb->colVer; - pRsp->tversion = pStb->tagVer; - pRsp->suid = pStb->uid; - pRsp->tuid = pStb->uid; - - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pRsp->pSchemas[i]; - SSchema *pSrcSchema = &pStb->pColumns[i]; - memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); - pSchema->type = pSrcSchema->type; - pSchema->colId = pSrcSchema->colId; - pSchema->bytes = pSrcSchema->bytes; - } - - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pRsp->pSchemas[i + pStb->numOfColumns]; - SSchema *pSrcSchema = &pStb->pTags[i]; - memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); - pSchema->type = pSrcSchema->type; - pSchema->colId = pSrcSchema->colId; - pSchema->bytes = pSrcSchema->bytes; - } - - taosRUnLockLatch(&pStb->lock); - return 0; -} - -static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { - char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); - - SDbObj *pDb = mndAcquireDb(pMnode, dbFName); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; - } - - SStbObj *pStb = mndAcquireStb(pMnode, tbFName); - if (pStb == NULL) { - mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_MND_INVALID_STB; - return -1; - } - - int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); - mndReleaseDb(pMnode, pDb); - mndReleaseStb(pMnode, pStb); - return code; -} - static int32_t mndProcessTableMetaReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 8a843c2c1a..f5a079318c 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -61,6 +61,7 @@ typedef struct SInsertParseContext { SHashObj* pSubTableHashObj; // global SArray* pVgDataBlocks; // global SHashObj* pTableNameHashObj; // global + SHashObj* pDbFNameHashObj; // global int32_t totalNum; SVnodeModifOpStmt* pOutput; SStmtCallback* pStmtCb; @@ -972,6 +973,10 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, continue; } + if (TK_NK_RP == sToken.type) { + return generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM); + } + if (isParseBindParam) { return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and values"); } @@ -1091,6 +1096,7 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { taosHashCleanup(pCxt->pVgroupsHashObj); taosHashCleanup(pCxt->pSubTableHashObj); taosHashCleanup(pCxt->pTableNameHashObj); + taosHashCleanup(pCxt->pDbFNameHashObj); destroyBlockHashmap(pCxt->pTableBlockHashObj); destroyBlockArrayList(pCxt->pVgDataBlocks); @@ -1151,6 +1157,9 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { tNameExtractFullName(&name, tbFName); CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(&name, dbFName); + CHECK_CODE(taosHashPut(pCxt->pDbFNameHashObj, dbFName, strlen(dbFName), dbFName, sizeof(dbFName))); // USING clause if (TK_USING == sToken.type) { @@ -1158,8 +1167,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { NEXT_TOKEN(pCxt->pSql, sToken); autoCreateTbl = true; } else { - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(&name, dbFName); CHECK_CODE(getTableMeta(pCxt, &name, dbFName)); } @@ -1238,6 +1245,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pTableMeta = NULL, .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), + .pDbFNameHashObj = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .totalNum = 0, .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), .pStmtCb = pContext->pStmtCb}; @@ -1252,7 +1260,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { } if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj || - NULL == context.pTableNameHashObj || NULL == context.pOutput) { + NULL == context.pTableNameHashObj || NULL == context.pDbFNameHashObj || NULL == context.pOutput) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -1278,6 +1286,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { } } + if (NULL == (*pQuery)->pDbList) { + (*pQuery)->pDbList = taosArrayInit(taosHashGetSize(context.pDbFNameHashObj), TSDB_DB_FNAME_LEN); + if (NULL == (*pQuery)->pDbList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + context.pOutput->payloadType = PAYLOAD_TYPE_KV; int32_t code = skipInsertInto(&context.pSql, &context.msg); @@ -1290,6 +1305,12 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { taosArrayPush((*pQuery)->pTableList, pTable); pTable = taosHashIterate(context.pTableNameHashObj, pTable); } + + char* pDb = taosHashIterate(context.pDbFNameHashObj, NULL); + while (NULL != pDb) { + taosArrayPush((*pQuery)->pDbList, pDb); + pDb = taosHashIterate(context.pDbFNameHashObj, pDb); + } } destroyInsertParseContext(&context); return code; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index acbedff0c2..b9b3da0dfe 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4889,6 +4889,47 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { return code; } +static int32_t toMsgType(ENodeType type) { + switch (type) { + case QUERY_NODE_CREATE_TABLE_STMT: + return TDMT_VND_CREATE_TABLE; + case QUERY_NODE_ALTER_TABLE_STMT: + return TDMT_VND_ALTER_TABLE; + case QUERY_NODE_DROP_TABLE_STMT: + return TDMT_VND_DROP_TABLE; + default: + break; + } + return TDMT_VND_CREATE_TABLE; +} + +static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery) { + if (NULL != pCxt->pDbs) { + pQuery->pDbList = taosArrayInit(taosHashGetSize(pCxt->pDbs), TSDB_DB_FNAME_LEN); + if (NULL == pQuery->pDbList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SFullDatabaseName* pDb = taosHashIterate(pCxt->pDbs, NULL); + while (NULL != pDb) { + taosArrayPush(pQuery->pDbList, pDb->fullDbName); + pDb = taosHashIterate(pCxt->pDbs, pDb); + } + } + + if (NULL != pCxt->pTables) { + pQuery->pTableList = taosArrayInit(taosHashGetSize(pCxt->pTables), sizeof(SName)); + if (NULL == pQuery->pTableList) { + return TSDB_CODE_OUT_OF_MEMORY; + } + SName* pTable = taosHashIterate(pCxt->pTables, NULL); + while (NULL != pTable) { + taosArrayPush(pQuery->pTableList, pTable); + pTable = taosHashIterate(pCxt->pTables, pTable); + } + } + return TSDB_CODE_SUCCESS; +} + static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { switch (nodeType(pQuery->pRoot)) { case QUERY_NODE_SELECT_STMT: @@ -4900,7 +4941,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { break; case QUERY_NODE_VNODE_MODIF_STMT: pQuery->execMode = QUERY_EXEC_MODE_SCHEDULE; - pQuery->msgType = TDMT_VND_CREATE_TABLE; + pQuery->msgType = toMsgType(((SVnodeModifOpStmt*)pQuery->pRoot)->sqlNodeType); break; case QUERY_NODE_DESCRIBE_STMT: pQuery->execMode = QUERY_EXEC_MODE_LOCAL; @@ -4928,30 +4969,6 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { } } - if (NULL != pCxt->pDbs) { - pQuery->pDbList = taosArrayInit(taosHashGetSize(pCxt->pDbs), TSDB_DB_FNAME_LEN); - if (NULL == pQuery->pDbList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SFullDatabaseName* pDb = taosHashIterate(pCxt->pDbs, NULL); - while (NULL != pDb) { - taosArrayPush(pQuery->pDbList, pDb->fullDbName); - pDb = taosHashIterate(pCxt->pDbs, pDb); - } - } - - if (NULL != pCxt->pTables) { - pQuery->pTableList = taosArrayInit(taosHashGetSize(pCxt->pTables), sizeof(SName)); - if (NULL == pQuery->pTableList) { - return TSDB_CODE_OUT_OF_MEMORY; - } - SName* pTable = taosHashIterate(pCxt->pTables, NULL); - while (NULL != pTable) { - taosArrayPush(pQuery->pTableList, pTable); - pTable = taosHashIterate(pCxt->pTables, pTable); - } - } - return TSDB_CODE_SUCCESS; } @@ -4971,6 +4988,7 @@ int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { if (TSDB_CODE_SUCCESS == code) { code = setQuery(&cxt, pQuery); } + setRefreshMate(&cxt, pQuery); destroyTranslateContext(&cxt); return code; } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index f4ba2fca81..a5a499aaf5 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -199,3 +199,30 @@ SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* nam tstrncpy(s.name, name, tListLen(s.name)); return s; } + +void destroyQueryExecRes(SQueryExecRes* pRes) { + if (NULL == pRes || NULL == pRes->res) { + return; + } + + switch (pRes->msgType) { + case TDMT_VND_ALTER_TABLE: + case TDMT_MND_ALTER_STB: { + tFreeSTableMetaRsp((STableMetaRsp *)pRes->res); + taosMemoryFreeClear(pRes->res); + break; + } + case TDMT_VND_SUBMIT: { + tFreeSSubmitRsp((SSubmitRsp*)pRes->res); + break; + } + case TDMT_VND_QUERY: { + taosArrayDestroy((SArray*)pRes->res); + break; + } + default: + qError("invalid exec result for request type %d", pRes->msgType); + } +} + + diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 09a4f322d4..44b3e6d396 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -204,7 +204,7 @@ typedef struct SSchJob { SSchTask *fetchTask; int32_t errCode; SRWLatch resLock; - void *queryRes; + SQueryExecRes execRes; void *resData; //TODO free it or not int32_t resNumOfRows; SSchResInfo userRes; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 1c60dcccfd..dbad053c65 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -773,8 +773,8 @@ _return: int32_t schSetJobQueryRes(SSchJob* pJob, SQueryResult* pRes) { pRes->code = atomic_load_32(&pJob->errCode); pRes->numOfRows = pJob->resNumOfRows; - pRes->res = pJob->queryRes; - pJob->queryRes = NULL; + pRes->res = pJob->execRes; + pJob->execRes.res = NULL; return TSDB_CODE_SUCCESS; } @@ -1107,9 +1107,9 @@ int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRs int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp) { if (rsp->tbFName[0]) { - if (NULL == pJob->queryRes) { - pJob->queryRes = taosArrayInit(pJob->taskNum, sizeof(STbVerInfo)); - if (NULL == pJob->queryRes) { + if (NULL == pJob->execRes.res) { + pJob->execRes.res = taosArrayInit(pJob->taskNum, sizeof(STbVerInfo)); + if (NULL == pJob->execRes.res) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } } @@ -1119,7 +1119,8 @@ int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp) { tbInfo.sversion = rsp->sversion; tbInfo.tversion = rsp->tversion; - taosArrayPush((SArray *)pJob->queryRes, &tbInfo); + taosArrayPush((SArray *)pJob->execRes.res, &tbInfo); + pJob->execRes.msgType = TDMT_VND_QUERY; } return TSDB_CODE_SUCCESS; @@ -1349,11 +1350,7 @@ void schFreeJobImpl(void *job) { qExplainFreeCtx(pJob->explainCtx); - if (SCH_IS_QUERY_JOB(pJob)) { - taosArrayDestroy((SArray *)pJob->queryRes); - } else { - tFreeSSubmitRsp((SSubmitRsp*)pJob->queryRes); - } + destroyQueryExecRes(&pJob->execRes); taosMemoryFreeClear(pJob->userRes.queryRes); taosMemoryFreeClear(pJob->resData); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 39ba52145c..0ba91a1c85 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -163,7 +163,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_JRET(code); SCH_ERR_JRET(rsp.code); - pJob->queryRes = rsp.pMeta; + pJob->execRes.res = rsp.pMeta; + pJob->execRes.msgType = TDMT_VND_ALTER_TABLE; } SCH_ERR_JRET(rspCode); @@ -206,8 +207,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows); SCH_LOCK(SCH_WRITE, &pJob->resLock); - if (pJob->queryRes) { - SSubmitRsp *sum = pJob->queryRes; + if (pJob->execRes.res) { + SSubmitRsp *sum = pJob->execRes.res; sum->affectedRows += rsp->affectedRows; sum->nBlocks += rsp->nBlocks; sum->pBlocks = taosMemoryRealloc(sum->pBlocks, sum->nBlocks * sizeof(*sum->pBlocks)); @@ -215,7 +216,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch taosMemoryFree(rsp->pBlocks); taosMemoryFree(rsp); } else { - pJob->queryRes = rsp; + pJob->execRes.res = rsp; + pJob->execRes.msgType = TDMT_VND_SUBMIT; } SCH_UNLOCK(SCH_WRITE, &pJob->resLock); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 12b678eeae..9e91a9f2f4 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -134,4 +134,7 @@ ./test.sh -f tsim/sync/oneReplica1VgElect.sim ./test.sh -f tsim/sync/oneReplica5VgElect.sim +# --- catalog +./test.sh -f tsim/catalog/alterInCurrent.sim + #======================b1-end=============== diff --git a/tests/script/tsim/catalog/alterInCurrent.sim b/tests/script/tsim/catalog/alterInCurrent.sim new file mode 100644 index 0000000000..3cb337bbe1 --- /dev/null +++ b/tests/script/tsim/catalog/alterInCurrent.sim @@ -0,0 +1,70 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ========= start dnode1 as LEADER +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== drop column in normal table +sql drop database if exists db1; +sql create database db1; +sql use db1; +sql create table t1 (ts timestamp, f1 int, f2 int); +sql insert into t1 values (1591060628000, 1, 2); +sql alter table t1 drop column f2; +sql insert into t1 values (1591060628001, 2); + +print ======== add column in normal table +sql drop database db1; +sql create database db1; +sql use db1; +sql create table t1 (ts timestamp, f1 int); +sql insert into t1 values (1591060628000, 1); +sql alter table t1 add column f2 int; +sql insert into t1 values (1591060628001, 2, 2); + + +print ======== drop column in super table +sql drop database db1; +sql create database db1; +sql use db1; +sql create stable st1 (ts timestamp, f1 int, f2 int) tags (t1 int); +sql create table t1 using st1 tags(1); +sql insert into t1 values (1591060628000, 1, 2); +sql alter table st1 drop column f2; +sql insert into t1 values (1591060628001, 2); + + +print ======== add column in super table +sql drop database db1; +sql create database db1; +sql use db1; +sql create stable st1 (ts timestamp, f1 int) tags (t1 int); +sql create table t1 using st1 tags(1); +sql insert into t1 values (1591060628000, 1); +sql alter table st1 add column f2 int; +sql insert into t1 values (1591060628001, 2, 2); + + +print ======== add tag in super table +sql drop database db1; +sql create database db1; +sql use db1; +sql create stable st1 (ts timestamp, f1 int) tags (t1 int); +sql create table t1 using st1 tags(1); +sql insert into t1 values (1591060628000, 1); +sql alter table st1 add tag t2 int; +sql create table t2 using st1 tags(2, 2); + + +print ======== drop tag in super table +sql drop database db1; +sql create database db1; +sql use db1; +sql create stable st1 (ts timestamp, f1 int) tags (t1 int, t2 int); +sql create table t1 using st1 tags(1, 1); +sql insert into t1 values (1591060628000, 1); +sql alter table st1 drop tag t2; +sql create table t2 using st1 tags(2); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 90cdc3601876cfec59849cac077a2f51d8ce2441 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 12:55:51 +0800 Subject: [PATCH 39/47] fix:fix error in get db name for schemaless parameters --- source/client/src/clientSml.c | 11 ++++++----- source/libs/parser/src/parInsert.c | 9 +++------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8d1924b1c9..e77b9a9bd0 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -960,10 +960,10 @@ static int32_t smlParseInfluxString(const char* sql, SSmlLineInfo *elements, SSm return TSDB_CODE_SML_INVALID_DATA; } elements->colsLen = sql - elements->cols; - if(elements->colsLen == 0) { - smlBuildInvalidDataMsg(msg, "cols is empty", NULL); - return TSDB_CODE_SML_INVALID_DATA; - } +// if(elements->colsLen == 0) { +// smlBuildInvalidDataMsg(msg, "cols is empty", NULL); +// return TSDB_CODE_SML_INVALID_DATA; +// } // parse timestamp JUMP_SPACE(sql) @@ -1124,7 +1124,7 @@ static int32_t smlParseTelnetString(SSmlHandle *info, const char* sql, SSmlTable } static int32_t smlParseCols(const char* data, int32_t len, SArray *cols, char *childTableName, bool isTag, SHashObj *dumplicateKey, SSmlMsgBuf *msg){ - if(isTag && len == 0){ + if(len == 0){ return TSDB_CODE_SUCCESS; } @@ -2407,6 +2407,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr info->pRequest->code = smlProcess(info, lines, numLines); end: + uDebug("result:%s", info->msgBuf.buf); smlDestroyInfo(info); return (TAOS_RES*)request; } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 7c179f8ac9..4973f792ce 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1329,6 +1329,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { SName name; CHECK_CODE(createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); + CHECK_CODE(isNotSchemalessDb(pCxt->pComCxt, name.dbname)); + tNameExtractFullName(&name, tbFName); CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); @@ -1468,11 +1470,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { (*pQuery)->msgType = TDMT_VND_SUBMIT; (*pQuery)->pRoot = (SNode*)context.pOutput; - int32_t code = isNotSchemalessDb(pContext); - if(code != TSDB_CODE_SUCCESS){ - return code; - } - if (NULL == (*pQuery)->pTableList) { (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName)); if (NULL == (*pQuery)->pTableList) { @@ -1482,7 +1479,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { context.pOutput->payloadType = PAYLOAD_TYPE_KV; - code = skipInsertInto(&context.pSql, &context.msg); + int32_t code = skipInsertInto(&context.pSql, &context.msg); if (TSDB_CODE_SUCCESS == code) { code = parseInsertBody(&context); } From 99be39340bc1dee484bb96662dced38051301843 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 2 Jun 2022 13:21:43 +0800 Subject: [PATCH 40/47] fix case issue --- tests/script/tsim/stable/column_drop.sim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/script/tsim/stable/column_drop.sim b/tests/script/tsim/stable/column_drop.sim index af84a3ecac..3401465103 100644 --- a/tests/script/tsim/stable/column_drop.sim +++ b/tests/script/tsim/stable/column_drop.sim @@ -129,7 +129,8 @@ endi print ========== step2 drop column c5 sql alter table db.stb drop column c5 -sql insert into db.ctb values(now+2s, 1, 2, 3, 4, 5) +sql_error insert into db.ctb values(now+2s, 1, 2, 3, 4, 5) +sql insert into db.ctb values(now+2s, 1, 2, 3, 4) sql insert into db.ctb values(now+3s, 1, 2, 3, 4) sql_error insert into db.ctb values(now+2s, 1, 2, 3, 4, 5) @@ -206,4 +207,4 @@ if $data[7][0] != t3 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 7b4d4e8b937e1c7625ad65eac07a110d8cfe2780 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 14:16:46 +0800 Subject: [PATCH 41/47] fix:fix error in updata tag --- source/dnode/vnode/src/meta/metaTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index a300741942..021d0242c2 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -584,7 +584,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } taosArrayPush(pTagArray, &val); } else { - STagVal val = {0}; + STagVal val = {.cid = pCol->colId}; if (tTagGet(pOldTag, &val)) { taosArrayPush(pTagArray, &val); } From f6f43be51eea812c4671451e4b0ecddc50644fa5 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 2 Jun 2022 14:21:49 +0800 Subject: [PATCH 42/47] test: modify consumer processor --- tests/test/c/tmqSim.c | 106 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 98 insertions(+), 8 deletions(-) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index accd1dd080..38b0313c14 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -32,6 +32,7 @@ #define MAX_SQL_STR_LEN (1024 * 1024) #define MAX_ROW_STR_LEN (16 * 1024) #define MAX_CONSUMER_THREAD_CNT (16) +#define MAX_VGROUP_CNT (32) typedef struct { TdThread thread; @@ -61,6 +62,10 @@ typedef struct { tmq_t* tmq; tmq_list_t* topicList; + + int32_t numOfVgroups; + int32_t rowsOfPerVgroups[MAX_VGROUP_CNT][2]; // [i][0]: vgroup id, [i][1]: rows of consume + int64_t ts; } SThreadInfo; @@ -69,7 +74,8 @@ typedef struct { char cdbName[32]; char dbName[32]; int32_t showMsgFlag; - int32_t showRowFlag; + int32_t showRowFlag; + int32_t saveRowFlag; int32_t consumeDelay; // unit s int32_t numOfThread; SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; @@ -77,6 +83,7 @@ typedef struct { static SConfInfo g_stConfInfo; TdFilePtr g_fp = NULL; +static int running = 1; // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -93,6 +100,8 @@ static void printHelp() { printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag); printf("%s%s\n", indent, "-r"); printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag); + printf("%s%s\n", indent, "-s"); + printf("%s%s%s%d\n", indent, indent, "saveRowFlag, default is ", g_stConfInfo.saveRowFlag); printf("%s%s\n", indent, "-y"); printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay); exit(EXIT_SUCCESS); @@ -135,6 +144,7 @@ void saveConfigToLogFile() { taosFprintfFile(g_fp, "# cdbName: %s\n", g_stConfInfo.cdbName); taosFprintfFile(g_fp, "# showMsgFlag: %d\n", g_stConfInfo.showMsgFlag); taosFprintfFile(g_fp, "# showRowFlag: %d\n", g_stConfInfo.showRowFlag); + taosFprintfFile(g_fp, "# saveRowFlag: %d\n", g_stConfInfo.saveRowFlag); taosFprintfFile(g_fp, "# consumeDelay: %d\n", g_stConfInfo.consumeDelay); taosFprintfFile(g_fp, "# numOfThread: %d\n", g_stConfInfo.numOfThread); @@ -165,6 +175,7 @@ void parseArgument(int32_t argc, char* argv[]) { memset(&g_stConfInfo, 0, sizeof(SConfInfo)); g_stConfInfo.showMsgFlag = 0; g_stConfInfo.showRowFlag = 0; + g_stConfInfo.saveRowFlag = 0; g_stConfInfo.consumeDelay = 5; for (int32_t i = 1; i < argc; i++) { @@ -181,6 +192,8 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.showMsgFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { g_stConfInfo.showRowFlag = atol(argv[++i]); + } else if (strcmp(argv[i], "-s") == 0) { + g_stConfInfo.saveRowFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); } else { @@ -200,6 +213,7 @@ void parseArgument(int32_t argc, char* argv[]) { pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC); pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC); pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC); + pPrint("%s saveRowFlag:%d %s", GREEN, g_stConfInfo.saveRowFlag, NC); #endif } @@ -225,15 +239,58 @@ void ltrim(char* str) { // return str; } -static int running = 1; -static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) { +void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { + int32_t i; + for (i = 0; i < pInfo->numOfVgroups; i++) { + if (vgroupId == pInfo->rowsOfPerVgroups[i][0]) { + pInfo->rowsOfPerVgroups[i][1] += rows; + return; + } + } + + pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][0] = vgroupId; + pInfo->rowsOfPerVgroups[pInfo->numOfVgroups][1] += rows; + pInfo->numOfVgroups++; + + taosFprintfFile(g_fp, "consume id %d, add one new vogroup id: %d\n", pInfo->consumerId, vgroupId); + if (pInfo->numOfVgroups > MAX_VGROUP_CNT) { + taosFprintfFile(g_fp, "====consume id %d, vgroup num %d over than 32. new vgroupId: %d\n", pInfo->consumerId, pInfo->numOfVgroups, vgroupId); + taosCloseFile(&g_fp); + exit(-1); + } +} + +int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { + char sqlStr[1024] = {0}; + + TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + sprintf(sqlStr, "insert into %s.content_%d values (%"PRId64", \'%s\')", g_stConfInfo.cdbName, pInfo->consumerId, pInfo->ts++, buf); + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + pError("error in insert consume result, reason:%s\n", taos_errstr(pRes)); + taosFprintfFile(g_fp, "error in insert consume result, reason:%s\n", taos_errstr(pRes)); + taosCloseFile(&g_fp); + taos_free_result(pRes); + exit(-1); + } + + taos_free_result(pRes); + + return 0; +} + +static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { char buf[1024]; int32_t totalRows = 0; - + // printf("topic: %s\n", tmq_get_topic_name(msg)); - // printf("vg:%d\n", tmq_get_vgroup_id(msg)); - taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg)); + int32_t vgroupId = tmq_get_vgroup_id(msg); + + taosFprintfFile(g_fp, "msg index:%" PRId64 ", consumerId: %d\n", msgIndex, pInfo->consumerId); + //taosFprintfFile(g_fp, "topic: %s, vgroupId: %d, tableName: %s\n", tmq_get_topic_name(msg), vgroupId, tmq_get_table_name(msg)); + taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), vgroupId); while (1) { TAOS_ROW row = taos_fetch_row(msg); @@ -247,11 +304,16 @@ static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf); + if (0 != g_stConfInfo.saveRowFlag) { + saveConsumeContentToTbl(pInfo, buf); + } } totalRows++; } + addRowsToVgroupId(pInfo, vgroupId, totalRows); + return totalRows; } @@ -344,6 +406,32 @@ int32_t saveConsumeResult(SThreadInfo* pInfo) { taos_free_result(pRes); + #if 0 + // vgroups + for (i = 0; i < pInfo->numOfVgroups; i++) { + // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int + sprintf(sqlStr, "insert into %s.vgroup_%d values (%"PRId64", %d, %" PRId64 ", %" PRId64 ", %d)", + g_stConfInfo.cdbName, + now, + pInfo->consumerId, + pInfo->consumeMsgCnt, + pInfo->consumeRowCnt, + pInfo->checkresult); + + char tmpString[128]; + taosFprintfFile(g_fp, "%s, consume id %d result: %s\n", getCurrentTimeString(tmpString), pInfo->consumerId ,sqlStr); + + TAOS_RES* pRes = taos_query(pConn, sqlStr); + if (taos_errno(pRes) != 0) { + pError("error in save consumeinfo, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + exit(-1); + } + + taos_free_result(pRes); + } + #endif + return 0; } @@ -356,11 +444,13 @@ void loop_consume(SThreadInfo* pInfo) { char tmpString[128]; taosFprintfFile(g_fp, "%s consumer id %d start to loop pull msg\n", getCurrentTimeString(tmpString), pInfo->consumerId); + pInfo->ts = taosGetTimestampMs(); + while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { - totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId); + totalRows += msg_process(tmqMsg, pInfo, totalMsgs); } taos_free_result(tmqMsg); From ceb04101ebb2d592d02e0479c56aa881ca23f50f Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 2 Jun 2022 14:37:45 +0800 Subject: [PATCH 43/47] test: modify consumer processer --- tests/test/c/tmqSim.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 38b0313c14..d0b582f375 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -261,7 +261,13 @@ void addRowsToVgroupId(SThreadInfo* pInfo, int32_t vgroupId, int32_t rows) { } int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { - char sqlStr[1024] = {0}; + char sqlStr[1100] = {0}; + + if (strlen(buf) > 1024) { + taosFprintfFile(g_fp, "The length of one row[%d] is overflow 1024\n", strlen(buf)); + taosCloseFile(&g_fp); + exit(-1); + } TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0); assert(pConn != NULL); From 8cba836611ad7fdea61a1227e371c469c5c4644d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Jun 2022 14:54:21 +0800 Subject: [PATCH 44/47] fix:fix CI error in schemaless parameters --- source/common/src/systable.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/common/src/systable.c b/source/common/src/systable.c index d5999411ae..b1baf5a01b 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -89,8 +89,8 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "cachelast", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, - {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "schemaless", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // {.name = "update", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT}, // disable update }; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9bba1a1b2f..6d7a638c30 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1550,11 +1550,10 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.numOfStables, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); - - pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)statusB, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.schemaless, false); } } From 0160d2d5883068d38aff3031cd4131101df7c3a4 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 2 Jun 2022 15:06:15 +0800 Subject: [PATCH 45/47] test: add case for tmq --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) mode change 100644 => 100755 tests/system-test/fulltest.sh diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh old mode 100644 new mode 100755 index f03a07e8e1..8481417dbd --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -101,3 +101,4 @@ python3 ./test.py -f 7-tmq/subscribeStb1.py python3 ./test.py -f 7-tmq/subscribeStb2.py python3 ./test.py -f 7-tmq/subscribeStb3.py python3 ./test.py -f 7-tmq/subscribeStb4.py +python3 ./test.py -f 7-tmq/db.py From 0ead238adbf98fa5e96ee7269f4ab0620a229aa2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 2 Jun 2022 15:52:59 +0800 Subject: [PATCH 46/47] feat(tmq): support check col and tag modifiable --- source/dnode/mnode/impl/inc/mndTopic.h | 2 + source/dnode/mnode/impl/src/mndTopic.c | 50 ++++++++++++++++++++++++ source/dnode/vnode/src/tq/tqPush.c | 54 +++++++++++++------------- source/libs/wal/src/walWrite.c | 2 +- 4 files changed, 79 insertions(+), 29 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndTopic.h b/source/dnode/mnode/impl/inc/mndTopic.h index c5c4800e02..4aa18ea591 100644 --- a/source/dnode/mnode/impl/inc/mndTopic.h +++ b/source/dnode/mnode/impl/inc/mndTopic.h @@ -37,6 +37,8 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]); int32_t mndSetTopicCommitLogs(SMnode *pMnode, STrans *pTrans, SMqTopicObj *pTopic); +bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colIds); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 21b5e37e1e..e0d565c9af 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -70,6 +70,56 @@ const char *mndTopicGetShowName(const char topic[TSDB_TOPIC_FNAME_LEN]) { return strchr(topic, '.') + 1; } +bool mndCheckColAndTagModifiable(SMnode *pMnode, int64_t suid, const SArray *colAndTagIds) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + bool found = false; + while (1) { + SMqTopicObj *pTopic = NULL; + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + sdbRelease(pSdb, pTopic); + continue; + } + + SNode *pAst = NULL; + if (nodesStringToNode(pTopic->ast, &pAst) != 0) { + ASSERT(0); + return false; + } + + SHashObj *pColHash = NULL; + SNodeList *pNodeList; + nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); + SNode *pNode = NULL; + FOREACH(pNode, pNodeList) { + SColumnNode *pCol = (SColumnNode *)pNode; + if (pCol->tableId != suid) goto NEXT; + if (pColHash == NULL) { + pColHash = taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + } + if (pCol->colId > 0) { + taosHashPut(pColHash, &pCol->colId, sizeof(int16_t), NULL, 0); + } + } + + for (int32_t i = 0; i < taosArrayGetSize(colAndTagIds); i++) { + int16_t *pColId = taosArrayGet(colAndTagIds, i); + if (taosHashGet(pColHash, pColId, sizeof(int16_t)) != NULL) { + found = true; + goto NEXT; + } + } + + NEXT: + sdbRelease(pSdb, pTopic); + nodesDestroyNode(pAst); + if (found) return false; + } + return true; +} + SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 26e9dfe2e2..d94c3e387a 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -20,6 +20,28 @@ void tqTmrRspFunc(void* param, void* tmrId) { atomic_store_8(&pHandle->pushHandle.tmrStopped, 1); } +static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataBlkRsp* pRsp) { + SStreamDataSubmit* pSubmit = *ppSubmit; + while (pSubmit != NULL) { + ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); + if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { + /*ASSERT(0);*/ + } + // update processed + atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); + streamQSetSuccess(&pHandle->pushHandle.inputQ); + streamDataSubmitRefDec(pSubmit); + if (pRsp->blockNum > 0) { + *ppSubmit = pSubmit; + return 0; + } else { + pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); + } + } + *ppSubmit = pSubmit; + return -1; +} + int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { SMqDataBlkRsp rsp = {0}; // 1. guard and set status executing @@ -42,38 +64,14 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { } // 3. exec, after each success, update processed ver // first run - while (pSubmit != NULL) { - ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); - if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, &rsp, 0) < 0) { - /*ASSERT(0);*/ - } - // update processed - atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); - streamQSetSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); - if (rsp.blockNum > 0) { - goto SEND_RSP; - } else { - pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); - } + if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) { + goto SEND_RSP; } // set exec status closing atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__CLOSING); // second run - while (pSubmit != NULL) { - ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); - if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, &rsp, 0) < 0) { - /*ASSERT(0);*/ - } - // update processed - atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); - streamQSetSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); - if (rsp.blockNum > 0) { - goto SEND_RSP; - } else { - pSubmit = streamQNextItem(&pHandle->pushHandle.inputQ); - } + if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) { + goto SEND_RSP; } // set exec status idle atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 4e2270c37b..a9a8f8a1f4 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -19,7 +19,7 @@ #include "walInt.h" void walRestoreFromSnapshot(SWal *pWal, int64_t ver) { - pWal->vers.firstVer = -1; + /*pWal->vers.firstVer = -1;*/ pWal->vers.lastVer = ver; pWal->vers.commitVer = ver - 1; pWal->vers.snapshotVer = ver - 1; From 7eecbe81346370bc48d3c1fc78cfb68a7391b4c9 Mon Sep 17 00:00:00 2001 From: Zhengmao Zhu <70138133+fenghuazzm@users.noreply.github.com> Date: Thu, 2 Jun 2022 16:05:00 +0800 Subject: [PATCH 47/47] docs: Update 02-rest-api.mdx MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加更多有关启动 taosAdapter 服务的描述。 --- docs-cn/14-reference/02-rest-api/02-rest-api.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-cn/14-reference/02-rest-api/02-rest-api.mdx b/docs-cn/14-reference/02-rest-api/02-rest-api.mdx index c7680ab3e9..43099319b9 100644 --- a/docs-cn/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs-cn/14-reference/02-rest-api/02-rest-api.mdx @@ -16,7 +16,7 @@ RESTful 接口不依赖于任何 TDengine 的库，因此客户端不需要安 在已经安装 TDengine 服务器端的情况下，可以按照如下方式进行验证。 -下面以 Ubuntu 环境中使用 curl 工具（确认已经安装）来验证 RESTful 接口的正常。 +下面以 Ubuntu 环境中使用 curl 工具（确认已经安装）来验证 RESTful 接口的正常，验证前请确认 taosAdapter 服务已开启，在 Linux 系统上此服务默认由 systemd 管理，使用命令 `systemctl start taosadapter` 启动。 下面示例是列出所有的数据库，请把 h1.taosdata.com 和 6041（缺省值）替换为实际运行的 TDengine 服务 FQDN 和端口号：