From 383e7a495bd2b533d46fa18b3b7eca1b3c2872b7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 28 Jun 2022 13:44:50 +0800 Subject: [PATCH 1/7] fix(query): enable output check --- examples/c/stream_demo.c | 7 +- examples/c/tmq.c | 4 +- include/libs/stream/tstream.h | 5 +- source/dnode/vnode/src/tq/tqRead.c | 6 +- source/libs/executor/src/executil.c | 128 +++++++++--------- source/libs/executor/src/timewindowoperator.c | 125 +++++++++-------- source/libs/stream/src/streamExec.c | 15 +- 7 files changed, 152 insertions(+), 138 deletions(-) diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index f5cb7f1120..1e9058d628 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -98,9 +98,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = taos_query(pConn, - "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 partition " - "by tbname interval(10s) "); + pRes = taos_query( + pConn, + "create stream stream1 trigger max_delay 10s into outstb as select _wstartts, sum(k) from st1 partition " + "by tbname session(ts, 10s) "); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 6fb7e7a1fc..53b259b0af 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -137,8 +137,8 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3,t1 from st1 where t1 = 2000"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index db928f194c..74c885ebcc 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -327,9 +327,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem taosWriteQitem(pTask->inputQueue->queue, pItem); } - if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0 && - pTask->triggerStatus == TASK_TRIGGER_STATUS__IN_ACTIVE) { - atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); + if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE); } // TODO: back pressure diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index cbee639911..be2a35c901 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -231,7 +231,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) { tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); pBlock->info.groupId = 0; - pBlock->info.uid = pHandle->msgIter.uid; // set the uid of table for submit block + pBlock->info.uid = pHandle->msgIter.uid; pBlock->info.rows = pHandle->msgIter.numOfRows; while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { @@ -251,8 +251,8 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) { } return 0; -FAIL: // todo refactor here - // if (*ppCols) taosArrayDestroy(*ppCols); +FAIL: + tDeleteSSDataBlock(pBlock); return -1; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index be6ca5145a..5c2f32126b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "os.h" -#include "index.h" #include "function.h" #include "functionMgt.h" +#include "index.h" +#include "os.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" @@ -25,45 +25,41 @@ #include "executorimpl.h" #include "tcompression.h" -void initResultRowInfo(SResultRowInfo *pResultRowInfo) { - pResultRowInfo->size = 0; +void initResultRowInfo(SResultRowInfo* pResultRowInfo) { + pResultRowInfo->size = 0; pResultRowInfo->cur.pageId = -1; } -void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { +void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo) { if (pResultRowInfo == NULL) { return; } - for(int32_t i = 0; i < pResultRowInfo->size; ++i) { -// if (pResultRowInfo->pResult[i]) { -// taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); -// } + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + // if (pResultRowInfo->pResult[i]) { + // taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); + // } } } -void closeAllResultRows(SResultRowInfo *pResultRowInfo) { -// do nothing +void closeAllResultRows(SResultRowInfo* pResultRowInfo) { + // do nothing } -bool isResultRowClosed(SResultRow* pRow) { - return (pRow->closed == true); -} +bool isResultRowClosed(SResultRow* pRow) { return (pRow->closed == true); } -void closeResultRow(SResultRow* pResultRow) { - pResultRow->closed = true; -} +void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; } // TODO refactor: use macro SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset) { assert(index >= 0 && offset != NULL); - return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]); + return (SResultRowEntryInfo*)((char*)pRow->pEntryInfo + offset[index]); } size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { int32_t rowSize = (numOfOutput * sizeof(SResultRowEntryInfo)) + sizeof(SResultRow); - for(int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t i = 0; i < numOfOutput; ++i) { rowSize += pCtx[i].resDataInfo.interBufSize; } @@ -74,31 +70,29 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); taosArrayDestroy(pGroupResInfo->pRows); - pGroupResInfo->pRows = NULL; - pGroupResInfo->index = 0; + pGroupResInfo->pRows = NULL; + pGroupResInfo->index = 0; } static int32_t resultrowComparAsc(const void* p1, const void* p2) { - SResKeyPos* pp1 = *(SResKeyPos**) p1; - SResKeyPos* pp2 = *(SResKeyPos**) p2; + SResKeyPos* pp1 = *(SResKeyPos**)p1; + SResKeyPos* pp2 = *(SResKeyPos**)p2; if (pp1->groupId == pp2->groupId) { - int64_t pts1 = *(int64_t*) pp1->key; - int64_t pts2 = *(int64_t*) pp2->key; + int64_t pts1 = *(int64_t*)pp1->key; + int64_t pts2 = *(int64_t*)pp2->key; if (pts1 == pts2) { return 0; } else { - return pts1 < pts2? -1:1; + return pts1 < pts2 ? -1 : 1; } } else { - return pp1->groupId < pp2->groupId? -1:1; + return pp1->groupId < pp2->groupId ? -1 : 1; } } -static int32_t resultrowComparDesc(const void* p1, const void* p2) { - return resultrowComparAsc(p2, p1); -} +static int32_t resultrowComparDesc(const void* p1, const void* p2) { return resultrowComparAsc(p2, p1); } void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) { if (pGroupResInfo->pRows != NULL) { @@ -110,20 +104,20 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); size_t keyLen = 0; - while((pData = taosHashIterate(pHashmap, pData)) != NULL) { + while ((pData = taosHashIterate(pHashmap, pData)) != NULL) { void* key = taosHashGetKey(pData, &keyLen); SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition)); - p->groupId = *(uint64_t*) key; - p->pos = *(SResultRowPosition*) pData; + p->groupId = *(uint64_t*)key; + p->pos = *(SResultRowPosition*)pData; memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t)); taosArrayPush(pGroupResInfo->pRows, &p); } if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) { - __compar_fn_t fn = (order == TSDB_ORDER_ASC)? resultrowComparAsc:resultrowComparDesc; + __compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc; qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn); } @@ -155,7 +149,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return 0; } - return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); + return (int32_t)taosArrayGetSize(pGroupResInfo->pRows); } SArray* createSortInfo(SNodeList* pNodeList) { @@ -194,12 +188,13 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { pBlock->info.type = STREAM_INVALID; for (int32_t i = 0; i < numOfCols; ++i) { - SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); -// if (!pDescNode->output) { // todo disable it temporarily -// continue; -// } + SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); + if (!pDescNode->output) { // todo disable it temporarily + continue; + } - SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); + SColumnInfoData idata = + createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; idata.info.precision = pDescNode->dataType.precision; @@ -211,10 +206,10 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { SMetaReader* mr = (SMetaReader*)pContext; - if(nodeType(*pNode) == QUERY_NODE_COLUMN){ + if (nodeType(*pNode) == QUERY_NODE_COLUMN) { SColumnNode* pSColumnNode = *(SColumnNode**)pNode; - SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { return DEAL_RES_ERROR; } @@ -227,8 +222,8 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; - }else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { - int32_t len = ((const STag*)p) -> len; + } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { + int32_t len = ((const STag*)p)->len; res->datum.p = taosMemoryCalloc(len + 1, 1); memcpy(res->datum.p, p, len); } else if (IS_VAR_DATA_TYPE(pSColumnNode->node.resType.type)) { @@ -240,10 +235,10 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { } nodesDestroyNode(*pNode); *pNode = (SNode*)res; - }else if (nodeType(*pNode) == QUERY_NODE_FUNCTION){ - SFunctionNode * pFuncNode = *(SFunctionNode**)pNode; - if(pFuncNode->funcType == FUNCTION_TYPE_TBNAME){ - SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + } else if (nodeType(*pNode) == QUERY_NODE_FUNCTION) { + SFunctionNode* pFuncNode = *(SFunctionNode**)pNode; + if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) { + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { return DEAL_RES_ERROR; } @@ -263,12 +258,12 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ - SMetaReader mr = {0}; +static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { + SMetaReader mr = {0}; metaReaderInit(&mr, metaHandle, 0); metaGetTableEntryByUid(&mr, info->uid); - SNode *pTagCondTmp = nodesCloneNode(pTagCond); + SNode* pTagCondTmp = nodesCloneNode(pTagCond); nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr); metaReaderClear(&mr); @@ -281,7 +276,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ } ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); - SValueNode *pValue = (SValueNode *)pNew; + SValueNode* pValue = (SValueNode*)pNew; ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL); bool result = pValue->datum.b; @@ -292,12 +287,12 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); - if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY; uint64_t tableUid = pScanNode->uid; pListInfo->suid = pScanNode->suid; - + SNode* pTagCond = (SNode*)pListInfo->pTagCond; SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond; if (pScanNode->tableType == TSDB_SUPER_TABLE) { @@ -306,7 +301,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; SArray* res = taosArrayInit(8, sizeof(uint64_t)); - //code = doFilterTag(pTagIndexCond, &metaArg, res); + // code = doFilterTag(pTagIndexCond, &metaArg, res); code = TSDB_CODE_INDEX_REBUILDING; if (code == TSDB_CODE_INDEX_REBUILDING) { code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList); @@ -328,33 +323,33 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList); } - if(pTagCond){ + if (pTagCond) { int32_t i = 0; - while(i < taosArrayGetSize(pListInfo->pTableList)) { + while (i < taosArrayGetSize(pListInfo->pTableList)) { STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); - bool isOk = isTableOk(info, pTagCond, metaHandle); - if(!isOk){ + bool isOk = isTableOk(info, pTagCond, metaHandle); + if (!isOk) { taosArrayRemove(pListInfo->pTableList, i); continue; } i++; } } - }else { // Create one table group. + } else { // Create one table group. STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); - if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY; - //put into list as default group, remove it if grouping sorting is required later + // put into list as default group, remove it if grouping sorting is required later taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); return code; } SArray* extractPartitionColInfo(SNodeList* pNodeList) { - if(!pNodeList) { + if (!pNodeList) { return NULL; } @@ -383,7 +378,6 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { return pList; } - SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type) { size_t numOfCols = LIST_LENGTH(pNodeList); @@ -399,7 +393,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchInfo c = {0}; c.output = true; - c.colId = pColNode->colId; + c.colId = pColNode->colId; c.srcSlotId = pColNode->slotId; c.matchType = type; c.targetSlotId = pNode->slotId; @@ -675,8 +669,8 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, } for (int32_t i = 1; i < numOfOutput; ++i) { - (*rowEntryInfoOffset)[i] = - (int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize); + (*rowEntryInfoOffset)[i] = (int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + + pFuncCtx[i - 1].resDataInfo.interBufSize); } setSelectValueColumnInfo(pFuncCtx, numOfOutput); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 963e714972..edd1ec0e75 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -34,7 +34,7 @@ typedef struct SWinRes { typedef struct SPullWindowInfo { STimeWindow window; - uint64_t groupId; + uint64_t groupId; } SPullWindowInfo; static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); @@ -695,11 +695,11 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL){ + if (pBlock == NULL) { qDebug("======printDataBlock Block is Null"); return; } - char *pBuf = NULL; + char* pBuf = NULL; qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); taosMemoryFree(pBuf); } @@ -1255,10 +1255,10 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* TSKEY* tsCols = (TSKEY*)pTsCol->pData; uint64_t* pGpDatas = NULL; if (pBlock->info.type == STREAM_RETRIEVE) { - SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); - pGpDatas = (uint64_t*)pGpCol->pData; + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); + pGpDatas = (uint64_t*)pGpCol->pData; } - int32_t step = 0; + int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -1297,8 +1297,8 @@ bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark; } -static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, - SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) { +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, + SHashObj* pPullDataMap, SArray* closeWins) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1309,15 +1309,18 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); - SWinRes winRe = {.ts = win.skey, .groupId = groupId,}; + SWinRes winRe = { + .ts = win.skey, + .groupId = groupId, + }; void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes)); if (isCloseWindow(&win, pSup)) { if (chIds && pPullDataMap) { - SArray* chAy = *(SArray**) chIds; + SArray* chAy = *(SArray**)chIds; int32_t size = taosArrayGetSize(chAy); - qInfo("======window %ld wait child size:%d", win.skey ,size); + qInfo("======window %ld wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i)); + qInfo("======window %ld wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { @@ -1341,7 +1344,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { - SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); @@ -1431,7 +1434,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - //it should be empty. + // it should be empty. taosHashCleanup(pInfo->pPullDataMap); taosArrayDestroy(pInfo->pPullWins); blockDataDestroy(pInfo->pPullDataRes); @@ -2236,10 +2239,10 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } -int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, - int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) { - int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, - eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC); +int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey, + STimeWindow* pNextWin) { + int32_t forwardRows = + getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC); int32_t prevEndPos = forwardRows - 1 + startPos; return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC); } @@ -2277,8 +2280,11 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc pInfo->interval.precision, NULL); while (1) { if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) { - bool ignore = true; - SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,}; + bool ignore = true; + SWinRes winRes = { + .ts = nextWin.skey, + .groupId = tableGroupId, + }; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; @@ -2289,7 +2295,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc int32_t index = -1; SArray* chArray = NULL; if (chIds) { - chArray = *(void**) chIds; + chArray = *(void**)chIds; int32_t chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } @@ -2300,7 +2306,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); } } - if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) { + if (index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) { ignore = false; } } @@ -2385,17 +2391,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB if (size - (*pIndex) == 0) { return; } - blockDataEnsureCapacity(pBlock, size - (*pIndex) ); + blockDataEnsureCapacity(pBlock, size - (*pIndex)); ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); for (; (*pIndex) < size; (*pIndex)++) { - SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) ); - SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0); + SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex)); + SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 0); colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1); + SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 1); colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); - SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2); + SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 2); colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); pBlock->info.rows++; } @@ -2408,15 +2414,15 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0); - TSKEY* tsData = (TSKEY*)pStartCol->pData; + TSKEY* tsData = (TSKEY*)pStartCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2); - uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; - int32_t chId = getChildIndex(pBlock); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); for (int32_t i = 0; i < pBlock->info.rows; i++) { SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]}; - void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes)); + void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes)); if (chIds) { - SArray* chArray = *(SArray**) chIds; + SArray* chArray = *(SArray**)chIds; int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); if (index != -1) { taosArrayRemove(chArray, index); @@ -2484,7 +2490,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); maxTs = TMAX(maxTs, pBlock->info.window.ekey); - if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) { + if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || + pBlock->info.type == STREAM_INVALID) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); @@ -2513,8 +2520,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, - pBlock, pUpWins); + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); removeResults(pUpWins, pUpdated); taosArrayDestroy(pUpWins); if (taosArrayGetSize(pUpdated) > 0) { @@ -2552,8 +2558,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, pInfo->pPullDataMap, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, + pUpdated); closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } @@ -2874,12 +2880,24 @@ static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) { } static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; + SResultWindowInfo win = { + .pos.offset = -1, + .pos.pageId = -1, + .win.skey = ts, + .win.ekey = ts, + .isOutput = false, + }; return taosArrayInsert(pWinInfos, index, &win); } static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; + SResultWindowInfo win = { + .pos.offset = -1, + .pos.pageId = -1, + .win.skey = ts, + .win.ekey = ts, + .isOutput = false, + }; return taosArrayPush(pWinInfos, &win); } @@ -3277,14 +3295,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3352,11 +3370,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -4150,16 +4168,16 @@ _error: // merge interval operator typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; - SList* groupIntervals; - SListIter groupIntervalsIter; - bool hasGroupId; - uint64_t groupId; - SSDataBlock* prefetchedBlock; - bool inputBlocksFinished; + SList* groupIntervals; + SListIter groupIntervalsIter; + bool hasGroupId; + uint64_t groupId; + SSDataBlock* prefetchedBlock; + bool inputBlocksFinished; } SMergeIntervalAggOperatorInfo; typedef struct SGroupTimeWindow { - uint64_t groupId; + uint64_t groupId; STimeWindow window; } SGroupTimeWindow; @@ -4169,7 +4187,8 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } -static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, SSDataBlock* pResultBlock) { +static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, + SSDataBlock* pResultBlock) { SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -4202,7 +4221,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t SListNode* listNode = NULL; while ((listNode = tdListNext(&iter)) != NULL) { SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data; - if (prevGrpWin->groupId != tableGroupId ) { + if (prevGrpWin->groupId != tableGroupId) { continue; } STimeWindow* prevWin = &prevGrpWin->window; @@ -4445,4 +4464,4 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; -} \ No newline at end of file +} diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index c75e6c004a..b3ceb92505 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -46,15 +46,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - //SSDataBlock block = {0}; - //block.info.type = STREAM_PUSH_EMPTY; - //block.info.childId = pTask->selfChildId; + SSDataBlock block = {0}; + /*block.info.type = STREAM_PUSH_EMPTY;*/ + // block.info.childId = pTask->selfChildId; SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); - SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true); - pBlock->info.type = STREAM_PUSH_EMPTY; - pBlock->info.childId = pTask->selfChildId; - taosArrayPush(pRes, pBlock); + /*SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true);*/ + assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); + block.info.type = STREAM_PUSH_EMPTY; + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); } break; } From 58803fafcfba4bc1c48d9648540d087ea2471e0d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 28 Jun 2022 13:59:49 +0800 Subject: [PATCH 2/7] fix(query): check datablock should check output flag --- examples/c/tmq.c | 2 +- include/common/tdatablock.h | 7 +- include/common/tmsg.h | 1 + source/client/src/clientMsgHandler.c | 28 +++-- source/common/src/tdatablock.c | 9 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 26 ++--- source/dnode/mnode/impl/src/mndShow.c | 4 +- source/dnode/vnode/src/tq/tqExec.c | 2 +- source/libs/command/src/command.c | 121 +++++++++----------- source/libs/command/src/explain.c | 70 +++++------ source/libs/executor/inc/executil.h | 82 ++++++------- source/libs/executor/src/dataDispatcher.c | 21 ++-- source/libs/executor/src/executil.c | 6 +- source/libs/executor/src/executorimpl.c | 82 ++++++------- source/libs/stream/src/streamData.c | 4 +- source/libs/stream/src/streamDispatch.c | 4 +- 16 files changed, 230 insertions(+), 239 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 53b259b0af..68b0492ace 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -138,7 +138,7 @@ int32_t create_topic() { taos_free_result(pRes); /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3,t1 from st1 where t1 = 2000"); + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1 where t1 = 2000"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 6653fdd9cd..72e0af657c 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -224,7 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); -int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); @@ -232,9 +232,8 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); -void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, - int8_t needCompress); -const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); +void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c5b0b89311..0afcaaa50d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -623,6 +623,7 @@ typedef struct { col_id_t colId; int16_t slotId; }; + bool output; // TODO remove it later int16_t type; int32_t bytes; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 45a525d124..94bd07a18a 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -67,11 +67,11 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { dstEpSet.eps[dstEpSet.inUse].fqdn); } else if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; - SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; - SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse]; - tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", - pOrig->inUse, pOrig->numOfEps, pOrigEp->fqdn, pOrigEp->port, - connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn, pNewEp->port); + SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; + SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse]; + tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", pOrig->inUse, pOrig->numOfEps, + pOrigEp->fqdn, pOrigEp->port, connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn, + pNewEp->port); updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet); } @@ -308,13 +308,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { blockDataEnsureCapacity(pBlock, numOfCfg); for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { - SVariablesInfo *pInfo = taosArrayGet(pVars, i); + SVariablesInfo* pInfo = taosArrayGet(pVars, i); char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataAppend(pColInfo, i, name, false); - + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); @@ -324,14 +324,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { pBlock->info.rows = numOfCfg; *block = pBlock; - + return TSDB_CODE_SUCCESS; } - static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = NULL; - int32_t code = buildShowVariablesBlock(pVars, &pBlock); + int32_t code = buildShowVariablesBlock(pVars, &pBlock); if (code) { return code; } @@ -351,7 +350,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -363,7 +362,7 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); } else { - SShowVariablesRsp rsp = {0}; + SShowVariablesRsp rsp = {0}; SRetrieveTableRsp* pRes = NULL; code = tDeserializeSShowVariablesRsp(pMsg->pData, pMsg->len, &rsp); if (TSDB_CODE_SUCCESS == code) { @@ -372,7 +371,7 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) if (TSDB_CODE_SUCCESS == code) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, false); } - + tFreeSShowVariablesRsp(&rsp); } @@ -384,7 +383,6 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) return code; } - __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { switch (msgType) { case TDMT_MND_CONNECT: diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1ec298ee15..dd9500e8d8 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1613,7 +1613,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); + printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, + pDataBlock->info.childId, pDataBlock->info.groupId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); for (int32_t k = 0; k < numOfCols; k++) { @@ -1902,8 +1903,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { return rname.childTableName; } -void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, - int8_t needCompress) { +void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { // todo extract method int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); @@ -1913,6 +1913,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + *((int16_t*)data) = pColInfoData->info.type; data += sizeof(int16_t); @@ -1960,7 +1961,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen *groupId = pBlock->info.groupId; } -const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { +const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { const char* pStart = pData; int32_t dataLen = *(int32_t*)pStart; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 980749ca70..234a133243 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -17,7 +17,6 @@ #include "dmInt.h" #include "systable.h" - extern SConfig *tsCfg; static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { @@ -83,7 +82,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (*pMgmt->getVnodeLoadsFp)(&vinfo); req.pVloads = vinfo.pVloads; - SMonMloadInfo minfo = {0}; + SMonMloadInfo minfo = {0}; (*pMgmt->getMnodeLoadsFp)(&minfo); req.mload = minfo.load; @@ -186,10 +185,10 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } -SSDataBlock* dmBuildVariablesBlock(void) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); +SSDataBlock *dmBuildVariablesBlock(void) { + SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); size_t size = 0; - const SSysTableMeta* pMeta = NULL; + const SSysTableMeta *pMeta = NULL; getInfosDbMeta(&pMeta, &size); int32_t index = 0; @@ -215,7 +214,7 @@ SSDataBlock* dmBuildVariablesBlock(void) { return pBlock; } -int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { +int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) { int32_t numOfCfg = taosArrayGetSize(tsCfg->array); int32_t numOfRows = 0; blockDataEnsureCapacity(pBlock, numOfCfg); @@ -230,8 +229,8 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataAppend(pColInfo, i, name, false); - - char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); @@ -241,9 +240,8 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { numOfRows++; } - pBlock->info.rows = numOfRows; - + return TSDB_CODE_SUCCESS; } @@ -267,7 +265,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - SSDataBlock* pBlock = dmBuildVariablesBlock(); + SSDataBlock *pBlock = dmBuildVariablesBlock(); dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); @@ -283,14 +281,14 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - char *pStart = pRsp->data; + char *pStart = pRsp->data; *(int32_t *)pStart = htonl(numOfCols); pStart += sizeof(int32_t); // number of columns for (int32_t i = 0; i < numOfCols; ++i) { SSysTableSchema *pSchema = (SSysTableSchema *)pStart; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); - + pSchema->bytes = htonl(pColInfo->info.bytes); pSchema->colId = htons(pColInfo->info.colId); pSchema->type = pColInfo->info.type; @@ -299,7 +297,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t len = 0; - blockCompressEncode(pBlock, pStart, &len, numOfCols, false); + blockEncode(pBlock, pStart, &len, numOfCols, false); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 164bcc7d60..6014adbe95 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "mndShow.h" -#include "systable.h" #include "mndPrivilege.h" +#include "systable.h" #define SHOW_STEP_SIZE 100 @@ -307,7 +307,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } int32_t len = 0; - blockCompressEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); + blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index afeeeab500..c7d7c787a2 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs // TODO enable compress int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); + blockEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index cb14b916af..d6b6456515 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -18,8 +18,8 @@ #include "tdatablock.h" #include "tglobal.h" -extern SConfig *tsCfg; -static int32_t getSchemaBytes(const SSchema* pSchema) { +extern SConfig* tsCfg; +static int32_t getSchemaBytes(const SSchema* pSchema) { switch (pSchema->type) { case TSDB_DATA_TYPE_BINARY: return (pSchema->bytes - VARSTR_HEADER_SIZE); @@ -104,7 +104,7 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -113,9 +113,8 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execResetQueryCache() { return catalogClearCache(); } - static SSDataBlock* buildCreateDBResultDataBlock() { - SSDataBlock* pBlock = createDataBlock(); + SSDataBlock* pBlock = createDataBlock(); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); blockDataAppendColInfo(pBlock, &infoData); @@ -149,14 +148,14 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { return v; } -char *buildRetension(SArray *pRetension) { +char* buildRetension(SArray* pRetension) { size_t size = taosArrayGetSize(pRetension); if (size == 0) { return NULL; } - char *p1 = taosMemoryCalloc(1, 100); - SRetention *p = taosArrayGet(pRetension, 0); + char* p1 = taosMemoryCalloc(1, 100); + SRetention* p = taosArrayGet(pRetension, 0); int32_t len = 0; @@ -185,8 +184,7 @@ char *buildRetension(SArray *pRetension) { return p1; } - -static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, SDbCfgInfo* pCfg) { +static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) { blockDataEnsureCapacity(pBlock, 1); pBlock->info.rows = 1; @@ -198,7 +196,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0}; int32_t len = 0; - char *prec = NULL; + char* prec = NULL; switch (pCfg->precision) { case TSDB_TIME_PRECISION_MILLI: prec = TSDB_TIME_PRECISION_MILLI_STR; @@ -214,15 +212,16 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S break; } - char *retentions = buildRetension(pCfg->pRetensions); - - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " - "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " - "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", - dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, - pCfg->fsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, - pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups, - 1 == pCfg->numOfStables); + char* retentions = buildRetension(pCfg->pRetensions); + + len += sprintf(buf2 + VARSTR_HEADER_SIZE, + "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " + "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " + "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", + dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod, + pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages, + pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups, + 1 == pCfg->numOfStables); if (retentions) { len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); @@ -230,11 +229,10 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S } (varDataLen(buf2)) = len; - + colDataAppend(pCol2, 0, buf2, false); } - static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = buildCreateDBResultDataBlock(); setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); @@ -254,7 +252,7 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT (*pRsp)->numOfCols = htonl(SHOW_CREATE_DB_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_DB_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_DB_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -276,14 +274,14 @@ static SSDataBlock* buildCreateTbResultDataBlock() { void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { SSchema* pSchema = pCfg->pSchemas + i; - char type[32]; + char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE)); + sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } @@ -291,19 +289,18 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; - char type[32]; + char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE)); + sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } - void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; @@ -311,13 +308,12 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { } } - int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { - SArray *pTagVals = NULL; - STag *pTag = (STag*)pCfg->pTags; - + SArray* pTagVals = NULL; + STag* pTag = (STag*)pCfg->pTags; + if (pCfg->pTags && pTag->flags & TD_TAG_JSON) { - char *pJson = parseTagDatatoJson(pTag); + char* pJson = parseTagDatatoJson(pTag); if (pJson) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); taosMemoryFree(pJson); @@ -325,8 +321,8 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { return TSDB_CODE_SUCCESS; } - - int32_t code = tTagToValArray((const STag *)pCfg->pTags, &pTagVals); + + int32_t code = tTagToValArray((const STag*)pCfg->pTags, &pTagVals); if (code) { return code; } @@ -339,20 +335,20 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (i > 0) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); } - + if (j >= valueNum) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); continue; } - - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, j); + + STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); if (pSchema->colId > pTagVal->cid) { qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid); taosArrayDestroy(pTagVals); return TSDB_CODE_APP_ERROR; } else if (pSchema->colId == pTagVal->cid) { - char type = pTagVal->type; - int32_t tlen = 0; + char type = pTagVal->type; + int32_t tlen = 0; if (IS_VAR_DATA_TYPE(type)) { dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); @@ -364,7 +360,6 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { } else { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); } - /* if (type == TSDB_DATA_TYPE_BINARY) { @@ -372,7 +367,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (num) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); } - + memcpy(buf + VARSTR_HEADER_SIZE + *len, pTagVal->pData, pTagVal->nData); *len += pTagVal->nData; } @@ -397,7 +392,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { taosArrayDestroy(pTagVals); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) { @@ -426,7 +421,7 @@ void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP("); for (int32_t i = 0; i < funcNum; ++i) { char* pFunc = taosArrayGet(pCfg->pFuncs, i); - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); } *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); } @@ -436,7 +431,7 @@ void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) { } } -static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, STableCfg* pCfg) { +static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char* tbName, STableCfg* pCfg) { int32_t code = 0; blockDataEnsureCapacity(pBlock, 1); pBlock->info.rows = 1; @@ -454,7 +449,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); - appendTagFields(buf2, &len, pCfg); + appendTagFields(buf2, &len, pCfg); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pCfg); } else if (TSDB_CHILD_TABLE == pCfg->tableType) { @@ -474,16 +469,15 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, } varDataLen(buf2) = len; - + colDataAppend(pCol2, 0, buf2, false); return TSDB_CODE_SUCCESS; } - static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = buildCreateTbResultDataBlock(); - int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg); + int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg); if (code) { return code; } @@ -503,7 +497,7 @@ static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRs (*pRsp)->numOfCols = htonl(SHOW_CREATE_TB_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_TB_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_TB_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -516,17 +510,17 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR terrno = TSDB_CODE_TSC_NOT_STABLE_ERROR; return terrno; } - + return execShowCreateTable(pStmt, pRsp); } -static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { +static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) { - return terrno; + return terrno; } if (taosSetCfg(tsCfg, pStmt->config)) { - return terrno; + return terrno; } return TSDB_CODE_SUCCESS; @@ -551,21 +545,20 @@ static SSDataBlock* buildLocalVariablesResultDataBlock() { return pBlock; } - int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { int32_t numOfCfg = taosArrayGetSize(tsCfg->array); int32_t numOfRows = 0; blockDataEnsureCapacity(pBlock, numOfCfg); for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { - SConfigItem *pItem = taosArrayGet(tsCfg->array, i); + SConfigItem* pItem = taosArrayGet(tsCfg->array, i); char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataAppend(pColInfo, i, name, false); - - char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); @@ -575,16 +568,14 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { numOfRows++; } - pBlock->info.rows = numOfRows; - + return TSDB_CODE_SUCCESS; } - static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = buildLocalVariablesResultDataBlock(); - int32_t code = setLocalVariablesResultIntoDataBlock(pBlock); + int32_t code = setLocalVariablesResultIntoDataBlock(pBlock); if (code) { return code; } @@ -604,7 +595,7 @@ static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_LOCAL_VARIABLES_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_LOCAL_VARIABLES_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_LOCAL_VARIABLES_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index ef4ac0b639..7af36a0842 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "tdatablock.h" #include "commandInt.h" #include "plannodes.h" #include "query.h" #include "tcommon.h" +#include "tdatablock.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level); @@ -216,7 +216,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode)); - QRY_ERR_RET(nodesListAppend(*pChildren, (SNode*)pResNode)); + QRY_ERR_RET(nodesListAppend(*pChildren, (SNode *)pResNode)); } return TSDB_CODE_SUCCESS; @@ -232,14 +232,14 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group SExplainRsp *rsp = NULL; for (int32_t i = 0; i < group->nodeNum; ++i) { rsp = taosArrayGet(group->nodeExecInfo, i); -/* - if (group->physiPlanExecIdx >= rsp->numOfPlans) { - qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; - } + /* + if (group->physiPlanExecIdx >= rsp->numOfPlans) { + qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); + return TSDB_CODE_QRY_APP_ERROR; + } - taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); -*/ + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); + */ taosArrayPush(*pExecInfo, rsp->subplanInfo); } @@ -426,23 +426,23 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (EXPLAIN_MODE_ANALYZE == ctx->mode) { EXPLAIN_ROW_NEW(level + 1, "I/O: "); - int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); + int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); struct STableScanAnalyzeInfo info = {0}; int32_t maxIndex = 0; int32_t totalRows = 0; - for(int32_t i = 0; i < nodeNum; ++i) { + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, i); STableScanAnalyzeInfo *pScanInfo = (STableScanAnalyzeInfo *)execInfo->verboseInfo; - info.totalBlocks += pScanInfo->totalBlocks; - info.loadBlocks += pScanInfo->loadBlocks; - info.totalRows += pScanInfo->totalRows; - info.skipBlocks += pScanInfo->skipBlocks; - info.filterTime += pScanInfo->filterTime; - info.loadBlockStatis += pScanInfo->loadBlockStatis; + info.totalBlocks += pScanInfo->totalBlocks; + info.loadBlocks += pScanInfo->loadBlocks; + info.totalRows += pScanInfo->totalRows; + info.skipBlocks += pScanInfo->skipBlocks; + info.filterTime += pScanInfo->filterTime; + info.loadBlockStatis += pScanInfo->loadBlockStatis; info.totalCheckedRows += pScanInfo->totalCheckedRows; - info.filterOutBlocks += pScanInfo->filterOutBlocks; + info.filterOutBlocks += pScanInfo->filterOutBlocks; if (pScanInfo->totalRows > totalRows) { totalRows = pScanInfo->totalRows; @@ -468,13 +468,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - //Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end, start offset by 1.470 ms. + // Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end, + // start offset by 1.470 ms. SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, maxIndex); STableScanAnalyzeInfo *p1 = (STableScanAnalyzeInfo *)execInfo->verboseInfo; EXPLAIN_ROW_NEW(level + 1, " "); - EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows, "tbd", - execInfo->startupCost, execInfo->totalCost); + EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows, + "tbd", execInfo->startupCost, execInfo->totalCost); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_END(); @@ -752,7 +753,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_NEW(level + 1, "Sort Key: "); if (pResNode->pExecInfo) { for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) { - SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pSortNode->pSortKeys, i); + SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pSortNode->pSortKeys, i); EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr)); } } @@ -907,16 +908,16 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pFillNode->pValues) { - SNodeListNode *pValues = (SNodeListNode*)pFillNode->pValues; + SNodeListNode *pValues = (SNodeListNode *)pFillNode->pValues; EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_VALUE_FORMAT); - SNode* tNode = NULL; + SNode *tNode = NULL; int32_t i = 0; FOREACH(tNode, pValues->pNodeList) { if (i) { EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } - SValueNode* tValue = (SValueNode*)tNode; - char *value = nodesGetStrValueFromNode(tValue); + SValueNode *tValue = (SValueNode *)tNode; + char *value = nodesGetStrValueFromNode(tValue); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, value); taosMemoryFree(value); ++i; @@ -926,8 +927,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey, - pFillNode->timeRange.ekey); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey, pFillNode->timeRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1070,13 +1070,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - + if (EXPLAIN_MODE_ANALYZE == ctx->mode) { // sort key EXPLAIN_ROW_NEW(level + 1, "Merge Key: "); if (pResNode->pExecInfo) { for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) { - SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i); + SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i); EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr)); } } @@ -1115,7 +1115,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGE_KEYS_FORMAT); for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) { - SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i); + SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i); EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr)); } EXPLAIN_ROW_END(); @@ -1130,7 +1130,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } } break; - } + } default: qError("not supported physical node type %d", pNode->type); return TSDB_CODE_QRY_APP_ERROR; @@ -1190,12 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SSDataBlock *pBlock = createDataBlock(); + SSDataBlock *pBlock = createDataBlock(); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1); blockDataAppendColInfo(pBlock, &infoData); blockDataEnsureCapacity(pBlock, rowNum); - SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData *pInfoData = taosArrayGet(pBlock->pDataBlock, 0); char buf[1024] = {0}; for (int32_t i = 0; i < rowNum; ++i) { @@ -1219,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); int32_t len = 0; - blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); + blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f3e1eb47e8..df961c00fa 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -25,37 +25,36 @@ #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ assert(sizeof(_uid) == sizeof(uint64_t)); \ - *(uint64_t *)(_k) = (_uid); \ + *(uint64_t*)(_k) = (_uid); \ memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ } while (0) -#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \ - do { \ - assert(sizeof(_uid) == sizeof(uint64_t)); \ - *(void **)(_k) = (_buf); \ - *(uint64_t *)((_k) + POINTER_BYTES) = (_uid); \ - memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \ +#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \ + do { \ + assert(sizeof(_uid) == sizeof(uint64_t)); \ + *(void**)(_k) = (_buf); \ + *(uint64_t*)((_k) + POINTER_BYTES) = (_uid); \ + memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \ } while (0) - -#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) +#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) typedef struct SGroupResInfo { int32_t index; - SArray* pRows; // SArray + SArray* pRows; // SArray } SGroupResInfo; typedef struct SResultRow { - int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t offset:29; // row index in buffer page - bool startInterp; // the time window start timestamp has done the interpolation already. - bool endInterp; // the time window end timestamp has done the interpolation already. - bool closed; // this result status: closed or opened - uint32_t numOfRows; // number of rows of current time window - STimeWindow win; + int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer + int32_t offset : 29; // row index in buffer page + bool startInterp; // the time window start timestamp has done the interpolation already. + bool endInterp; // the time window end timestamp has done the interpolation already. + bool closed; // this result status: closed or opened + uint32_t numOfRows; // number of rows of current time window + STimeWindow win; struct SResultRowEntryInfo pEntryInfo[]; // For each result column, there is a resultInfo } SResultRow; @@ -66,57 +65,58 @@ typedef struct SResultRowPosition { typedef struct SResKeyPos { SResultRowPosition pos; - uint64_t groupId; - char key[]; + uint64_t groupId; + char key[]; } SResKeyPos; typedef struct SResultRowInfo { - int32_t size; // number of result set + int32_t size; // number of result set SResultRowPosition cur; - SList* openWindow; + SList* openWindow; } SResultRowInfo; struct SqlFunctionCtx; -size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); -void initResultRowInfo(SResultRowInfo* pResultRowInfo); -void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); +size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); +void initResultRowInfo(SResultRowInfo* pResultRowInfo); +void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void closeAllResultRows(SResultRowInfo* pResultRowInfo); +void closeAllResultRows(SResultRowInfo* pResultRowInfo); -void initResultRow(SResultRow *pResultRow); -void closeResultRow(SResultRow* pResultRow); -bool isResultRowClosed(SResultRow* pResultRow); +void initResultRow(SResultRow* pResultRow); +void closeResultRow(SResultRow* pResultRow); +bool isResultRowClosed(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); -static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { - SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); +static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { + SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); return pRow; } -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); -void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); -void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); -bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); +void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); +bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); -SArray* createSortInfo(SNodeList* pNodeList); -SArray* extractPartitionColInfo(SNodeList* pNodeList); -SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type); +int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); +SArray* createSortInfo(SNodeList* pNodeList); +SArray* extractPartitionColInfo(SNodeList* pNodeList); +SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, + int32_t type); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); -void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); -void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); +void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 802f9ea5b5..5ee222efb7 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,14 +69,16 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // data format: // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... -// | | (4 bytes) |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | actual size | | +// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 +// length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// actual size | | // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { int32_t numOfCols = 0; - SNode* pNode; + SNode* pNode; FOREACH(pNode, pHandle->pSchema->pSlots) { SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; if (pSlotDesc->output) { @@ -90,12 +92,12 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - blockCompressEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); pBuf->useSize += pEntry->dataLen; - - atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); - atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + + atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); + atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); } static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { @@ -187,8 +189,8 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); @@ -198,7 +200,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->precision = pDispatcher->pSchema->precision; taosThreadMutexUnlock(&pDispatcher->mutex); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5c2f32126b..2561516f6a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -189,9 +189,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); - if (!pDescNode->output) { // todo disable it temporarily - continue; - } + /*if (!pDescNode->output) { // todo disable it temporarily*/ + /*continue;*/ + /*}*/ SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ad0d5c1447..706fc87520 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2048,7 +2048,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); // blockDataEnsureCapacity(pRes, numOfRows); - blockCompressDecode(pRes, numOfOutput, numOfRows, pData); + blockDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2072,7 +2072,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockDataAppendColInfo(pBlock, &idata); } - blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); + blockDecode(pBlock, numOfCols, numOfRows, pStart); blockDataEnsureCapacity(pRes, numOfRows); // data from mnode @@ -2934,7 +2934,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len *length = 0; return TSDB_CODE_SUCCESS; } - + *result = (char*)taosMemoryCalloc(1, totalSize); if (*result == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3898,60 +3898,59 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI return TSDB_CODE_SUCCESS; } -static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){ +static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum) { taosArrayClear(pTableListInfo->pGroupList); - SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); - if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; + SArray* sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); + if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t)); + uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t)); int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ); - if (index == -1){ - void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT); - SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo)); - if(tGroup == NULL) { + if (index == -1) { + void* p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT); + SArray* tGroup = taosArrayInit(8, sizeof(STableKeyInfo)); + if (tGroup == NULL) { taosArrayDestroy(sortSupport); return TSDB_CODE_OUT_OF_MEMORY; } - if(taosArrayPush(tGroup, info) == NULL){ + if (taosArrayPush(tGroup, info) == NULL) { qError("taos push info array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - if(p == NULL){ - if(taosArrayPush(sortSupport, groupId) != NULL){ + if (p == NULL) { + if (taosArrayPush(sortSupport, groupId) != NULL) { qError("taos push support array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){ + if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL) { qError("taos push group array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - }else{ + } else { int32_t pos = TARRAY_ELEM_IDX(sortSupport, p); - if(taosArrayInsert(sortSupport, pos, groupId) == NULL){ + if (taosArrayInsert(sortSupport, pos, groupId) == NULL) { qError("taos insert support array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){ + if (taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL) { qError("taos insert group array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } } - }else{ + } else { SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index); - if(taosArrayPush(tGroup, info) == NULL){ + if (taosArrayPush(tGroup, info) == NULL) { qError("taos push uid array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } } - } taosArrayDestroy(sortSupport); return TDB_CODE_SUCCESS; @@ -3969,9 +3968,9 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t keyLen = 0; void* keyBuf = NULL; - SNode* node; + SNode* node; FOREACH(node, group) { - SExprNode *pExpr = (SExprNode *)node; + SExprNode* pExpr = (SExprNode*)node; keyLen += pExpr->resType.bytes; } @@ -3990,15 +3989,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, metaReaderInit(&mr, pHandle->meta, 0); metaGetTableEntryByUid(&mr, info->uid); - SNodeList *groupNew = nodesCloneList(group); + SNodeList* groupNew = nodesCloneList(group); nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); char* isNull = (char*)keyBuf; char* pStart = (char*)keyBuf + nullFlagSize; - SNode* pNode; + SNode* pNode; int32_t index = 0; - FOREACH(pNode, groupNew){ + FOREACH(pNode, groupNew) { SNode* pNew = NULL; int32_t code = scalarCalculateConstants(pNode, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -4010,15 +4009,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); - SValueNode *pValue = (SValueNode *)pNew; + SValueNode* pValue = (SValueNode*)pNew; if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) { isNull[index++] = 1; continue; } else { isNull[index++] = 0; - char* data = nodesGetValueFromNode(pValue); - if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){ + char* data = nodesGetValueFromNode(pValue); + if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { int32_t len = getJsonValueLen(data); memcpy(pStart, data, len); pStart += len; @@ -4031,7 +4030,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); + int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t groupId = calcGroupId(keyBuf, len); taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); info->groupId = groupId; @@ -4042,7 +4041,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } taosMemoryFree(keyBuf); - if(pTableListInfo->needSortTableByGroupId){ + if (pTableListInfo->needSortTableByGroupId) { return sortTableGroup(pTableListInfo, groupNum); } @@ -4050,7 +4049,8 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) { + uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, + const char* pUser) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4058,7 +4058,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - if(code){ + if (code) { pTaskInfo->code = code; return NULL; } @@ -4076,7 +4076,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - if(code){ + if (code) { return NULL; } code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); @@ -4085,7 +4085,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + SOperatorInfo* pOperator = + createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; @@ -4105,7 +4106,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); } - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); + SOperatorInfo* pOperator = + createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4603,8 +4605,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->sql = sql; (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; - (*pTaskInfo)->pRoot = - createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, + &(*pTaskInfo)->tableqinfoList, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; @@ -4622,8 +4624,8 @@ _complete: static void doDestroyTableList(STableListInfo* pTableqinfoList) { taosArrayDestroy(pTableqinfoList->pTableList); taosHashCleanup(pTableqinfoList->map); - if(pTableqinfoList->needSortTableByGroupId){ - for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){ + if (pTableqinfoList->needSortTableByGroupId) { + for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) { SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i); taosArrayDestroy(tmp); } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 3efe3e6c9c..e9b311db9a 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -30,7 +30,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock /*int32_t len = *(int32_t*)taosArrayGet(pReq->dataLen, i);*/ SRetrieveTableRsp* pRetrieve = taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); - blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -50,7 +50,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock taosArraySetSize(pArray, 1); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); - blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index a5e9b8edd9..8034840fce 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -108,7 +108,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); SStreamRetrieveReq req = { .streamId = pTask->streamId, @@ -181,7 +181,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From 5e31c9e8fe664af8be2c4292a65dad9380802e8a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 29 Jun 2022 17:50:33 +0800 Subject: [PATCH 3/7] refactor(query): function adoption to us/ns db precision TD-16436 --- source/libs/function/src/builtins.c | 4 ++++ source/libs/scalar/src/sclfunc.c | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6fa7d10ca3..440e12c3d7 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1460,6 +1460,10 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + //add database precision as param + uint8_t dbPrec = pFunc->node.resType.precision; + addDbPrecisonParam(pFunc->pParameterList, dbPrec); + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index b7c171db2e..dc4bf33a2b 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1116,7 +1116,8 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); - int32_t timePrec = GET_PARAM_PRECISON(pInput); + int64_t timePrec; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { From c2955a807b6c60bd6a6c105ca0477dcddd952971 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 29 Jun 2022 20:57:15 +0800 Subject: [PATCH 4/7] feat(tmq): offset support snapshot --- examples/c/tmq.c | 4 +- include/common/tcommon.h | 17 +- include/common/tmsg.h | 70 ++- include/libs/executor/executor.h | 7 - include/libs/stream/tstream.h | 19 +- source/client/inc/clientInt.h | 25 +- source/client/src/clientMain.c | 10 +- source/client/src/tmq.c | 414 ++++++++---------- source/common/src/tmsg.c | 168 +++++-- source/dnode/vnode/inc/vnode.h | 20 +- source/dnode/vnode/src/inc/tq.h | 19 +- source/dnode/vnode/src/sma/smaRollup.c | 14 +- source/dnode/vnode/src/tq/tq.c | 299 +++++++++++-- source/dnode/vnode/src/tq/tqExec.c | 35 +- source/dnode/vnode/src/tq/tqOffset.c | 4 +- source/dnode/vnode/src/tq/tqPush.c | 12 +- source/dnode/vnode/src/tq/tqRead.c | 29 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/executor/src/executor.c | 10 +- source/libs/executor/src/scanoperator.c | 12 +- source/libs/executor/src/timewindowoperator.c | 26 +- source/libs/function/src/builtins.c | 25 +- source/libs/stream/src/streamExec.c | 19 +- source/libs/stream/src/streamTask.c | 8 +- 24 files changed, 810 insertions(+), 458 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 68b0492ace..4226587d56 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -138,7 +138,7 @@ int32_t create_topic() { taos_free_result(pRes); /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1 where t1 = 2000"); + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -225,7 +225,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } int32_t cnt = 0; while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, -1); if (tmqmessage) { cnt++; msg_process(tmqmessage); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 928fe0aa0e..54044f7d89 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -25,10 +25,11 @@ extern "C" { #endif +// TODO remove it enum { - TMQ_CONF__RESET_OFFSET__LATEST = -1, - TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, TMQ_CONF__RESET_OFFSET__NONE = -3, + TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, + TMQ_CONF__RESET_OFFSET__LATEST = -1, }; enum { @@ -39,6 +40,16 @@ enum { TMQ_MSG_TYPE__END_RSP, }; +enum { + STREAM_INPUT__DATA_SUBMIT = 1, + STREAM_INPUT__DATA_BLOCK, + STREAM_INPUT__DATA_SCAN, + STREAM_INPUT__DATA_RETRIEVE, + STREAM_INPUT__TRIGGER, + STREAM_INPUT__CHECKPOINT, + STREAM_INPUT__DROP, +}; + typedef enum EStreamType { STREAM_NORMAL = 1, STREAM_INVERT, @@ -48,7 +59,7 @@ typedef enum EStreamType { STREAM_DELETE, STREAM_RETRIEVE, STREAM_PUSH_DATA, - STREAM_PUSH_EMPTY, + STREAM_PUSH_OVER, } EStreamType; typedef struct { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0afcaaa50d..71867d9741 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2462,22 +2462,37 @@ int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pRe // tqOffset enum { - TMQ_OFFSET__SNAPSHOT = 1, - TMQ_OFFSET__LOG, + TMQ_OFFSET__RESET_NONE = -3, + TMQ_OFFSET__RESET_EARLIEAST = -2, + TMQ_OFFSET__RESET_LATEST = -1, + TMQ_OFFSET__LOG = 1, + TMQ_OFFSET__SNAPSHOT_DATA = 2, + TMQ_OFFSET__SNAPSHOT_META = 3, }; typedef struct { int8_t type; union { + // snapshot data struct { int64_t uid; int64_t ts; }; + // log struct { int64_t version; }; }; - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; +} STqOffsetVal; + +int32_t tEncodeSTqOffsetVal(SEncoder* pEncoder, const STqOffsetVal* pOffsetVal); +int32_t tDecodeSTqOffsetVal(SDecoder* pDecoder, STqOffsetVal* pOffsetVal); +int32_t tFormatOffset(char* buf, int32_t maxLen, const STqOffsetVal* pVal); +bool tOffsetEqual(const STqOffsetVal* pLeft, const STqOffsetVal* pRight); + +typedef struct { + STqOffsetVal val; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; } STqOffset; int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); @@ -2710,7 +2725,8 @@ typedef struct { uint64_t reqId; int64_t consumerId; int64_t timeout; - int64_t currentOffset; + // int64_t currentOffset; + STqOffsetVal reqOffset; } SMqPollReq; typedef struct { @@ -2779,12 +2795,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { } typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int16_t resMsgType; - int32_t metaRspLen; - void* metaRsp; + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + STqOffsetVal reqOffsetNew; + STqOffsetVal rspOffsetNew; + int16_t resMsgType; + int32_t metaRspLen; + void* metaRsp; } SMqMetaRsp; static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) { @@ -2806,6 +2824,24 @@ static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) { return (void*)buf; } +typedef struct { + SMqRspHead head; + STqOffsetVal reqOffset; + STqOffsetVal rspOffset; + int32_t skipLogNum; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + SArray* blockDataLen; + SArray* blockData; + SArray* blockTbName; + SArray* blockSchema; +} SMqDataRsp; + +int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); +int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); + +#if 0 typedef struct { SMqRspHead head; int64_t reqOffset; @@ -2814,13 +2850,10 @@ typedef struct { int32_t blockNum; int8_t withTbName; int8_t withSchema; - int8_t withTag; - SArray* blockDataLen; // SArray - SArray* blockData; // SArray - SArray* blockTbName; // SArray - SArray* blockSchema; // SArray - SArray* blockTags; // SArray - SArray* blockTagSchema; // SArray + SArray* blockDataLen; // SArray + SArray* blockData; // SArray + SArray* blockTbName; // SArray + SArray* blockSchema; // SArray } SMqDataBlkRsp; static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { @@ -2832,7 +2865,6 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp if (pRsp->blockNum != 0) { tlen += taosEncodeFixedI8(buf, pRsp->withTbName); tlen += taosEncodeFixedI8(buf, pRsp->withSchema); - tlen += taosEncodeFixedI8(buf, pRsp->withTag); for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); @@ -2862,7 +2894,6 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); buf = taosDecodeFixedI8(buf, &pRsp->withTbName); buf = taosDecodeFixedI8(buf, &pRsp->withSchema); - buf = taosDecodeFixedI8(buf, &pRsp->withTag); if (pRsp->withTbName) { pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); } @@ -2891,6 +2922,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p } return (void*)buf; } +#endif typedef struct { SMqRspHead head; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 00acc4741d..564c208c69 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -36,15 +36,8 @@ typedef struct SReadHandle { void* vnode; void* mnd; SMsgCb* pMsgCb; -// int8_t initTsdbReader; } SReadHandle; -enum { - STREAM_DATA_TYPE_SUBMIT_BLOCK = 1, - STREAM_DATA_TYPE_SSDATA_BLOCK = 2, - STREAM_DATA_TYPE_FROM_SNAPSHOT = 3, -}; - typedef enum { OPTR_EXEC_MODEL_BATCH = 0x1, OPTR_EXEC_MODEL_STREAM = 0x2, diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 74c885ebcc..67074c789e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -55,15 +55,6 @@ enum { TASK_OUTPUT_STATUS__BLOCKED, }; -enum { - STREAM_INPUT__DATA_SUBMIT = 1, - STREAM_INPUT__DATA_BLOCK, - STREAM_INPUT__DATA_RETRIEVE, - STREAM_INPUT__TRIGGER, - STREAM_INPUT__CHECKPOINT, - STREAM_INPUT__DROP, -}; - typedef struct { int8_t type; } SStreamQueueItem; @@ -152,10 +143,6 @@ typedef struct { void* executor; } STaskExec; -typedef struct { - int32_t taskId; -} STaskDispatcherInplace; - typedef struct { int32_t taskId; int32_t nodeId; @@ -208,7 +195,6 @@ enum { enum { TASK_DISPATCH__NONE = 1, - TASK_DISPATCH__INPLACE, TASK_DISPATCH__FIXED, TASK_DISPATCH__SHUFFLE, }; @@ -260,7 +246,7 @@ struct SStreamTask { // exec STaskExec exec; - // TODO: merge sink and dispatch + // TODO: unify sink and dispatch // local sink union { @@ -269,9 +255,8 @@ struct SStreamTask { STaskSinkFetch fetchSink; }; - // dispatch + // remote dispatcher union { - STaskDispatcherInplace inplaceDispatcher; STaskDispatcherFixedEp fixedEpDispatcher; STaskDispatcherShuffle shuffleDispatcher; }; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cca79186d0..5ffc9ecae8 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -54,11 +54,10 @@ enum { RES_TYPE__TMQ_META, }; -#define SHOW_VARIABLES_RESULT_COLS 2 +#define SHOW_VARIABLES_RESULT_COLS 2 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) - #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) #define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) @@ -194,7 +193,7 @@ typedef struct { int32_t vgId; SSchemaWrapper schema; int32_t resIter; - SMqDataBlkRsp rsp; + SMqDataRsp rsp; SReqResultInfo resInfo; } SMqRspObj; @@ -238,18 +237,18 @@ typedef struct SSyncQueryParam { void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); -void doSetOneRowPtr(SReqResultInfo* pResultInfo); -void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, - bool freeAfterUse); -void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); -void doFreeReqResultInfo(SReqResultInfo* pResInfo); -int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); -void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); +void doSetOneRowPtr(SReqResultInfo* pResultInfo); +void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, + bool freeAfterUse); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +void doFreeReqResultInfo(SReqResultInfo* pResInfo); +int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); +void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); -TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly); -void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly); +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); +void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 52574dcc9f..90df3eb31a 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -81,16 +81,16 @@ void taos_cleanup(void) { taosCloseLog(); } -static setConfRet taos_set_config_imp(const char *config){ +static setConfRet taos_set_config_imp(const char *config) { setConfRet ret = {SET_CONF_RET_SUCC, {0}}; // TODO: need re-implementation return ret; } -setConfRet taos_set_config(const char *config){ -// TODO pthread_mutex_lock(&setConfMutex); +setConfRet taos_set_config(const char *config) { + // TODO pthread_mutex_lock(&setConfMutex); setConfRet ret = taos_set_config_imp(config); -// pthread_mutex_unlock(&setConfMutex); + // pthread_mutex_unlock(&setConfMutex); return ret; } @@ -179,8 +179,6 @@ void taos_free_result(TAOS_RES *res) { SMqRspObj *pRsp = (SMqRspObj *)res; if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); - if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags); - if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema); if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 637a7ee5dd..4cef25b9ce 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -126,8 +126,10 @@ typedef struct { // statistics int64_t pollCnt; // offset - int64_t committedOffset; - int64_t currentOffset; + /*int64_t committedOffset;*/ + /*int64_t currentOffset;*/ + STqOffsetVal committedOffsetNew; + STqOffsetVal currentOffsetNew; // connection info int32_t vgId; int32_t vgStatus; @@ -152,8 +154,8 @@ typedef struct { SMqClientVg* vgHandle; SMqClientTopic* topicHandle; union { - SMqDataBlkRsp dataRsp; - SMqMetaRsp metaRsp; + SMqDataRsp dataRsp; + SMqMetaRsp metaRsp; }; } SMqPollRspWrapper; @@ -179,6 +181,7 @@ typedef struct { tsem_t rspSem; } SMqPollCbParam; +#if 0 typedef struct { tmq_t* tmq; int8_t async; @@ -190,12 +193,13 @@ typedef struct { SArray* offsets; void* userParam; } SMqCommitCbParam; +#endif typedef struct { - tmq_t* tmq; - int8_t automatic; - int8_t async; - int8_t freeOffsets; + tmq_t* tmq; + int8_t automatic; + int8_t async; + /*int8_t freeOffsets;*/ int32_t waitingRspNum; int32_t totalRspNum; int32_t rspErr; @@ -351,6 +355,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } +#if 0 int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; pParam->rspErr = code; @@ -371,6 +376,7 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { } return 0; } +#endif int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; @@ -413,139 +419,148 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { return 0; } +static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pTopic, SMqCommitCbParamSet* pParamSet) { + STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); + if (pOffset == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pOffset->val = pVg->currentOffsetNew; + + int32_t groupLen = strlen(tmq->groupId); + memcpy(pOffset->subKey, tmq->groupId, groupLen); + pOffset->subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSTqOffset, pOffset, len, code); + if (code < 0) { + ASSERT(0); + return -1; + } + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + if (buf == NULL) return -1; + ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSTqOffset(&encoder, pOffset); + + // build param + SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + pParam->params = pParamSet; + pParam->pOffset = pOffset; + + // build send info + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pMsgSendInfo == NULL) { + return -1; + } + pMsgSendInfo->msgInfo = (SDataBuf){ + .pData = buf, + .len = sizeof(SMsgHead) + len, + .handle = NULL, + }; + + tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId, + pOffset->val.version); + + // TODO: put into cb + pVg->committedOffsetNew = pVg->currentOffsetNew; + + pMsgSendInfo->requestId = generateRequestId(); + pMsgSendInfo->requestObjRefId = 0; + pMsgSendInfo->param = pParam; + pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; + // send msg + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); + pParamSet->waitingRspNum++; + pParamSet->totalRspNum++; + return 0; +} + +int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) { + char* topic; + int32_t vgId; + ASSERT(msg != NULL); + if (TD_RES_TMQ(msg)) { + SMqRspObj* pRspObj = (SMqRspObj*)msg; + topic = pRspObj->topic; + vgId = pRspObj->vgId; + } else if (TD_RES_TMQ_META(msg)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; + topic = pMetaRspObj->topic; + vgId = pMetaRspObj->vgId; + } else { + return TSDB_CODE_TMQ_INVALID_MSG; + } + + SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); + if (pParamSet == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pParamSet->tmq = tmq; + pParamSet->automatic = 0; + pParamSet->async = async; + /*pParamSet->freeOffsets = 1;*/ + pParamSet->userCb = userCb; + pParamSet->userParam = userParam; + tsem_init(&pParamSet->rspSem, 0, 0); + + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(pTopic->topicName, topic) != 0) continue; + for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg->vgId != vgId) continue; + + if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + goto FAIL; + } + goto HANDLE_RSP; + } + } + } + + int32_t code = -1; + +HANDLE_RSP: + if (pParamSet->totalRspNum == 0) { + tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); + return 0; + } + + if (!async) { + tsem_wait(&pParamSet->rspSem); + code = pParamSet->rspErr; + tsem_destroy(&pParamSet->rspSem); + return code; + } else { + code = 0; + } + +FAIL: + if (code != 0 && async) { + userCb(tmq, code, userParam); + } + return 0; +} + int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { int32_t code = -1; if (msg != NULL) { - char* topic; - int32_t vgId; - if (TD_RES_TMQ(msg)) { - SMqRspObj* pRspObj = (SMqRspObj*)msg; - topic = pRspObj->topic; - vgId = pRspObj->vgId; - } else if (TD_RES_TMQ_META(msg)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; - topic = pMetaRspObj->topic; - vgId = pMetaRspObj->vgId; - } else { - return TSDB_CODE_TMQ_INVALID_MSG; - } - - SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); - if (pParamSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pParamSet->tmq = tmq; - pParamSet->automatic = automatic; - pParamSet->async = async; - pParamSet->freeOffsets = 1; - pParamSet->userCb = userCb; - pParamSet->userParam = userParam; - tsem_init(&pParamSet->rspSem, 0, 0); - - for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopic->topicName, topic) == 0) { - for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->vgId == vgId) { - if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { - tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); - - return 0; - } - - STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); - if (pOffset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pOffset->type = TMQ_OFFSET__LOG; - pOffset->version = pVg->currentOffset; - - int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->subKey, tmq->groupId, groupLen); - pOffset->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); - - int32_t len; - int32_t code; - tEncodeSize(tEncodeSTqOffset, pOffset, len, code); - if (code < 0) { - ASSERT(0); - } - void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); - - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - - SEncoder encoder; - tEncoderInit(&encoder, abuf, len); - tEncodeSTqOffset(&encoder, pOffset); - - // build param - SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); - pParam->params = pParamSet; - pParam->pOffset = pOffset; - - // build send info - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (pMsgSendInfo == NULL) { - // TODO - continue; - } - pMsgSendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = sizeof(SMsgHead) + len, - .handle = NULL, - }; - - tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, - pVg->vgId, pOffset->version); - - // TODO: put into cb - pVg->committedOffset = pVg->currentOffset; - - pMsgSendInfo->requestId = generateRequestId(); - pMsgSendInfo->requestObjRefId = 0; - pMsgSendInfo->param = pParam; - pMsgSendInfo->fp = tmqCommitCb2; - pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; - // send msg - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); - pParamSet->waitingRspNum++; - pParamSet->totalRspNum++; - } - } - } - } - if (pParamSet->totalRspNum == 0) { - tsem_destroy(&pParamSet->rspSem); - taosMemoryFree(pParamSet); - return 0; - } - - if (!async) { - tsem_wait(&pParamSet->rspSem); - code = pParamSet->rspErr; - tsem_destroy(&pParamSet->rspSem); - } else { - code = 0; - } - - if (code != 0 && async) { - if (automatic) { - tmq->commitCb(tmq, code, tmq->commitCbUserParam); - } else { - userCb(tmq, code, userParam); - } - } - return 0; + return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); } SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); @@ -556,7 +571,7 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ pParamSet->tmq = tmq; pParamSet->automatic = automatic; pParamSet->async = async; - pParamSet->freeOffsets = 1; + /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -572,75 +587,11 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ tscDebug("consumer %ld begin commit for topic %s, vgId %d", tmq->consumerId, pTopic->topicName, pVg->vgId); - /*if (pVg->currentOffset < 0) {*/ - if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { - tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); - - continue; + if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + continue; + } } - STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); - if (pOffset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pOffset->type = TMQ_OFFSET__LOG; - pOffset->version = pVg->currentOffset; - - int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->subKey, tmq->groupId, groupLen); - pOffset->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); - - int32_t len; - int32_t code; - tEncodeSize(tEncodeSTqOffset, pOffset, len, code); - if (code < 0) { - ASSERT(0); - } - void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); - - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - - SEncoder encoder; - tEncoderInit(&encoder, abuf, len); - tEncodeSTqOffset(&encoder, pOffset); - - // build param - SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); - pParam->params = pParamSet; - pParam->pOffset = pOffset; - - // build send info - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (pMsgSendInfo == NULL) { - // TODO - continue; - } - pMsgSendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = sizeof(SMsgHead) + len, - .handle = NULL, - }; - - tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId, - pOffset->version); - - // TODO: put into cb - pVg->committedOffset = pVg->currentOffset; - - pMsgSendInfo->requestId = generateRequestId(); - pMsgSendInfo->requestObjRefId = 0; - pMsgSendInfo->param = pParam; - pMsgSendInfo->fp = tmqCommitCb2; - pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; - // send msg - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); - pParamSet->waitingRspNum++; - pParamSet->totalRspNum++; } } @@ -1174,7 +1125,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if (rspType == TMQ_MSG_TYPE__POLL_RSP) { memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp); + SDecoder decoder; + tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); + tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp); + /*tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);*/ } else { ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP); memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); @@ -1226,9 +1180,11 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); - tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %ld", tmq->consumerId, epoch, pVgCur->vgId, vgKey, - pVgCur->currentOffset); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); + char buf[50]; + tFormatOffset(buf, 50, &pVgCur->currentOffsetNew); + tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, + buf); + taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); } } } @@ -1257,7 +1213,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { pVgEp->vgId, offset, vgKey); SMqClientVg clientVg = { .pollCnt = 0, - .currentOffset = offset, + .currentOffsetNew.type = tmq->resetOffsetCfg, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, @@ -1281,7 +1237,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } -#if 1 +#if 0 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ bool set = false; @@ -1516,16 +1472,16 @@ int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { #endif SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - int64_t reqOffset; - if (pVg->currentOffset >= 0) { - reqOffset = pVg->currentOffset; - } else { - /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ - /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ - /*return NULL;*/ - /*}*/ - reqOffset = tmq->resetOffsetCfg; - } + /*int64_t reqOffset;*/ + /*if (pVg->currentOffset >= 0) {*/ + /*reqOffset = pVg->currentOffset;*/ + /*} else {*/ + /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ + /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ + /*return NULL;*/ + /*}*/ + /*reqOffset = tmq->resetOffsetCfg;*/ + /*}*/ SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); if (pReq == NULL) { @@ -1544,7 +1500,8 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->timeout = timeout; pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; - pReq->currentOffset = reqOffset; + /*pReq->currentOffset = reqOffset;*/ + pReq->reqOffset = pVg->currentOffsetNew; pReq->reqId = generateRequestId(); pReq->useSnapshot = tmq->useSnapshot; @@ -1572,7 +1529,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; - memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataBlkRsp)); + memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; @@ -1645,8 +1602,11 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int64_t transporterId = 0; /*printf("send poll\n");*/ - tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, - pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); + + char offsetFormatBuf[50]; + tFormatOffset(offsetFormatBuf, 50, &pVg->currentOffsetNew); + tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId, + pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; @@ -1695,7 +1655,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; + pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { taosFreeQitem(pollRspWrapper); @@ -1717,7 +1677,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; + pVg->currentOffsetNew = pollRspWrapper->metaRsp.rspOffsetNew; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e9b5c67d76..8c1daae3fa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2300,7 +2300,6 @@ int32_t tDeserializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp return 0; } - int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2387,7 +2386,6 @@ int32_t tDeserializeSDnodeListRsp(void *buf, int32_t bufLen, SDnodeListRsp *pRsp void tFreeSDnodeListRsp(SDnodeListRsp *pRsp) { taosArrayDestroy(pRsp->dnodeList); } - int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2909,20 +2907,19 @@ int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesR return 0; } -int32_t tEncodeSVariablesInfo(SEncoder* pEncoder, SVariablesInfo* pInfo) { +int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; if (tEncodeCStr(pEncoder, pInfo->value) < 0) return -1; return 0; } -int32_t tDecodeSVariablesInfo(SDecoder* pDecoder, SVariablesInfo* pInfo) { +int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { if (tDecodeCStrTo(pDecoder, pInfo->name) < 0) return -1; if (tDecodeCStrTo(pDecoder, pInfo->value) < 0) return -1; return 0; } - -int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) { +int32_t tSerializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2930,7 +2927,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp int32_t varNum = taosArrayGetSize(pRsp->variables); if (tEncodeI32(&encoder, varNum) < 0) return -1; for (int32_t i = 0; i < varNum; ++i) { - SVariablesInfo* pInfo = taosArrayGet(pRsp->variables, i); + SVariablesInfo *pInfo = taosArrayGet(pRsp->variables, i); if (tEncodeSVariablesInfo(&encoder, pInfo) < 0) return -1; } tEndEncode(&encoder); @@ -2940,7 +2937,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp return tlen; } -int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) { +int32_t tDeserializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2962,11 +2959,11 @@ int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesR return 0; } -void tFreeSShowVariablesRsp(SShowVariablesRsp* pRsp) { +void tFreeSShowVariablesRsp(SShowVariablesRsp *pRsp) { if (NULL == pRsp) { return; } - + taosArrayDestroy(pRsp->variables); } @@ -5360,30 +5357,149 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { } } -int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { - if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; - if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { - if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; - if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1; - } else if (pOffset->type == TMQ_OFFSET__LOG) { - if (tEncodeI64(pEncoder, pOffset->version) < 0) return -1; +int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal) { + if (tEncodeI8(pEncoder, pOffsetVal->type) < 0) return -1; + if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (tEncodeI64(pEncoder, pOffsetVal->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pOffsetVal->ts) < 0) return -1; + } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { + if (tEncodeI64(pEncoder, pOffsetVal->version) < 0) return -1; + } else if (pOffsetVal->type < 0) { + // do nothing } else { ASSERT(0); } + return 0; +} + +int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) { + if (tDecodeI8(pDecoder, &pOffsetVal->type) < 0) return -1; + if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (tDecodeI64(pDecoder, &pOffsetVal->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffsetVal->ts) < 0) return -1; + } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { + if (tDecodeI64(pDecoder, &pOffsetVal->version) < 0) return -1; + } else if (pOffsetVal->type < 0) { + // do nothing + } else { + ASSERT(0); + } + return 0; +} + +#if 1 +int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { + if (pVal->type == TMQ_OFFSET__RESET_NONE) { + snprintf(buf, maxLen, "offset(reset to none)"); + } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEAST) { + snprintf(buf, maxLen, "offset(reset to earlieast)"); + } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) { + snprintf(buf, maxLen, "offset(reset to latest)"); + } else if (pVal->type == TMQ_OFFSET__LOG) { + snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version); + } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { + snprintf(buf, maxLen, "offset(snapshot data) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) { + snprintf(buf, maxLen, "offset(snapshot meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + } else { + ASSERT(0); + } + return 0; +} +#endif + +bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) { + if (pLeft->type == pRight->type) { + if (pLeft->type == TMQ_OFFSET__LOG) { + return pLeft->version == pRight->version; + } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_DATA) { + return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts; + } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_META) { + ASSERT(0); + // TODO + return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts; + } + } + return false; +} + +int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { + if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1; if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1; return 0; } int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { - if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1; - if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { - if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1; - if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1; - } else if (pOffset->type == TMQ_OFFSET__LOG) { - if (tDecodeI64(pDecoder, &pOffset->version) < 0) return -1; - } else { - ASSERT(0); - } + if (tDecodeSTqOffsetVal(pDecoder, &pOffset->val) < 0) return -1; if (tDecodeCStrTo(pDecoder, pOffset->subKey) < 0) return -1; return 0; } + +int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { + if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; + if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->skipLogNum) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; + if (pRsp->blockNum != 0) { + if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->withSchema) < 0) return -1; + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); + void *data = taosArrayGetP(pRsp->blockData, i); + if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; + if (pRsp->withSchema) { + SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); + if (tEncodeSSchemaWrapper(pEncoder, pSW) < 0) return -1; + } + if (pRsp->withTbName) { + char *tbName = (char *)taosArrayGetP(pRsp->blockTbName, i); + if (tEncodeCStr(pEncoder, tbName) < 0) return -1; + } + } + } + return 0; +} + +int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { + if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; + if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->skipLogNum) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; + if (pRsp->blockNum != 0) { + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); + if (tDecodeI8(pDecoder, &pRsp->withTbName) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->withSchema) < 0) return -1; + if (pRsp->withTbName) { + pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void *)); + } + if (pRsp->withSchema) { + pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void *)); + } + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + void *data; + uint64_t bLen; + if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; + taosArrayPush(pRsp->blockData, &data); + int32_t len = bLen; + taosArrayPush(pRsp->blockDataLen, &len); + + if (pRsp->withSchema) { + SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pSW == NULL) return -1; + if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) return -1; + taosArrayPush(pRsp->blockSchema, &pSW); + } + + if (pRsp->withTbName) { + char *tbName; + if (tDecodeCStrAlloc(pDecoder, &tbName) < 0) return -1; + taosArrayPush(pRsp->blockTbName, &tbName); + } + } + } + return 0; +} + diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 5c2d2cd712..cef0bbbc01 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -139,19 +139,19 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle); // tq -typedef struct STqReadHandle STqReadHandle; +typedef struct STqReadHandle SStreamReader; -STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); +SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta); -void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList); -int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); -int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); -int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); +void tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList); +int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList); +int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList); +int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList); -int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReadHandle *pHandle); -bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle); +int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock(SStreamReader *pHandle); +bool tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 59c3e95b9c..38a4f28041 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -39,6 +39,16 @@ extern "C" { #define tqInfo(...) do { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) #define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) + +#define IS_META_MSG(x) ( \ + x == TDMT_VND_CREATE_STB \ + || x == TDMT_VND_ALTER_STB \ + || x == TDMT_VND_DROP_STB \ + || x == TDMT_VND_CREATE_TABLE \ + || x == TDMT_VND_ALTER_TABLE \ + || x == TDMT_VND_DROP_TABLE \ + || x == TDMT_VND_DROP_TTL_TABLE \ +) // clang-format on typedef struct STqOffsetStore STqOffsetStore; @@ -101,12 +111,13 @@ typedef struct { typedef struct { int8_t subType; - STqReadHandle* pExecReader[5]; + SStreamReader* pExecReader[5]; union { STqExecCol execCol; STqExecTb execTb; STqExecDb execDb; }; + } STqExecHandle; typedef struct { @@ -149,9 +160,9 @@ static STqMgmt tqMgmt = {0}; int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** pHeadWithCkSum); // tqExec -int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId); -int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId); -int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp); +int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId); +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, int32_t workerId); +int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); // tqMeta int32_t tqMetaOpen(STQ* pTq); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ed5b6f4055..e1e195f14c 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -306,7 +306,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_FAILED; } - STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta); + SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta); if (!pReadHandle) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -590,8 +590,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { tdRefSmaStat(pSma, (SSmaStat *)pStat); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); - tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); + qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); + tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK); tdUnRefSmaStat(pSma, (SSmaStat *)pStat); @@ -611,12 +611,12 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, pItem->taskInfo, suid); - if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // STREAM_DATA_TYPE_SUBMIT_BLOCK + if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // INPUT__DATA_SUBMIT smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } - tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); + tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma)); @@ -654,7 +654,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb return TSDB_CODE_SUCCESS; } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (inputType == STREAM_INPUT__DATA_SUBMIT) { tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1); tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2); } @@ -675,7 +675,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (inputType == STREAM_INPUT__DATA_SUBMIT) { STbUidStore uidStore = {0}; tdFetchSubmitReqSuids(pMsg, &uidStore); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 766cf7af35..f36998d606 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -113,8 +113,23 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, return 0; } -int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) { - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp); +int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) { + ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); + + if (pRsp->withSchema) { + ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); + } else { + ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + } + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code); + if (code < 0) { + return -1; + } + int32_t tlen = sizeof(SMqRspHead) + len; void* buf = rpcMallocCont(tlen); if (buf == NULL) { return -1; @@ -125,18 +140,26 @@ int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con ((SMqRspHead*)buf)->consumerId = pReq->consumerId; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqDataBlkRsp(&abuf, pRsp); - SRpcMsg resp = { + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + tEncodeSMqDataRsp(&encoder, pRsp); + /*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/ + + SRpcMsg rsp = { .info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0, }; - tmsgSendRsp(&resp); + tmsgSendRsp(&rsp); - tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, pRsp->reqOffset, pRsp->rspOffset); + char buf1[50]; + char buf2[50]; + tFormatOffset(buf1, 50, &pRsp->reqOffset); + tFormatOffset(buf2, 50, &pRsp->rspOffset); + tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s", + TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2); return 0; } @@ -151,17 +174,17 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { } tDecoderClear(&decoder); - if (offset.type == TMQ_OFFSET__SNAPSHOT) { + if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) { tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey, - TD_VID(pTq->pVnode), offset.uid, offset.ts); - } else if (offset.type == TMQ_OFFSET__LOG) { + TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); + } else if (offset.val.type == TMQ_OFFSET__LOG) { tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey, - TD_VID(pTq->pVnode), offset.version); + TD_VID(pTq->pVnode), offset.val.version); } else { ASSERT(0); } STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); - if (pOffset == NULL || pOffset->version < offset.version) { + if (pOffset == NULL || pOffset->val.version < offset.val.version) { if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { ASSERT(0); return -1; @@ -171,6 +194,237 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { return 0; } +static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { + return -1; + } + + pRsp->withTbName = pReq->withTbName; + if (pRsp->withTbName) { + pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); + if (pRsp->blockTbName == NULL) { + // TODO free + return -1; + } + } + + if (subType == TOPIC_SUB_TYPE__COLUMN) { + pRsp->withSchema = false; + } else { + pRsp->withSchema = true; + pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); + if (pRsp->blockSchema == NULL) { + // TODO free + return -1; + } + } + return 0; +} + +static int32_t tqInitMetaRsp(SMqMetaRsp* pRsp, const SMqPollReq* pReq) { return 0; } + +static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t uid, int64_t ts) { + pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA; + pOffsetVal->uid = uid; + pOffsetVal->ts = ts; +} + +static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ver) { + pOffsetVal->type = TMQ_OFFSET__LOG; + pOffsetVal->version = ver; +} + +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { + SMqPollReq* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int64_t timeout = pReq->timeout; + int32_t reqEpoch = pReq->epoch; + int32_t code = 0; + STqOffsetVal reqOffset = pReq->reqOffset; + STqOffsetVal fetchOffsetNew; + + // 1.find handle + char buf[50]; + tFormatOffset(buf, 50, &reqOffset); + tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), buf); + + STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); + /*ASSERT(pHandle);*/ + if (pHandle == NULL) { + tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode), + pReq->subKey); + return -1; + } + + // check rebalance + if (pHandle->consumerId != consumerId) { + tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld", + consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + return -1; + } + + // update epoch if need + int32_t consumerEpoch = atomic_load_32(&pHandle->epoch); + while (consumerEpoch < reqEpoch) { + consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch); + } + + // 2.reset offset if needed + if (reqOffset.type > 0) { + fetchOffsetNew = reqOffset; + } else { + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); + if (pOffset != NULL) { + fetchOffsetNew = pOffset->val; + char formatBuf[50]; + tFormatOffset(formatBuf, 50, &fetchOffsetNew); + tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf); + } else { + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + if (pReq->useSnapshot) { + if (!pHandle->fetchMeta) { + tqOffsetResetToData(&fetchOffsetNew, 0, 0); + } else { + // reset to meta + ASSERT(0); + } + } else { + tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal)); + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal)); + } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { + tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId, + TD_VID(pTq->pVnode), pReq->subKey); + terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; + return -1; + } + } + } + + // 3.query + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + + if (fetchOffsetNew.type == TMQ_OFFSET__LOG) { + int64_t fetchVer = fetchOffsetNew.version + 1; + SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); + if (pHeadWithCkSum == NULL) { + return -1; + } + + walSetReaderCapacity(pHandle->pWalReader, 2048); + + while (1) { + consumerEpoch = atomic_load_32(&pHandle->epoch); + if (consumerEpoch > reqEpoch) { + tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d", + consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); + break; + } + + if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) { + // TODO add push mgr + + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + goto OVER; + } + + SWalReadHead* pHead = &pHeadWithCkSum->head; + + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchVer, pHead->msgType); + + if (pHead->msgType == TDMT_VND_SUBMIT) { + SSubmitReq* pCont = (SSubmitReq*)&pHead->body; + + if (tqLogScanExec(pTq, &pHandle->execHandle, pCont, &dataRsp, workerId) < 0) { + ASSERT(0); + } + // TODO batch optimization: + // TODO continue scan until meeting batch requirement + if (dataRsp.blockNum > 0 /* threshold */) { + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + + } else { + fetchVer++; + } + goto OVER; + + } else { + ASSERT(pHandle->fetchMeta); + ASSERT(IS_META_MSG(pHead->msgType)); + tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); + SMqMetaRsp metaRsp = {0}; + metaRsp.reqOffset = pReq->reqOffset.version; + /*tqOffsetResetToLog(&metaR)*/ + metaRsp.rspOffset = fetchVer; + metaRsp.resMsgType = pHead->msgType; + metaRsp.metaRspLen = pHead->bodyLen; + metaRsp.metaRsp = pHead->body; + if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { + code = -1; + goto OVER; + } + code = 0; + goto OVER; + } + } + + taosMemoryFree(pHeadWithCkSum); + } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) { + // 1. set uid and ts + // 2. get data (rebuild reader if needed) + // 3. get new uid and ts + + char formatBuf[50]; + tFormatOffset(formatBuf, 50, &dataRsp.reqOffset); + tqInfo("retrieve using snapshot req offset %s", formatBuf); + if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, workerId) < 0) { + ASSERT(0); + } + + // 4. send rsp + if (dataRsp.blockNum != 0) { + tqOffsetResetToData(&dataRsp.rspOffset, 0, 0); + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + } + } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) { + ASSERT(0); + } + +OVER: + // TODO wrap in destroy func + taosArrayDestroy(dataRsp.blockDataLen); + taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree); + + if (dataRsp.withSchema) { + taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + } + + if (dataRsp.withTbName) { + taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree); + } + + return code; +} + +#if 0 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; @@ -185,10 +439,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } else { STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); if (pOffset != NULL) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); + ASSERT(pOffset->val.type == TMQ_OFFSET__LOG); tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey, - TD_VID(pTq->pVnode), pOffset->version); - fetchOffset = pOffset->version + 1; + TD_VID(pTq->pVnode), pOffset->val.version); + fetchOffset = pOffset->val.version + 1; } else { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = walGetFirstVer(pTq->pWal); @@ -241,12 +495,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (rsp.withTbName) { rsp.blockTbName = taosArrayInit(0, sizeof(void*)); } + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { rsp.withSchema = false; - rsp.withTag = false; } else { rsp.withSchema = true; - rsp.withTag = false; rsp.blockSchema = taosArrayInit(0, sizeof(void*)); } @@ -302,10 +555,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } } else { ASSERT(pHandle->fetchMeta); - ASSERT(pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB || - pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || - pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || - pHead->msgType == TDMT_VND_DROP_TTL_TABLE); + ASSERT(IS_META_MSG(pHead->msgType)); tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; metaRsp.reqOffset = pReq->currentOffset; @@ -341,7 +591,7 @@ SEND_RSP: rsp.rspOffset = fetchOffset; - if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) { + if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) { code = -1; } OVER: @@ -359,6 +609,7 @@ OVER: return code; } +#endif int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; @@ -403,7 +654,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { .reader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, -// .initTsdbReader = 1, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -474,12 +724,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { if (pTask->execType != TASK_EXEC__NONE) { // expand runners if (pTask->isDataScan) { - STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { .reader = pStreamReader, .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, -// .initTsdbReader = 1, }; /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index c7d7c787a2..eb09199c70 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -15,7 +15,7 @@ #include "tq.h" -static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRsp) { +static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) return -1; @@ -37,7 +37,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs return 0; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) { +static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataRsp* pRsp) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper); if (pSW == NULL) { return -1; @@ -46,10 +46,9 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); - int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; if (metaGetTableEntryByUid(&mr, uid) < 0) { ASSERT(0); return -1; @@ -60,13 +59,13 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD return 0; } -int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) { +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, int32_t workerId) { ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); qTaskInfo_t task = pExec->execCol.task[workerId]; + // TODO set uid and ts if (qStreamScanSnapshot(task) < 0) { ASSERT(0); } - // set version while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -79,17 +78,24 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); + + if (pRsp->withTbName) { + // TODO + pRsp->withTbName = 0; + /*int64_t uid = 0;*/ + /*tqAddTbNameToRsp(pTq, uid, pRsp, workerId);*/ + } pRsp->blockNum++; } return 0; } -int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) { +int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) { if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { qTaskInfo_t task = pExec->execCol.task[workerId]; ASSERT(task); - qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); + qSetStreamInput(task, pReq, STREAM_INPUT__DATA_SUBMIT, false); while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -102,13 +108,14 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { - tqAddTbNameToRsp(pTq, pExec, pRsp, workerId); + int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp, workerId); } pRsp->blockNum++; } } else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { pRsp->withSchema = 1; - STqReadHandle* pReader = pExec->pExecReader[workerId]; + SStreamReader* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; @@ -118,14 +125,15 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR } tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { - tqAddTbNameToRsp(pTq, pExec, pRsp, workerId); + int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp, workerId); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { pRsp->withSchema = 1; - STqReadHandle* pReader = pExec->pExecReader[workerId]; + SStreamReader* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; @@ -135,7 +143,8 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR } tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { - tqAddTbNameToRsp(pTq, pExec, pRsp, workerId); + int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp, workerId); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index e536e87032..16b8872098 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -92,8 +92,8 @@ STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) { } int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); - ASSERT(pOffset->version >= 0); + /*ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);*/ + /*ASSERT(pOffset->val.version >= 0);*/ return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)); } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 2ec627bd5c..bd7cda4de3 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -15,16 +15,17 @@ #include "tq.h" +#if 0 void tqTmrRspFunc(void* param, void* tmrId) { STqHandle* pHandle = (STqHandle*)param; atomic_store_8(&pHandle->pushHandle.tmrStopped, 1); } -static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataBlkRsp* pRsp) { +static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) { SStreamDataSubmit* pSubmit = *ppSubmit; while (pSubmit != NULL) { ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); - if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { + if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { /*ASSERT(0);*/ } // update processed @@ -43,7 +44,7 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm } int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { - SMqDataBlkRsp rsp = {0}; + SMqDataRsp rsp = {0}; // 1. guard and set status executing int8_t execStatus = atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE, TASK_EXEC_STATUS__EXECUTING); @@ -175,13 +176,13 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ taosWLockLatch(&pHandle->pushHandle.lock); - SMqDataBlkRsp rsp = {0}; + SMqDataRsp rsp = {0}; rsp.reqOffset = pHandle->pushHandle.reqOffset; rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); if (msgType == TDMT_VND_SUBMIT) { - tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); + tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); } else { // TODO ASSERT(0); @@ -233,6 +234,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ return 0; } +#endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType == TDMT_VND_SUBMIT) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index be2a35c901..7d7b9636df 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -44,10 +44,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead* } else { if (pHandle->fetchMeta) { SWalReadHead* pHead = &((*ppHeadWithCkSum)->head); - if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB || - pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || - pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || - pHead->msgType == TDMT_VND_DROP_TTL_TABLE) { + if (IS_META_MSG(pHead->msgType)) { code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum); if (code < 0) { @@ -76,8 +73,8 @@ END: return code; } -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { - STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle)); +SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) { + SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader)); if (pReadHandle == NULL) { return NULL; } @@ -85,15 +82,15 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { pReadHandle->pMsg = NULL; pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; - pReadHandle->cachedSchemaVer = -1; - pReadHandle->cachedSchemaSuid = -1; + pReadHandle->cachedSchemaVer = 0; + pReadHandle->cachedSchemaSuid = 0; pReadHandle->pSchema = NULL; pReadHandle->pSchemaWrapper = NULL; pReadHandle->tbIdHash = NULL; return pReadHandle; } -int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) { +int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) { pReadHandle->pMsg = pMsg; if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; @@ -108,7 +105,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t return 0; } -bool tqNextDataBlock(STqReadHandle* pHandle) { +bool tqNextDataBlock(SStreamReader* pHandle) { if (pHandle->pMsg == NULL) return false; while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { @@ -130,7 +127,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { return false; } -bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { +bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; @@ -146,7 +143,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { // TODO: cache multiple schema int32_t sversion = htonl(pHandle->pBlock->sversion); if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || @@ -256,9 +253,9 @@ FAIL: return -1; } -void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } +void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } -int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { +int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash) { taosHashClear(pHandle->tbIdHash); } @@ -277,7 +274,7 @@ int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { +int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash == NULL) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { @@ -294,7 +291,7 @@ int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { +int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { ASSERT(pHandle->tbIdHash != NULL); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 72c766e0ae..d27fcb4f03 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -780,7 +780,7 @@ _exit: // TODO: the partial success scenario and the error case // TODO: refactor if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { - tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT); } return 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6de364e63a..31edc46b4d 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -44,12 +44,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu // prevent setting a different type of block pInfo->blockType = type; - if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (type == STREAM_INPUT__DATA_SUBMIT) { if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) { qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - } else if (type == STREAM_DATA_TYPE_SSDATA_BLOCK) { + } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; @@ -60,9 +60,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } - } else if (type == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + } else if (type == STREAM_INPUT__DATA_SCAN) { // do nothing - ASSERT(pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT); + ASSERT(pInfo->blockType == STREAM_INPUT__DATA_SCAN); } else { ASSERT(0); } @@ -76,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { return TSDB_CODE_QRY_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL); + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__DATA_SCAN, 0, NULL); } int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c51ef44154..8cfdb22313 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -962,7 +962,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { size_t total = taosArrayGetSize(pInfo->pBlockLists); // TODO: refactor - if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { if (pInfo->validBlockIndex >= total) { /*doClearBufferedBlocks(pInfo);*/ pOperator->status = OP_EXEC_DONE; @@ -973,7 +973,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); blockDataUpdateTsWindow(pBlock, 0); if (pBlock->info.type == STREAM_RETRIEVE) { - pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK; + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; copyDataBlock(pInfo->pPullDataRes, pBlock); pInfo->pullDataResIndex = 0; @@ -981,7 +981,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } return pBlock; - } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -1133,7 +1133,9 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; - } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + } else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) { + // check reader last status + // if not match, reset status SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); return pResult && pResult->info.rows > 0 ? pResult : NULL; @@ -1213,7 +1215,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tableUid = pScanPhyNode->uid; // set the extract column id to streamHandle - tqReadHandleSetColIdList((STqReadHandle*)pHandle->reader, pColIds); + tqReadHandleSetColIdList((SStreamReader*)pHandle->reader, pColIds); SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList); if (code != 0) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index edd1ec0e75..6d287ef887 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -970,9 +970,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { getTableScanInfo(pOperator, &pInfo->order, &scanFlag); if (pInfo->scalarSupp.pExprInfo != NULL) { - SExprSupp* pExprSup =& pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, - pExprSup->numOfExprs, NULL); + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } // the pDataBlock are always the same one, no need to call this again @@ -1512,23 +1511,24 @@ void increaseTs(SqlFunctionCtx* pCtx) { SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) { + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, bool isStream) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->win = pTaskInfo->window; - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = *pTwAggSupp; if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2498,10 +2498,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + int32_t childIndex = getChildIndex(pBlock); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; + SExprSupp* pChildSup = &pChildOp->exprSupp; doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex, pChildSup->numOfExprs, pBlock, NULL); @@ -2527,7 +2527,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } continue; - } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) { + } else if (pBlock->info.type == STREAM_PUSH_OVER && IS_FINAL_OP(pInfo)) { processPushEmpty(pBlock, pInfo->pPullDataMap); continue; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 699a050aea..1ff1cbe833 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1052,7 +1052,8 @@ static int32_t translateUniqueMode(SFunctionNode* pFunc, char* pErrBuf, int32_t SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (!nodesExprHasColumn(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of %s must contain columns", isUnique ? "UNIQUE" : "MODE"); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of %s must contain columns", + isUnique ? "UNIQUE" : "MODE"); } pFunc->node.resType = ((SExprNode*)pPara)->resType; @@ -1228,19 +1229,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // The number of parameters has been limited by the syntax definition - //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + // uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && - // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && - // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { - // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - //} - //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || - // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { - // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - //} + // if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && + // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && + // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + // } + // if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || + // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + // } int32_t para2Bytes = pFunc->node.resType.bytes; if (IS_VAR_DATA_TYPE(para2Type)) { @@ -1639,7 +1640,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "leastsquares", .type = FUNCTION_TYPE_LEASTSQUARES, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateLeastSQR, .getEnvFunc = getLeastSQRFuncEnv, .initFunc = leastSQRFunctionSetup, diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b3ceb92505..3aee636ac9 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -17,21 +17,20 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { void* exec = pTask->exec.executor; - bool hasData = false; // set input SStreamQueueItem* pItem = (SStreamQueueItem*)data; if (pItem->type == STREAM_INPUT__TRIGGER) { SStreamTrigger* pTrigger = (SStreamTrigger*)data; - qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->isDataScan); SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); + qSetStreamInput(exec, pSubmit->data, STREAM_INPUT__DATA_SUBMIT, false); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SStreamDataBlock* pBlock = (SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DROP) { // TODO exec drop return 0; @@ -46,20 +45,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; - /*block.info.type = STREAM_PUSH_EMPTY;*/ - // block.info.childId = pTask->selfChildId; + SSDataBlock block = {0}; SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); - /*SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true);*/ assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); - block.info.type = STREAM_PUSH_EMPTY; + block.info.type = STREAM_PUSH_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } break; } - hasData = true; if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { @@ -73,9 +68,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); - /*SSDataBlock* outputCopy = createOneDataBlock(output, true);*/ - /*outputCopy->info.childId = pTask->selfChildId;*/ - /*taosArrayPush(pRes, outputCopy);*/ } return 0; } @@ -164,4 +156,3 @@ FAIL: atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); return -1; } - diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 1daa9bd50e..97aa182dc9 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -86,9 +86,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { ASSERT(pTask->sinkType == TASK_SINK__NONE); } - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (pTask->dispatchType == TASK_DISPATCH__FIXED) { if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; @@ -147,9 +145,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { ASSERT(pTask->sinkType == TASK_SINK__NONE); } - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (pTask->dispatchType == TASK_DISPATCH__FIXED) { if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; From e74187248bfae7d56924b4c6c98524a79b1d68b4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 29 Jun 2022 22:29:02 +0800 Subject: [PATCH 5/7] fix(tmq): offset --- source/client/src/tmq.c | 16 ++++++++-------- source/dnode/vnode/src/tq/tq.c | 4 ++-- tests/script/tsim/tmq/basic2Of2ConsOverlap.sim | 2 ++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 4cef25b9ce..ec2ebdfae9 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1124,10 +1124,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { pRspWrapper->topicHandle = pTopic; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { - memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp); + memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); /*tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);*/ } else { ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP); @@ -1138,7 +1138,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId, - pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset, rspType); + pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, rspType); taosWriteQitem(tmq->mqueue, pRspWrapper); tsem_post(&tmq->rspSem); @@ -1203,17 +1203,17 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { for (int32_t j = 0; j < vgNumGet; j++) { SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); - int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); - int64_t offset = tmq->resetOffsetCfg; + STqOffsetVal* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); + STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg}; if (pOffset != NULL) { - offset = *pOffset; + offsetNew = *pOffset; } - tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch, - pVgEp->vgId, offset, vgKey); + /*tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch,*/ + /*pVgEp->vgId, offset, vgKey);*/ SMqClientVg clientVg = { .pollCnt = 0, - .currentOffsetNew.type = tmq->resetOffsetCfg, + .currentOffsetNew = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac36dbf774..aadacf1e72 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -142,7 +142,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); SEncoder encoder; - tEncoderInit(&encoder, abuf, tlen); + tEncoderInit(&encoder, abuf, len); tEncodeSMqDataRsp(&encoder, pRsp); /*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/ @@ -332,7 +332,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) { // TODO add push mgr - tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer - 1); if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; } diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index 480cf520d9..d796d00f03 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -134,6 +134,8 @@ endi $totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfMsgCnt = $data[0][2] + $data[1][2] if $sumOfMsgCnt != $totalMsgCons then + print total: $totalMsgCons + print sum: $sumOfMsgCnt return -1 endi From 94c3eb29d26d7296b8702e60cc16187c5f2ed574 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 29 Jun 2022 23:25:04 +0800 Subject: [PATCH 6/7] fix(tmq): rsp error offset --- source/client/src/tmq.c | 4 ++-- source/dnode/vnode/src/tq/tq.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ec2ebdfae9..5493628e74 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -514,6 +514,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); + int32_t code = -1; + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (strcmp(pTopic->topicName, topic) != 0) continue; @@ -530,8 +532,6 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm } } - int32_t code = -1; - HANDLE_RSP: if (pParamSet->totalRspNum == 0) { tsem_destroy(&pParamSet->rspSem); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aadacf1e72..a08515ed5c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -332,7 +332,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) { // TODO add push mgr - tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer - 1); + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version); if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; } @@ -354,6 +355,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO continue scan until meeting batch requirement if (dataRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version); if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; From 3f00a6db8ec95731d18600f7713faba01c7ab256 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Wed, 29 Jun 2022 23:26:10 +0800 Subject: [PATCH 7/7] feat(taosAdapter): remake REST api (#14351) --- tools/taosadapter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taosadapter b/tools/taosadapter index c1dc11f3ca..c885e967e4 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit c1dc11f3cae64adf31dbd9a954ef8372d1e8f671 +Subproject commit c885e967e490105999b84d009a15168728dfafaf