From 8bf49fdbd2f9fa0a60f3c532f3d049f22bfc19d3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 24 Mar 2023 17:16:06 +0800 Subject: [PATCH 01/47] fix: skiplist level --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d34af9acae..d8253e25f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -16,7 +16,7 @@ #include "tsdb.h" #define MEM_MIN_HASH 1024 -#define SL_MAX_LEVEL 5 +#define SL_MAX_LEVEL 1 // sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l) * 2 #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) From 8a89550f16f48d47a39c6e6605909b2f7ddb129a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sat, 25 Mar 2023 13:10:14 +0800 Subject: [PATCH 02/47] fix: skiplist level --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d8253e25f7..d34af9acae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -16,7 +16,7 @@ #include "tsdb.h" #define MEM_MIN_HASH 1024 -#define SL_MAX_LEVEL 1 +#define SL_MAX_LEVEL 5 // sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l) * 2 #define SL_NODE_SIZE(l) (sizeof(SMemSkipListNode) + ((l) << 4)) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index c326c8bfac..54d2eabfa4 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -29,7 +29,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI, .update = 1, .compression = 2, - .slLevel = 5, + .slLevel = 1, .days = 14400, .minRows = 100, .maxRows = 4096, From 61571d92967c9da09aaf6ee1666aa83977ebb7e0 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 27 Mar 2023 19:40:29 +0800 Subject: [PATCH 03/47] enh: optimize count performance --- source/common/src/tdatablock.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 209 +++++++++++++++++++++++- source/libs/executor/src/scanoperator.c | 9 +- 3 files changed, 213 insertions(+), 7 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c8d394b43..a75046d06d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1180,7 +1180,7 @@ void blockDataCleanup(SSDataBlock* pDataBlock) { void blockDataEmpty(SSDataBlock* pDataBlock) { SDataBlockInfo* pInfo = &pDataBlock->info; - if (pInfo->capacity == 0 || pInfo->rows > pDataBlock->info.capacity) { + if (pInfo->capacity == 0) { return; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 96bce02b67..c6444b5ded 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -24,6 +24,11 @@ typedef enum { EXTERNAL_ROWS_NEXT = 0x3, } EContentData; +typedef enum { + READ_MODE_COUNT_ONLY = 0x1, + READ_MODE_ALL, +} EReadMode; + typedef struct { STbDataIter* iter; int32_t index; @@ -167,6 +172,8 @@ struct STsdbReader { uint64_t suid; int16_t order; bool freeBlock; + EReadMode readMode; + uint64_t rowsNum; STimeWindow window; // the primary query time window that applies to all queries SSDataBlock* pResBlock; int32_t capacity; @@ -2998,6 +3005,9 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + return code; + } code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3006,12 +3016,19 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + return code; + } // data in memory that are earlier than current file block // rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + return code; + } + // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); ASSERT(tsLast >= pBlock->maxKey.ts); @@ -3069,6 +3086,131 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return code; } + +static int32_t doSumBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { + int64_t st = taosGetTimestampUs(); + LRUHandle* handle = NULL; + int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); + if (code != TSDB_CODE_SUCCESS || handle == NULL) { + goto _end; + } + + int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); + + SArray* aBlockIdx = (SArray*)taosLRUCacheValue(pFileReader->pTsdb->biCache, handle); + size_t num = taosArrayGetSize(aBlockIdx); + if (num == 0) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return TSDB_CODE_SUCCESS; + } + + SBlockIdx* pBlockIdx = NULL; + int32_t i = 0; + for (int32_t i = 0; i < num; ++i) { + pBlockIdx = (SBlockIdx*)taosArrayGet(aBlockIdx, i); + if (pBlockIdx->suid != pReader->suid) { + continue; + } + + STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); + if (p == NULL || *p == NULL) { + continue; + } + + STableBlockScanInfo *pScanInfo = *p; + tMapDataReset(&pScanInfo->mapData); + tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); + + SDataBlk block = {0}; + for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { + tGetDataBlk(pScanInfo->mapData.pData + pScanInfo->mapData.aOffset[j], &block); + pReader->rowsNum += block.nRow; + } + } + +_end: + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); + return code; +} + + +static int32_t readRowsCountFromFile(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + while (1) { + bool hasNext = false; + int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + if (code) { + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + code = doSumBlockRows(pReader, pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pReader->status.loadFromFile = false; + + return code; +} + +static int32_t readRowsCountFromStt(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + SSttBlockLoadInfo* pBlockLoadInfo = NULL; + + for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file + pBlockLoadInfo = &pLastBlockReader->pInfo[i]; + + if (!pLastBlockReader->pInfo[i].sttBlockLoaded) { + pLastBlockReader->pInfo[i].sttBlockLoaded = true; + + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); + if (code) { + return code; + } + } + + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + + if (size >= 1) { + SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); + SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); + + // all identical + if (pStart->suid == pEnd->suid) { + if (pStart->suid != pReader->suid) { + // no qualified stt block existed + taosArrayClear(pBlockLoadInfo->aSttBlk); + continue; + } + } else { + for (int32_t i = 0; i < size; ++i) { + SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + uint64_t s = p->suid; + if (s < pReader->suid) { + continue; + } + + if (s == pReader->suid) { + pReader->rowsNum += p->nRow; + } else if (s > pReader->suid) { + break; + } + } + } + } + } + + return code; +} + + static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; STableUidList* pUidList = &pStatus->uidList; @@ -3212,6 +3354,12 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { initBlockDumpInfo(pReader, pBlockIter); } else { if (pReader->status.pCurrentFileset->nSttF > 0) { + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + pReader->pResBlock->info.rows = pReader->rowsNum; + pReader->rowsNum = 0; + return TSDB_CODE_SUCCESS; + } + // data blocks in current file are exhausted, let's try the next file now SBlockData* pBlockData = &pReader->status.fileBlockData; if (pBlockData->uid != 0) { @@ -3226,7 +3374,17 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked - if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { + pReader->pResBlock->info.rows = pReader->rowsNum; + pReader->rowsNum = 0; + return TSDB_CODE_SUCCESS; + } + + if (pReader->status.loadFromFile == false) { return code; } @@ -3240,6 +3398,17 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } code = doBuildDataBlock(pReader); + if (READ_MODE_COUNT_ONLY == pReader->readMode) { + if (false == pReader->status.composedDataBlock && pDumpInfo->allDumped) { + pReader->rowsNum += pReader->pResBlock->info.rows; + pReader->pResBlock->info.rows = 0; + continue; + } else if (pReader->pResBlock->info.rows == 0 && pReader->rowsNum > 0) { + pReader->pResBlock->info.rows = pReader->rowsNum; + pReader->rowsNum = 0; + return TSDB_CODE_SUCCESS; + } + } } if (code != TSDB_CODE_SUCCESS) { @@ -3986,6 +4155,8 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; if (pStatus->fileIter.numOfFiles == 0) { pStatus->loadFromFile = false; + } else if (READ_MODE_COUNT_ONLY == pReader->readMode) { + // DO NOTHING } else { code = initForFirstBlockInFile(pReader, pBlockIter); } @@ -4091,6 +4262,9 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->suspended = true; + + pReader->readMode = READ_MODE_COUNT_ONLY; + tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -4394,6 +4568,35 @@ _err: return code; } +static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + SSDataBlock* pBlock = pReader->pResBlock; + + while (1) { + if (pReader->status.loadFromFile == false) { + break; + } + + code = readRowsCountFromFile(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = readRowsCountFromStt(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + } + + pBlock->info.rows = pReader->rowsNum; + pBlock->info.id.uid = 0; + pBlock->info.dataLoad = 0; + + pReader->rowsNum = 0; + + return pBlock->info.rows > 0; +} + static bool doTsdbNextDataBlock(STsdbReader* pReader) { // cleanup the data that belongs to the previous data block SSDataBlock* pBlock = pReader->pResBlock; @@ -4404,6 +4607,10 @@ static bool doTsdbNextDataBlock(STsdbReader* pReader) { return false; } + if (READ_MODE_COUNT_ONLY == pReader->readMode) { + return tsdbReadRowsCountOnly(pReader); + } + if (pStatus->loadFromFile) { int32_t code = buildBlockFromFiles(pReader); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 5dff1abb97..2af13f083d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -654,9 +654,10 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - ASSERT(pBlock->info.id.uid != 0); - pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); - + if (pBlock->info.id.uid) { + pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid); + } + uint32_t status = 0; int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { @@ -680,7 +681,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; - ASSERT(pBlock->info.id.uid != 0); return pBlock; } return NULL; @@ -797,7 +797,6 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { SSDataBlock* result = doGroupedTableScan(pOperator); if (result != NULL) { - ASSERT(result->info.id.uid != 0); return result; } From a87d35be082d1409a1c9a5d384f59472b1b3cec9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 17:40:46 +0800 Subject: [PATCH 04/47] fix: optimize performance --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 2 ++ source/dnode/vnode/src/tsdb/tsdbMemTable.c | 30 ++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbRead.c | 34 +++++++++++++++++++--- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executor.c | 4 +-- source/libs/executor/src/scanoperator.c | 13 +++++++-- source/libs/executor/src/sysscanoperator.c | 2 +- 8 files changed, 76 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2d053d04ae..eae398880b 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -178,7 +178,7 @@ typedef struct STsdbReader STsdbReader; int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t num); int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pTableList, int32_t numOfTables, - SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr); + SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly); void tsdbReaderSetId(STsdbReader* pReader, const char* idstr); void tsdbReaderClose(STsdbReader *pReader); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 452b1f6c0b..0c4ada2cb1 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -224,6 +224,8 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum); + // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); // tsdbFile.c ============================================================================================== diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index d34af9acae..46d0509c38 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -282,6 +282,34 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { return true; } +int64_t tsdbCountTbDataRows(STbData *pTbData) { + SMemSkipListNode *pNode = NULL; + int64_t rowsNum = 0; + + while (true) { + pNode = SL_GET_NODE_FORWARD(pTbData->sl.pHead, 0); + if (pNode == pTbData->sl.pTail) { + return rowsNum; + } + + rowsNum++; + } +} + +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) { + taosRLockLatch(&pMemTable->latch); + for (int32_t i = 0; i < pMemTable->nBucket; ++i) { + STbData *pTbData = pMemTable->aBucket[i]; + + void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + if (p == NULL) { + continue; + } + rowsNum += tsdbCountTbDataRows(pTbData); + } + taosRUnLockLatch(&pMemTable->latch); +} + static int32_t tsdbMemTableRehash(SMemTable *pMemTable) { int32_t code = 0; @@ -787,4 +815,4 @@ SArray *tsdbMemTableGetTbDataArray(SMemTable *pMemTable) { _exit: return aTbDataP; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c6444b5ded..8b6f318434 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3113,7 +3113,7 @@ static int32_t doSumBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { } STableBlockScanInfo** p = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(pBlockIdx->uid)); - if (p == NULL || *p == NULL) { + if (p == NULL) { continue; } @@ -3189,6 +3189,10 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { taosArrayClear(pBlockLoadInfo->aSttBlk); continue; } + for (int32_t i = 0; i < size; ++i) { + SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + pReader->rowsNum += p->nRow; + } } else { for (int32_t i = 0; i < size; ++i) { SSttBlk *p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); @@ -3210,6 +3214,22 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { return code; } +static int32_t readRowsCountFromMem(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + int64_t memNum = 0, imemNum = 0; + if (pReader->pReadSnap->pMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pMem, pReader->status.pTableMap, &memNum); + } + + if (pReader->pReadSnap->pIMem != NULL) { + tsdbMemTableCountRows(pReader->pReadSnap->pIMem, pReader->status.pTableMap, &imemNum); + } + + pReader->rowsNum += memNum + imemNum; + + return code; +} + static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { SReaderStatus* pStatus = &pReader->status; @@ -4170,7 +4190,7 @@ static int32_t doOpenReaderImpl(STsdbReader* pReader) { // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableList, int32_t numOfTables, - SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr) { + SSDataBlock* pResBlock, STsdbReader** ppReader, const char* idstr, bool countOnly) { STimeWindow window = pCond->twindows; if (pCond->type == TIMEWINDOW_RANGE_EXTERNAL) { pCond->twindows.skey += 1; @@ -4262,9 +4282,10 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL pReader->suspended = true; + if (countOnly) { + pReader->readMode = READ_MODE_COUNT_ONLY; + } - pReader->readMode = READ_MODE_COUNT_ONLY; - tsdbDebug("%p total numOfTable:%d in this query %s", pReader, numOfTables, pReader->idStr); return code; @@ -4588,6 +4609,11 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { } } + code = readRowsCountFromMem(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + pBlock->info.rows = pReader->rowsNum; pBlock->info.id.uid = 0; pBlock->info.dataLoad = 0; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 3f519568c4..fd29df1acc 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -339,6 +339,7 @@ typedef struct STableScanInfo { int8_t scanMode; int8_t assignBlockUid; bool hasGroupByTag; + bool countOnly; } STableScanInfo; typedef struct STableMergeScanInfo { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index db58bd6f68..a1b27114b3 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -1140,7 +1140,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT int32_t num = tableListGetSize(pTaskInfo->pTableInfoList); if (tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &pTableScanInfo->base.cond, pList, num, - pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL) < 0 || + pTableScanInfo->pResBlock, &pTableScanInfo->base.dataReader, NULL, false) < 0 || pTableScanInfo->base.dataReader == NULL) { qError("tsdbReaderOpen failed. uid:%" PRIi64, pOffset->uid); return -1; @@ -1192,7 +1192,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT STableKeyInfo* pList = tableListGetInfo(pTaskInfo->pTableInfoList, 0); int32_t size = tableListGetSize(pTaskInfo->pTableInfoList); - tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL); + tsdbReaderOpen(pInfo->vnode, &pTaskInfo->streamInfo.tableCond, pList, size, NULL, &pInfo->dataReader, NULL, false); cleanupQueryTableDataCond(&pTaskInfo->streamInfo.tableCond); strcpy(pTaskInfo->streamInfo.tbName, mtInfo.tbName); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 2af13f083d..854a2c1aaf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -31,6 +31,9 @@ #include "thash.h" #include "ttypes.h" +int32_t scanDebug = 0; + + #define MULTI_READER_MAX_TABLE_NUM 5000 #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) @@ -785,7 +788,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { ASSERT(pInfo->base.dataReader == NULL); int32_t code = tsdbReaderOpen(pInfo->base.readHandle.vnode, &pInfo->base.cond, pList, num, pInfo->pResBlock, - (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo)); + (STsdbReader**)&pInfo->base.dataReader, GET_TASKID(pTaskInfo), pInfo->countOnly); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -916,6 +919,10 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, goto _error; } + if (scanDebug) { + pInfo->countOnly = true; + } + taosLRUCacheSetStrictCapacity(pInfo->base.metaCache.pTableMetaEntryCache, false); pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doTableScan, NULL, destroyTableScanOperatorInfo, optrDefaultBufFn, getTableScannerExecInfo); @@ -1007,7 +1014,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU SSDataBlock* pBlock = pTableScanInfo->pResBlock; STsdbReader* pReader = NULL; int32_t code = tsdbReaderOpen(pTableScanInfo->base.readHandle.vnode, &cond, &tblInfo, 1, pBlock, - (STsdbReader**)&pReader, GET_TASKID(pTaskInfo)); + (STsdbReader**)&pReader, GET_TASKID(pTaskInfo), false); if (code != TSDB_CODE_SUCCESS) { terrno = code; T_LONG_JMP(pTaskInfo->env, code); @@ -2601,7 +2608,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) { SReadHandle* pHandle = &pInfo->base.readHandle; if (NULL == source->dataReader || !source->multiReader) { - code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo)); + code = tsdbReaderOpen(pHandle->vnode, pQueryCond, p, 1, pBlock, &source->dataReader, GET_TASKID(pTaskInfo), false); if (code != 0) { T_LONG_JMP(pTaskInfo->env, code); } diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index f24d3523c8..d05c692f19 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2267,7 +2267,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi size_t num = tableListGetSize(pTableListInfo); void* pList = tableListGetInfo(pTableListInfo, 0); - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str); + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, pInfo->pResBlock, &pInfo->pHandle, pTaskInfo->id.str, false); cleanupQueryTableDataCond(&cond); if (code != 0) { goto _error; From 0cbcf624b4043771bcb01f44bfc1209013ce6880 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 18:24:26 +0800 Subject: [PATCH 05/47] fix: add debug --- source/common/src/tglobal.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index aeeec1d61c..e97523df3d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1347,9 +1347,15 @@ void taosCleanupCfg() { } } +extern int32_t scanDebug; void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { int32_t flag = atoi(value); + if (1 == flag) { + scanDebug = 1; + } else { + scanDebug = 0; + } taosSetAllDebugFlag(flag, true); return; } From e0f951e668f3c7cced572627124ff0491a520c1f Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 19:15:38 +0800 Subject: [PATCH 06/47] fix: memory rows issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 46d0509c38..23c5ddcf4c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -300,12 +300,16 @@ void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int taosRLockLatch(&pMemTable->latch); for (int32_t i = 0; i < pMemTable->nBucket; ++i) { STbData *pTbData = pMemTable->aBucket[i]; - - void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); - if (p == NULL) { - continue; + while (pTbData) { + void* p = taosHashGet(pTableMap, &pTbData->uid, sizeof(pTbData->uid)); + if (p == NULL) { + pTbData = pTbData->next; + continue; + } + + rowsNum += tsdbCountTbDataRows(pTbData); + pTbData = pTbData->next; } - rowsNum += tsdbCountTbDataRows(pTbData); } taosRUnLockLatch(&pMemTable->latch); } From c5f37d284ecdd2c9deab38b321f6a5a9b1fca1e9 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 19:27:49 +0800 Subject: [PATCH 07/47] fix: memory row count issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 23c5ddcf4c..46f5aefba0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -283,11 +283,11 @@ bool tsdbTbDataIterNext(STbDataIter *pIter) { } int64_t tsdbCountTbDataRows(STbData *pTbData) { - SMemSkipListNode *pNode = NULL; + SMemSkipListNode *pNode = pTbData->sl.pHead; int64_t rowsNum = 0; while (true) { - pNode = SL_GET_NODE_FORWARD(pTbData->sl.pHead, 0); + pNode = SL_GET_NODE_FORWARD(pNode, 0); if (pNode == pTbData->sl.pTail) { return rowsNum; } From a0c8dd52b1429dc6308fc329212d73f3e7dc3b09 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Mar 2023 19:42:09 +0800 Subject: [PATCH 08/47] fix: memory count issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 46f5aefba0..bf47f25e15 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -307,7 +307,7 @@ void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int continue; } - rowsNum += tsdbCountTbDataRows(pTbData); + *rowsNum += tsdbCountTbDataRows(pTbData); pTbData = pTbData->next; } } From 0327a8eaa3450c8d33da983c081a68d9e16ce88d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 10:46:56 +0800 Subject: [PATCH 09/47] fix: block rows number type --- include/common/tcommon.h | 2 +- source/common/src/tdatablock.c | 10 +++---- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 2 +- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 6 ++-- source/dnode/vnode/src/tsdb/tsdbRead.c | 32 ++++++++++------------ source/libs/executor/src/executorimpl.c | 4 +-- source/libs/executor/src/projectoperator.c | 4 +-- source/libs/executor/src/scanoperator.c | 24 ++++++++-------- source/libs/executor/src/sortoperator.c | 2 +- source/libs/qworker/src/qworker.c | 2 +- 11 files changed, 45 insertions(+), 45 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9e928a79ac..51a714c792 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -185,7 +185,7 @@ typedef struct SBlockID { typedef struct SDataBlockInfo { STimeWindow window; int32_t rowSize; - int32_t rows; // todo hide this attribute + int64_t rows; // todo hide this attribute uint32_t capacity; SBlockID id; int16_t hasVarCol; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index a75046d06d..0692db6a25 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -973,7 +973,7 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { taosSort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); int64_t p1 = taosGetTimestampUs(); - uDebug("blockDataSort easy cost:%" PRId64 ", rows:%d\n", p1 - p0, pDataBlock->info.rows); + uDebug("blockDataSort easy cost:%" PRId64 ", rows:%" PRId64 "\n", p1 - p0, pDataBlock->info.rows); return TSDB_CODE_SUCCESS; } else { // var data type @@ -1739,14 +1739,14 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.id.uid; int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int16_t hasVarCol = pBlock->info.hasVarCol; - int32_t rows = pBlock->info.rows; + int64_t rows = pBlock->info.rows; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); int32_t tlen = 0; tlen += taosEncodeFixedI64(buf, tbUid); tlen += taosEncodeFixedI16(buf, numOfCols); tlen += taosEncodeFixedI16(buf, hasVarCol); - tlen += taosEncodeFixedI32(buf, rows); + tlen += taosEncodeFixedI64(buf, rows); tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { SColumnInfoData* pColData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); @@ -1777,7 +1777,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { buf = taosDecodeFixedU64(buf, &pBlock->info.id.uid); buf = taosDecodeFixedI16(buf, &numOfCols); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); - buf = taosDecodeFixedI32(buf, &pBlock->info.rows); + buf = taosDecodeFixedI64(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &sz); pBlock->pDataBlock = taosArrayInit(sz, sizeof(SColumnInfoData)); for (int32_t i = 0; i < sz; i++) { @@ -1981,7 +1981,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) int32_t len = 0; len += snprintf(dumpBuf + len, size - len, "===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64 - "|rows:%d|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", + "|rows:%" PRId64 "|version:%" PRIu64 "|cal start:%" PRIu64 "|cal end:%" PRIu64 "|tbl:%s\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId, pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version, pDataBlock->info.calWin.skey, pDataBlock->info.calWin.ekey, pDataBlock->info.parTbName); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 3ed1b083e4..c75c675ec3 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -669,7 +669,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma #endif for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { SSDataBlock *output = taosArrayGetP(pResList, i); - smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, + smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRId64, output->info.id.uid, output->info.id.groupId, output->info.rows); STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 7f9563ae5f..90ff1f8a84 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -340,7 +340,7 @@ int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { continue; } ret->fetchType = FETCH_TYPE__DATA; - tqDebug("return data rows %d", ret->data.info.rows); + tqDebug("return data rows %" PRId64, ret->data.info.rows); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index bf47f25e15..15fdbe2212 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -286,14 +286,16 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) { SMemSkipListNode *pNode = pTbData->sl.pHead; int64_t rowsNum = 0; - while (true) { - pNode = SL_GET_NODE_FORWARD(pNode, 0); + while (NULL != pNode) { if (pNode == pTbData->sl.pTail) { return rowsNum; } rowsNum++; + pNode = SL_GET_NODE_FORWARD(pNode, 0); } + + return rowsNum; } void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 8b6f318434..63d9b702fd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -1773,7 +1773,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* setComposedBlockFlag(pReader, true); double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; - tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%d, brange:%" PRId64 + tsdbDebug("%p build data block from cache completed, elapsed time:%.2f ms, numOfRows:%" PRId64 ", brange:%" PRId64 " - %" PRId64 ", uid:%" PRIu64 ", %s", pReader, elapsedTime, pBlock->info.rows, pBlock->info.window.skey, pBlock->info.window.ekey, pBlockScanInfo->uid, pReader->idStr); @@ -2723,7 +2723,7 @@ _end: if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + " rows:%" PRId64 ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -2970,7 +2970,7 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + " rows:%" PRId64 ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -3060,7 +3060,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 - " rows:%d, elapsed time:%.2f ms %s", + " rows:%" PRId64 ", elapsed time:%.2f ms %s", pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, pResBlock->info.rows, el, pReader->idStr); } @@ -4593,20 +4593,18 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; SSDataBlock* pBlock = pReader->pResBlock; - while (1) { - if (pReader->status.loadFromFile == false) { - break; - } - - code = readRowsCountFromFile(pReader); - if (code != TSDB_CODE_SUCCESS) { - return false; - } + if (pReader->status.loadFromFile == false) { + return false; + } - code = readRowsCountFromStt(pReader); - if (code != TSDB_CODE_SUCCESS) { - return false; - } + code = readRowsCountFromFile(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; + } + + code = readRowsCountFromStt(pReader); + if (code != TSDB_CODE_SUCCESS) { + return false; } code = readRowsCountFromMem(pReader); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 67174c3267..8ef2a2b584 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -1201,7 +1201,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { blockDataEnsureCapacity(pBlock, pBlock->info.rows + pRow->numOfRows); - qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", + qDebug("datablock capacity not sufficient, expand to required:%" PRId64 ", current capacity:%d, %s", (pRow->numOfRows+pBlock->info.rows), pBlock->info.capacity, GET_TASKID(pTaskInfo)); // todo set the pOperator->resultInfo size @@ -1214,7 +1214,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS pBlock->info.rows += pRow->numOfRows; } - qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, + qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, pBlock->info.id.groupId); pBlock->info.dataLoad = 1; blockDataUpdateTsWindow(pBlock, 0); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index 4e2e105d14..c943270df9 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -271,7 +271,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { return NULL; } } - qDebug("set op close, exec %d, status %d rows %d", pTaskInfo->execModel, pOperator->status, + qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); setOperatorCompleted(pOperator); break; @@ -337,7 +337,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // when apply the limit/offset for each group, pRes->info.rows may be 0, due to limit constraint. if (pFinalRes->info.rows > 0 || (pOperator->status == OP_EXEC_DONE)) { - qDebug("project return %d rows, status %d", pFinalRes->info.rows, pOperator->status); + qDebug("project return %" PRId64 " rows, status %d", pFinalRes->info.rows, pOperator->status); break; } } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 854a2c1aaf..12eaad8e2a 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -311,14 +311,14 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca taosMemoryFreeClear(pBlock->pBlockAgg); if (*status == FUNC_DATA_REQUIRED_FILTEROUT) { - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; pCost->totalRows += pBlock->info.rows; tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%d, uid:%" PRIu64, GET_TASKID(pTaskInfo), + qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); pCost->skipBlocks += 1; @@ -329,7 +329,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca loadSMA = true; // mark the operation of load sma; bool success = doLoadBlockSMA(pTableScanInfo, pBlock, pTaskInfo); if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead - qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); tsdbReleaseDataBlock(pTableScanInfo->dataReader); @@ -349,7 +349,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca size_t size = taosArrayGetSize(pBlock->pDataBlock); bool keep = doFilterByBlockSMA(pOperator->exprSupp.pFilterInfo, pBlock->pBlockAgg, size, pBlockInfo->rows); if (!keep) { - qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block filter out by block SMA, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->filterOutBlocks += 1; (*status) = FUNC_DATA_REQUIRED_FILTEROUT; @@ -366,7 +366,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca // try to filter data block according to current results doDynamicPruneDataBlock(pOperator, pBlockInfo, status); if (*status == FUNC_DATA_REQUIRED_NOT_LOAD) { - qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%d", GET_TASKID(pTaskInfo), + qDebug("%s data block skipped due to dynamic prune, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 , GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); @@ -397,7 +397,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca if (pBlock->info.rows == 0) { pCost->filterOutBlocks += 1; - qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%d, elapsed time:%.2f ms", + qDebug("%s data block filter out, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", elapsed time:%.2f ms", GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, el); } else { qDebug("%s data block filter applied, elapsed time:%.2f ms", GET_TASKID(pTaskInfo), el); @@ -1028,7 +1028,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU } tsdbReaderClose(pReader); - qDebug("retrieve prev rows:%d, skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 + qDebug("retrieve prev rows:%" PRId64 ", skey:%" PRId64 ", ekey:%" PRId64 " uid:%" PRIu64 ", max ver:%" PRId64 ", suid:%" PRIu64, pBlock->info.rows, startTs, endTs, tbUid, maxVersion, cond.suid); @@ -1621,7 +1621,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %d rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, + qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, pInfo->tqReader->pWalReader->curVersion); pTaskInfo->streamInfo.returned = 1; return pResult; @@ -1658,7 +1658,7 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { setBlockIntoRes(pInfo, &ret.data, true); if (pInfo->pRes->info.rows > 0) { pOperator->status = OP_EXEC_RECV; - qDebug("queue scan log return %d rows", pInfo->pRes->info.rows); + qDebug("queue scan log return %" PRId64 " rows", pInfo->pRes->info.rows); return pInfo->pRes; } } else if (ret.fetchType == FETCH_TYPE__META) { @@ -1848,7 +1848,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pCreateTbRes, "recover createTbl"); return pInfo->pCreateTbRes; } - qDebug("stream recover scan get block, rows %d", pInfo->pRecoverRes->info.rows); + qDebug("stream recover scan get block, rows %" PRId64 , pInfo->pRecoverRes->info.rows); printDataBlock(pInfo->pRecoverRes, "scan recover"); return pInfo->pRecoverRes; } @@ -2077,7 +2077,7 @@ FETCH_NEXT_BLOCK: pOperator->resultInfo.totalRows += pBlockInfo->rows; // printDataBlock(pInfo->pRes, "stream scan"); - qDebug("scan rows: %d", pBlockInfo->rows); + qDebug("scan rows: %" PRId64 , pBlockInfo->rows); if (pBlockInfo->rows > 0) { return pInfo->pRes; } @@ -2822,7 +2822,7 @@ SSDataBlock* getSortedTableMergeScanBlockData(SSortHandle* pHandle, SSDataBlock* } bool limitReached = applyLimitOffset(&pInfo->limitInfo, pResBlock, pTaskInfo); - qDebug("%s get sorted row block, rows:%d, limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, + qDebug("%s get sorted row block, rows:%" PRId64 ", limit:%" PRId64, GET_TASKID(pTaskInfo), pResBlock->info.rows, pInfo->limitInfo.numOfOutputRows); return (pResBlock->info.rows > 0) ? pResBlock : NULL; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 90c7fa10ca..cb0f1aa068 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -698,7 +698,7 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData pDataBlock->info.dataLoad = 1; } - qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, + qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%" PRId64 , GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId, pDataBlock->info.rows); return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ec4049a3eb..92c7852dbc 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -199,7 +199,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { QW_ERR_JRET(code); } - QW_TASK_DLOG("data put into sink, rows:%d, continueExecTask:%d", pRes->info.rows, qcontinue); + QW_TASK_DLOG("data put into sink, rows:%" PRId64 ", continueExecTask:%d", pRes->info.rows, qcontinue); } if (numOfResBlock == 0 || (hasMore == false)) { From 5f68b65675d8509ccf79b213ab92ec5e6c39f74e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 11:20:37 +0800 Subject: [PATCH 10/47] fix: count elements number type issue --- include/libs/function/function.h | 2 +- source/libs/function/src/builtinsimpl.c | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/libs/function/function.h b/include/libs/function/function.h index fb6ef26a8a..aa5c78195a 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -112,7 +112,7 @@ typedef struct SResultDataInfo { typedef struct SInputColumnInfoData { int32_t totalRows; // total rows in current columnar data int32_t startRowIndex; // handle started row index - int32_t numOfRows; // the number of rows needs to be handled + int64_t numOfRows; // the number of rows needs to be handled int32_t numOfInputCols; // PTS is not included bool colDataSMAIsSet; // if agg is set or not SColumnInfoData *pPTS; // primary timestamp column diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3c9f2fe8ca..313ab93f8d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -494,8 +494,8 @@ bool getCountFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { - int32_t numOfElem = 0; +static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { + int64_t numOfElem = 0; /* * 1. column data missing (schema modified) causes pInputCol->hasNull == true. pInput->colDataSMAIsSet == true; @@ -528,7 +528,7 @@ static int32_t getNumOfElems(SqlFunctionCtx* pCtx) { * count function does not use the pCtx->interResBuf to keep the intermediate buffer */ int32_t countFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = 0; + int64_t numOfElem = 0; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SInputColumnInfoData* pInput = &pCtx->input; @@ -555,7 +555,7 @@ int32_t countFunction(SqlFunctionCtx* pCtx) { } int32_t countInvertFunction(SqlFunctionCtx* pCtx) { - int32_t numOfElem = getNumOfElems(pCtx); + int64_t numOfElem = getNumOfElems(pCtx); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); char* buf = GET_ROWCELL_INTERBUF(pResInfo); @@ -1929,7 +1929,7 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { SAPercentileInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - qDebug("%s total %d rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto); + qDebug("%s total %" PRId64 " rows will merge, %p", __FUNCTION__, pInput->numOfRows, pInfo->pHisto); int32_t start = pInput->startRowIndex; for (int32_t i = start; i < start + pInput->numOfRows; ++i) { From 2714ce2af830e260449963dbbd3d181debdc629d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 13:22:12 +0800 Subject: [PATCH 11/47] fix: rows count issue --- source/dnode/vnode/src/tsdb/tsdbRead.c | 77 ++++++++++++-------------- 1 file changed, 36 insertions(+), 41 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 63d9b702fd..3e41bd0d11 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3087,7 +3087,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { } -static int32_t doSumBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { +static int32_t doSumFileBlockRows(STsdbReader* pReader, SDataFReader* pFileReader) { int64_t st = taosGetTimestampUs(); LRUHandle* handle = NULL; int32_t code = tsdbCacheGetBlockIdx(pFileReader->pTsdb->biCache, pFileReader, &handle); @@ -3134,32 +3134,7 @@ _end: } -static int32_t readRowsCountFromFile(STsdbReader* pReader) { - int32_t code = TSDB_CODE_SUCCESS; - - while (1) { - bool hasNext = false; - int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); - if (code) { - return code; - } - - if (!hasNext) { // no data files on disk - break; - } - - code = doSumBlockRows(pReader, pReader->pFileReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } - - pReader->status.loadFromFile = false; - - return code; -} - -static int32_t readRowsCountFromStt(STsdbReader* pReader) { +static int32_t doSumSttBlockRows(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SSttBlockLoadInfo* pBlockLoadInfo = NULL; @@ -3167,17 +3142,12 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { for (int32_t i = 0; i < pReader->pFileReader->pSet->nSttF; ++i) { // open all last file pBlockLoadInfo = &pLastBlockReader->pInfo[i]; - if (!pLastBlockReader->pInfo[i].sttBlockLoaded) { - pLastBlockReader->pInfo[i].sttBlockLoaded = true; - - code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); - if (code) { - return code; - } + code = tsdbReadSttBlk(pReader->pFileReader, i, pBlockLoadInfo->aSttBlk); + if (code) { + return code; } size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); - if (size >= 1) { SSttBlk *pStart = taosArrayGet(pBlockLoadInfo->aSttBlk, 0); SSttBlk *pEnd = taosArrayGet(pBlockLoadInfo->aSttBlk, size - 1); @@ -3214,6 +3184,36 @@ static int32_t readRowsCountFromStt(STsdbReader* pReader) { return code; } +static int32_t readRowsCountFromFiles(STsdbReader* pReader) { + int32_t code = TSDB_CODE_SUCCESS; + + while (1) { + bool hasNext = false; + int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + if (code) { + return code; + } + + if (!hasNext) { // no data files on disk + break; + } + + code = doSumFileBlockRows(pReader, pReader->pFileReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + code = doSumSttBlockRows(pReader); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + + pReader->status.loadFromFile = false; + + return code; +} + static int32_t readRowsCountFromMem(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; int64_t memNum = 0, imemNum = 0; @@ -4597,12 +4597,7 @@ static bool tsdbReadRowsCountOnly(STsdbReader* pReader) { return false; } - code = readRowsCountFromFile(pReader); - if (code != TSDB_CODE_SUCCESS) { - return false; - } - - code = readRowsCountFromStt(pReader); + code = readRowsCountFromFiles(pReader); if (code != TSDB_CODE_SUCCESS) { return false; } From c1452851e39cf323865895cb72226a3f26ee4ec2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Wed, 29 Mar 2023 14:39:37 +0800 Subject: [PATCH 12/47] fix: error in determining whether last(t.*) is a selection function --- source/libs/parser/src/parTranslater.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 726790443e..7fb5b0ba40 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1634,13 +1634,15 @@ static bool isTableStar(SNode* pNode) { (0 == strcmp(((SColumnNode*)pNode)->colName, "*")); } +static bool isStarParam(SNode* pNode) { return isStar(pNode) || isTableStar(pNode); } + static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsMultiResFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; } if (SQL_CLAUSE_SELECT != pCxt->currClause) { SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); - if (isStar(pPara) || isTableStar(pPara)) { + if (isStarParam(pPara)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, "%s(*) is only supported in SELECTed list", pFunc->functionName); } @@ -1654,7 +1656,7 @@ static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFu static int32_t getMultiResFuncNum(SNodeList* pParameterList) { if (1 == LIST_LENGTH(pParameterList)) { - return isStar(nodesListGetNode(pParameterList, 0)) ? 2 : 1; + return isStarParam(nodesListGetNode(pParameterList, 0)) ? 2 : 1; } return LIST_LENGTH(pParameterList); } From 84eef542d5850f0a8a8852c81a998cf706a2dfda Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 15:45:18 +0800 Subject: [PATCH 13/47] fix: memory rows count issue --- source/dnode/vnode/src/tsdb/tsdbMemTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index 15fdbe2212..d0ff403bf7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -287,12 +287,12 @@ int64_t tsdbCountTbDataRows(STbData *pTbData) { int64_t rowsNum = 0; while (NULL != pNode) { + pNode = SL_GET_NODE_FORWARD(pNode, 0); if (pNode == pTbData->sl.pTail) { return rowsNum; } rowsNum++; - pNode = SL_GET_NODE_FORWARD(pNode, 0); } return rowsNum; From 4f06728c1797a065badfda71b217f5ec4f1f83e8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 15:48:11 +0800 Subject: [PATCH 14/47] fix: restore sl level --- source/dnode/vnode/src/vnd/vnodeCfg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 54d2eabfa4..c326c8bfac 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -29,7 +29,7 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, .tsdbCfg = {.precision = TSDB_TIME_PRECISION_MILLI, .update = 1, .compression = 2, - .slLevel = 1, + .slLevel = 5, .days = 14400, .minRows = 100, .maxRows = 4096, From 8b7622b5e76fdf96a2f938649676abf3d78215c7 Mon Sep 17 00:00:00 2001 From: dapan1121 <72057773+dapan1121@users.noreply.github.com> Date: Wed, 29 Mar 2023 15:51:49 +0800 Subject: [PATCH 15/47] Update tglobal.c --- source/common/src/tglobal.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index e97523df3d..aeeec1d61c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -1347,15 +1347,9 @@ void taosCleanupCfg() { } } -extern int32_t scanDebug; void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { int32_t flag = atoi(value); - if (1 == flag) { - scanDebug = 1; - } else { - scanDebug = 0; - } taosSetAllDebugFlag(flag, true); return; } From a90354cd928b3fbbc0b8ce07aba622ddbb88ab05 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 15:56:08 +0800 Subject: [PATCH 16/47] fix: restore debug code --- source/dnode/vnode/src/tsdb/tsdbRead.c | 39 +------------------------- 1 file changed, 1 insertion(+), 38 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index b9a06c787a..99e2e620e2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3064,9 +3064,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - return code; - } code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; @@ -3075,19 +3072,12 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - return code; - } // data in memory that are earlier than current file block // rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - return code; - } - // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); ASSERT(tsLast >= pBlock->maxKey.ts); @@ -3437,12 +3427,6 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { initBlockDumpInfo(pReader, pBlockIter); } else { if (pReader->status.pCurrentFileset->nSttF > 0) { - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - pReader->pResBlock->info.rows = pReader->rowsNum; - pReader->rowsNum = 0; - return TSDB_CODE_SUCCESS; - } - // data blocks in current file are exhausted, let's try the next file now SBlockData* pBlockData = &pReader->status.fileBlockData; if (pBlockData->uid != 0) { @@ -3457,17 +3441,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { code = initForFirstBlockInFile(pReader, pBlockIter); // error happens or all the data files are completely checked - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (READ_MODE_COUNT_ONLY == pReader->readMode && pReader->rowsNum > 0) { - pReader->pResBlock->info.rows = pReader->rowsNum; - pReader->rowsNum = 0; - return TSDB_CODE_SUCCESS; - } - - if (pReader->status.loadFromFile == false) { + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { return code; } @@ -3481,17 +3455,6 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } code = doBuildDataBlock(pReader); - if (READ_MODE_COUNT_ONLY == pReader->readMode) { - if (false == pReader->status.composedDataBlock && pDumpInfo->allDumped) { - pReader->rowsNum += pReader->pResBlock->info.rows; - pReader->pResBlock->info.rows = 0; - continue; - } else if (pReader->pResBlock->info.rows == 0 && pReader->rowsNum > 0) { - pReader->pResBlock->info.rows = pReader->rowsNum; - pReader->rowsNum = 0; - return TSDB_CODE_SUCCESS; - } - } } if (code != TSDB_CODE_SUCCESS) { From f61732dca32c228f0167de961e80012b2a85ac10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 29 Mar 2023 16:56:53 +0800 Subject: [PATCH 17/47] test: refine query cases --- tests/system-test/2-query/columnLenUpdated.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index d6940fde8b..ea01cd623c 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -147,9 +147,9 @@ class TDTestCase: tdSql.checkData(1, 1, '55555') - tdSql.query("create table stb (ts timestamp, f1 int) tags (tg1 binary(2))") + tdSql.query("create table stb (ts timestamp, f1 int, f2 binary(2)) tags (tg1 binary(2))") tdSql.query("create table tb1 using stb tags('bb')") - tdSql.query("insert into tb1 values (now, 2)") + tdSql.query("insert into tb1 values (now, 2,'22')") tdSql.query("select count(*) from stb group by tg1") tdSql.checkData(0, 0, 1) @@ -163,13 +163,23 @@ class TDTestCase: if retCode != "TAOS_OK": tdLog.exit("taos -s fail") - keyDict['s'] = "\"insert into db1.tb2 values (now, 2)\"" + keyDict['s'] = "\"insert into db1.tb2 values (now, 2,'22')\"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"alter table db1.stb modify column f2 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + keyDict['s'] = "\"insert into db1.tb2 values (now, 3,'55555')\"" retCode = taos_command(buildPath, "s", keyDict['s'], "Insert OK", '') if retCode != "TAOS_OK": tdLog.exit("taos -s fail") tdSql.query("select count(*) from stb group by tg1") - tdSql.checkData(0, 0, 1) + tdSql.checkData(0, 0, 2) tdSql.checkData(1, 0, 1) From 6a1150301eae3fa6c76784e99a0b6814a233654e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 17:30:20 +0800 Subject: [PATCH 18/47] fix: column rows size --- source/dnode/vnode/src/tq/tqScan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 1a166d326f..43e214137f 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -114,7 +114,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs pRsp->blockNum++; - tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%d, total blocks:%d", vgId, pHandle->consumerId, + tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%" PRId64 ", total blocks:%d", vgId, pHandle->consumerId, pDataBlock->info.rows, pRsp->blockNum); if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { From b25b6a8333c9280b4a616a4b1d4dae348667f304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Wed, 29 Mar 2023 18:16:46 +0800 Subject: [PATCH 19/47] test: refine query cases --- tests/parallel_test/cases.task | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 260d47032a..a50503e24b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -14,8 +14,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 @@ -24,7 +25,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py From 3810e57c1542c4deb332d6c20cafda120a7c99ea Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Mar 2023 19:10:10 +0800 Subject: [PATCH 20/47] fix: tag copy issue --- source/libs/executor/src/executil.c | 4 ++++ tests/parallel_test/cases.task | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 953d614951..0081ffcb0b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -571,6 +571,10 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis memcpy(pStart, data, len); pStart += len; } else if (IS_VAR_DATA_TYPE(pValue->info.type)) { + if (varDataTLen(data) > pValue->info.bytes) { + code = TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER; + goto end; + } memcpy(pStart, data, varDataTLen(data)); pStart += varDataTLen(data); } else { diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index a50503e24b..ab7540ee59 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -14,9 +14,9 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 @@ -25,7 +25,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -#,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py From 84d75c2ec09da7454ba83ab8d945ff46616f3126 Mon Sep 17 00:00:00 2001 From: slzhou Date: Wed, 29 Mar 2023 19:51:13 +0800 Subject: [PATCH 21/47] fix: join after interval --- source/libs/executor/src/timewindowoperator.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b01143841c..70febcaf6a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -128,8 +128,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn if (end >= 0) { forwardRows = end; - if (pData[end + pos] == ekey) { + while (pData[end + pos] == ekey) { forwardRows += 1; + ++pos; } } } else { @@ -137,8 +138,9 @@ FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn if (end >= 0) { forwardRows = end; - if (pData[end + pos] == ekey) { + while (pData[end + pos] == ekey) { forwardRows += 1; + ++pos; } } // int32_t end = searchFn((char*)pData, pos + 1, ekey, order); From d9b3c638c3a2a7c5e299793faa1c57b8c3c9ced4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Mar 2023 21:03:39 +0800 Subject: [PATCH 22/47] fix:add assert for debug --- source/common/src/tdataformat.c | 3 +++ source/libs/parser/src/parInsertUtil.c | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 34808aa389..9632750a18 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2453,6 +2453,9 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { + if(ASSERT(varDataTLen(data + offset) <= bytes)){ + uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + } code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), varDataLen(data + offset)); } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 132a3b2618..bbe36e0c80 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -656,6 +656,9 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[j]; } + if(ASSERT(pCol->nVal == numOfRows)){ + uError("tFields is null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + } } }else{ for (int i = 0; i < numFields; i++) { @@ -684,6 +687,9 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[i]; } + if(ASSERT(pCol->nVal == numOfRows)){ + uError("tFields is not null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + } boundInfo->pColIndex[j] = -1; break; } From c1b4f941997c039d0bfa7de110551c3c689843ca Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Mar 2023 22:12:13 +0800 Subject: [PATCH 23/47] fix:add assert for debug --- source/common/src/tdataformat.c | 2 ++ source/libs/parser/src/parInsertUtil.c | 6 ------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 9632750a18..d6ab974c6c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2455,6 +2455,8 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt } else { if(ASSERT(varDataTLen(data + offset) <= bytes)){ uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + code = TSDB_CODE_INVALID_PARA; + goto _exit; } code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, (uint8_t *)varDataVal(data + offset), varDataLen(data + offset)); diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index bbe36e0c80..132a3b2618 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -656,9 +656,6 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[j]; } - if(ASSERT(pCol->nVal == numOfRows)){ - uError("tFields is null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); - } } }else{ for (int i = 0; i < numFields; i++) { @@ -687,9 +684,6 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } else { pStart += colLength[i]; } - if(ASSERT(pCol->nVal == numOfRows)){ - uError("tFields is not null, pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); - } boundInfo->pColIndex[j] = -1; break; } From 06aa7d3750c74193711797bffcf3445598beb325 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 30 Mar 2023 08:54:00 +0800 Subject: [PATCH 24/47] fix: add test case for interval after join --- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/join_interval.sim | 42 +++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 tests/script/tsim/query/join_interval.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 260d47032a..cf2d62a2c3 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -868,6 +868,7 @@ ,,y,script,./test.sh -f tsim/query/session.sim ,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/query/udf_with_const.sim +,,y,script,./test.sh -f tsim/query/join_interval.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/event.sim diff --git a/tests/script/tsim/query/join_interval.sim b/tests/script/tsim/query/join_interval.sim new file mode 100644 index 0000000000..14994a5cc1 --- /dev/null +++ b/tests/script/tsim/query/join_interval.sim @@ -0,0 +1,42 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c udf -v 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step create databases +sql create database d1 +sql create database d2 +sql create table d1.t1(ts timestamp, i int) tags(t int); +sql create table d2.t1(ts timestamp, i int); +sql insert into d1.t11 using d1.t1 tags(1) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) +sql insert into d1.t12 using d1.t1 tags(2) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) +sql insert into d1.t13 using d1.t1 tags(3) values(1500000000000, 0)(1500000000001, 1)(1500000000002,2)(1500000000003,3)(1500000000004,4) + +sql insert into d2.t1 values(1500000000000,0)(1500000000001,1)(1500000000002,2) + +sql select _wstart,_wend,count((a.ts)),count(b.ts) from d1.t1 a, d2.t1 b where a.ts is not null and a.ts = b.ts interval(1a) ; +if $data02 != 3 then + return -1 +endi + +if $data03 != 3 then + return -1 +endi + +if $data12 != 3 then + return -1 +endi + +if $data13 != 3 then + return -1 +endi +if $data22 != 3 then + return -1 +endi + +if $data23 != 3 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGINT From f93efc8538f4c4eeebc8d5b0ae93492776b0ac7a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 09:21:34 +0800 Subject: [PATCH 25/47] fix:telemetry.py failed in windows --- tests/system-test/0-others/telemetry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/telemetry.py b/tests/system-test/0-others/telemetry.py index bc5d276faa..c62e3c2487 100644 --- a/tests/system-test/0-others/telemetry.py +++ b/tests/system-test/0-others/telemetry.py @@ -181,7 +181,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "8" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) From a5c8713517f751a0999527c0d08499308d9cf8a4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 09:40:16 +0800 Subject: [PATCH 26/47] opti:disable set enable.heartbeat.background --- source/client/src/clientTmq.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 111ca28cdc..9e585f2957 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -320,15 +320,16 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } if (strcasecmp(key, "enable.heartbeat.background") == 0) { - if (strcasecmp(value, "true") == 0) { - conf->hbBgEnable = true; - return TMQ_CONF_OK; - } else if (strcasecmp(value, "false") == 0) { - conf->hbBgEnable = false; - return TMQ_CONF_OK; - } else { +// if (strcasecmp(value, "true") == 0) { +// conf->hbBgEnable = true; +// return TMQ_CONF_OK; +// } else if (strcasecmp(value, "false") == 0) { +// conf->hbBgEnable = false; +// return TMQ_CONF_OK; +// } else { + tscError("the default value of enable.heartbeat.background is true, can not be seted"); return TMQ_CONF_INVALID; - } +// } } if (strcasecmp(key, "td.connect.ip") == 0) { From 347c08bfeac0fe28c28adf46e300785e81d81ba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:17:52 +0800 Subject: [PATCH 27/47] test: refine query cases --- tests/parallel_test/cases.task | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ab7540ee59..fcece56816 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -10,22 +10,21 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/columnLenUpdated.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -47,6 +46,11 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/out_of_order.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/nestedQuery_26.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3 From 2e8e5a98e9125e3729c4956266c19289b3019470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:05 +0800 Subject: [PATCH 28/47] test: refine query cases --- tests/system-test/2-query/nestedQuery.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/system-test/2-query/nestedQuery.py b/tests/system-test/2-query/nestedQuery.py index 6557aad05f..1b843defce 100755 --- a/tests/system-test/2-query/nestedQuery.py +++ b/tests/system-test/2-query/nestedQuery.py @@ -6144,7 +6144,7 @@ class TDTestCase: startTime = time.time() - self.function_before_26() + #self.function_before_26() self.math_nest(['UNIQUE']) self.math_nest(['MODE']) @@ -6157,9 +6157,9 @@ class TDTestCase: # self.math_nest(['MAVG']) # self.math_nest(['HYPERLOGLOG']) # self.math_nest(['TAIL']) - # self.math_nest(['CSUM']) - # self.math_nest(['statecount','stateduration']) - # self.math_nest(['HISTOGRAM']) + self.math_nest(['CSUM']) + self.math_nest(['statecount','stateduration']) + self.math_nest(['HISTOGRAM']) # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) # self.str_nest(['LENGTH','CHAR_LENGTH']) From 01ae6da8a45c2d2e1af2fecb1d2d9d33e687e062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:22 +0800 Subject: [PATCH 29/47] test: refine query cases --- tests/system-test/2-query/nestedQuery_math.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/nestedQuery_math.py b/tests/system-test/2-query/nestedQuery_math.py index 2d0bbcb352..3e37a2c15e 100755 --- a/tests/system-test/2-query/nestedQuery_math.py +++ b/tests/system-test/2-query/nestedQuery_math.py @@ -34,9 +34,9 @@ class TDTestCase(TDTestCase): self.math_nest(['MAVG']) self.math_nest(['HYPERLOGLOG']) self.math_nest(['TAIL']) - self.math_nest(['CSUM']) - self.math_nest(['statecount','stateduration']) - self.math_nest(['HISTOGRAM']) + # self.math_nest(['CSUM']) + # self.math_nest(['statecount','stateduration']) + # self.math_nest(['HISTOGRAM']) # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) # self.str_nest(['LENGTH','CHAR_LENGTH']) From 8825b8931441684d7ee070629ea4df0babd52df8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:36 +0800 Subject: [PATCH 30/47] test: refine query cases --- tests/system-test/2-query/out_of_order.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py index 148b89fc58..47a9cc3c11 100644 --- a/tests/system-test/2-query/out_of_order.py +++ b/tests/system-test/2-query/out_of_order.py @@ -162,19 +162,18 @@ class TDTestCase: sql = "select count(*) from (select distinct(tbname) from %s.meters)" %dbname tdSql.query(sql) - num = tdSql.getData(0,0) + # 目前不需要了 + # num = tdSql.getData(0,0) - for i in range(0,num): - sql1 = "select count(*) from %s.d%d" %(dbname,i) - tdSql.query(sql1) - sql1_result = tdSql.getData(0,0) - tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + # for i in range(0,num): + # sql1 = "select count(*) from %s.d%d" %(dbname,i) + # tdSql.query(sql1) + # sql1_result = tdSql.getData(0,0) + # tdLog.info("sql:%s , result: %s" %(sql1,sql1_result)) + def check_out_of_order(self,dbname,tables,per_table_num,order,replica): self.run_benchmark(dbname,tables,per_table_num,order,replica) - print("sleep 10 seconds") - #time.sleep(10) - print("sleep 10 seconds finish") self.run_sql(dbname) From cf99c2e69dc207fd80e81dee5cf0872b7739004a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:18:53 +0800 Subject: [PATCH 31/47] test: refine query cases --- tests/system-test/2-query/nestedQuery_26.py | 76 +++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100755 tests/system-test/2-query/nestedQuery_26.py diff --git a/tests/system-test/2-query/nestedQuery_26.py b/tests/system-test/2-query/nestedQuery_26.py new file mode 100755 index 0000000000..9d5f31d1e0 --- /dev/null +++ b/tests/system-test/2-query/nestedQuery_26.py @@ -0,0 +1,76 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- +from util.cases import tdCases +from .nestedQuery import * + +class TDTestCase(TDTestCase): + + + def run(self): + tdSql.prepare() + + startTime = time.time() + + self.function_before_26() + + # self.math_nest(['UNIQUE']) + # self.math_nest(['MODE']) + # self.math_nest(['SAMPLE']) + + # self.math_nest(['ABS','SQRT']) + # self.math_nest(['SIN','COS','TAN','ASIN','ACOS','ATAN']) + # self.math_nest(['POW','LOG']) + # self.math_nest(['FLOOR','CEIL','ROUND']) + # self.math_nest(['MAVG']) + # self.math_nest(['HYPERLOGLOG']) + # self.math_nest(['TAIL']) + # self.math_nest(['CSUM']) + # self.math_nest(['statecount','stateduration']) + # self.math_nest(['HISTOGRAM']) + + # self.str_nest(['LTRIM','RTRIM','LOWER','UPPER']) + # self.str_nest(['LENGTH','CHAR_LENGTH']) + # self.str_nest(['SUBSTR']) + # self.str_nest(['CONCAT']) + # self.str_nest(['CONCAT_WS']) + # self.time_nest(['CAST']) #放到time里起来弄 + # self.time_nest(['CAST_1']) + # self.time_nest(['CAST_2']) + # self.time_nest(['CAST_3']) + # self.time_nest(['CAST_4']) + + + + # self.time_nest(['NOW','TODAY']) + # self.time_nest(['TIMEZONE']) + # self.time_nest(['TIMETRUNCATE']) + # self.time_nest(['TO_ISO8601']) + # self.time_nest(['TO_UNIXTIMESTAMP']) + # self.time_nest(['ELAPSED']) + #self.time_nest(['TIMEDIFF_1']) + #self.time_nest(['TIMEDIFF_2']) + + + endTime = time.time() + print("total time %ds" % (endTime - startTime)) + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 2aeda10174be4cfb674009683710db6b93220843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Thu, 30 Mar 2023 10:22:17 +0800 Subject: [PATCH 32/47] test: refine query cases --- tests/system-test/2-query/out_of_order.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/out_of_order.py b/tests/system-test/2-query/out_of_order.py index 47a9cc3c11..ff2b71193b 100644 --- a/tests/system-test/2-query/out_of_order.py +++ b/tests/system-test/2-query/out_of_order.py @@ -181,7 +181,7 @@ class TDTestCase: startTime = time.time() #self.check_out_of_order('db1',10,random.randint(10000,50000),random.randint(1,10),1) - self.check_out_of_order('db1',random.randint(50,200),random.randint(10000,20000),random.randint(1,5),1) + self.check_out_of_order('db1',random.randint(50,100),random.randint(10000,20000),random.randint(1,5),1) # self.check_out_of_order('db2',random.randint(50,200),random.randint(10000,50000),random.randint(5,50),1) From 07171a01b4da0962919ddf8dca19e31181655520 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 10:52:39 +0800 Subject: [PATCH 33/47] fix:taosdMonitor.py failed in windows --- tests/system-test/0-others/taosdMonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/0-others/taosdMonitor.py b/tests/system-test/0-others/taosdMonitor.py index 944ff52d5b..195f1ba5bc 100644 --- a/tests/system-test/0-others/taosdMonitor.py +++ b/tests/system-test/0-others/taosdMonitor.py @@ -292,7 +292,7 @@ class TDTestCase: def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() # time.sleep(2) - vgroups = "30" + vgroups = "8" sql = "create database db3 vgroups " + vgroups tdSql.query(sql) sql = "create table db3.stb (ts timestamp, f int) tags (t int)" From f30cb8e52722b1b2edf74bb7e3bab69d2cb5aa73 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 30 Mar 2023 10:58:33 +0800 Subject: [PATCH 34/47] fix: when the set operator statement is used for subquery, the outer layer filtering fails. --- source/libs/planner/src/planPhysiCreater.c | 13 +++++++++++-- source/libs/planner/src/planSpliter.c | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index e9a2efaac7..e2c2e4c655 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1089,9 +1089,15 @@ static int32_t doCreateExchangePhysiNode(SPhysiPlanContext* pCxt, SExchangeLogic pExchange->srcStartGroupId = pExchangeLogicNode->srcStartGroupId; pExchange->srcEndGroupId = pExchangeLogicNode->srcEndGroupId; pExchange->seqRecvData = pExchangeLogicNode->seqRecvData; - *pPhyNode = (SPhysiNode*)pExchange; - return TSDB_CODE_SUCCESS; + int32_t code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pExchange); + if (TSDB_CODE_SUCCESS == code) { + *pPhyNode = (SPhysiNode*)pExchange; + } else { + nodesDestroyNode((SNode*)pExchange); + } + + return code; } static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExchangeLogicNode* pExchangeLogicNode, @@ -1119,6 +1125,9 @@ static int32_t createStreamScanPhysiNodeByExchange(SPhysiPlanContext* pCxt, SExc if (TSDB_CODE_SUCCESS == code) { code = addDataBlockSlots(pCxt, pScan->pScanCols, pScan->node.pOutputDataBlockDesc); } + if (TSDB_CODE_SUCCESS == code) { + code = setConditionsSlotId(pCxt, (const SLogicNode*)pExchangeLogicNode, (SPhysiNode*)pScan); + } if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pScan; diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 361cf33d58..4a3d689f04 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1365,7 +1365,8 @@ static int32_t unAllSplCreateExchangeNode(SSplitContext* pCxt, int32_t startGrou pExchange->srcEndGroupId = pCxt->groupId - 1; pExchange->node.precision = pProject->node.precision; pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); - if (NULL == pExchange->node.pTargets) { + pExchange->node.pConditions = nodesCloneNode(pProject->node.pConditions); + if (NULL == pExchange->node.pTargets || NULL == pExchange->node.pConditions) { return TSDB_CODE_OUT_OF_MEMORY; } TSWAP(pExchange->node.pLimit, pProject->node.pLimit); From 8f3c41fc709d38a9735d644fa8cfd1759fcfc65a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 30 Mar 2023 11:03:40 +0800 Subject: [PATCH 35/47] fix: add more column update cases --- tests/system-test/2-query/columnLenUpdated.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/system-test/2-query/columnLenUpdated.py b/tests/system-test/2-query/columnLenUpdated.py index ea01cd623c..3b87cbe22a 100644 --- a/tests/system-test/2-query/columnLenUpdated.py +++ b/tests/system-test/2-query/columnLenUpdated.py @@ -147,6 +147,30 @@ class TDTestCase: tdSql.checkData(1, 1, '55555') + + keyDict['s'] = "\"alter table db1.tba add column f2 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select * from tba order by ts") + tdSql.query("select * from tba order by ts") + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + + + + + keyDict['s'] = "\"alter table db1.tba add column f3 binary(5) \"" + retCode = taos_command(buildPath, "s", keyDict['s'], "Query OK", '') + if retCode != "TAOS_OK": + tdLog.exit("taos -s fail") + + tdSql.query("select f3 from tba order by ts") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + + tdSql.query("create table stb (ts timestamp, f1 int, f2 binary(2)) tags (tg1 binary(2))") tdSql.query("create table tb1 using stb tags('bb')") tdSql.query("insert into tb1 values (now, 2,'22')") From b7e6e4197dbc125093759eaea41afa6095803e6c Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 11:18:35 +0800 Subject: [PATCH 36/47] opti:disable set enable.heartbeat.background --- docs/en/07-develop/07-tmq.mdx | 3 --- docs/examples/go/sub/main.go | 1 - docs/zh/07-develop/07-tmq.mdx | 3 --- tests/system-test/0-others/tmqBasic.json | 1 - utils/test/c/tmq_taosx_ci.c | 1 - 5 files changed, 9 deletions(-) diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index c85109d3c5..dc0f07d3d5 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -293,7 +293,6 @@ You configure the following parameters when creating a consumer: | `auto.offset.reset` | enum | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | | `enable.auto.commit` | boolean | Commit automatically | Specify `true` or `false`. | | `auto.commit.interval.ms` | integer | Interval for automatic commits, in milliseconds | -| `enable.heartbeat.background` | boolean | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | | | `experimental.snapshot.enable` | boolean | Specify whether to consume messages from the WAL or from TSBS | | | `msg.with.table.name` | boolean | Specify whether to deserialize table names from messages | @@ -368,7 +367,6 @@ conf := &tmq.ConfigMap{ "td.connect.port": "6030", "client.id": "test_tmq_c", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", } @@ -418,7 +416,6 @@ Python programs use the following parameters: | `auto.commit.interval.ms` | string | Interval for automatic commits, in milliseconds | | | `auto.offset.reset` | string | Initial offset for the consumer group | Specify `earliest`, `latest`, or `none`(default) | | `experimental.snapshot.enable` | string | Specify whether to consume messages from the WAL or from TSDB | Specify `true` or `false` | -| `enable.heartbeat.background` | string | Backend heartbeat; if enabled, the consumer does not go offline even if it has not polled for a long time | Specify `true` or `false` | diff --git a/docs/examples/go/sub/main.go b/docs/examples/go/sub/main.go index 1f7218936f..334b8f290b 100644 --- a/docs/examples/go/sub/main.go +++ b/docs/examples/go/sub/main.go @@ -35,7 +35,6 @@ func main() { "td.connect.port": "6030", "client.id": "test_tmq_client", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", }) diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index fb171042d9..5771e5053a 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -291,7 +291,6 @@ CREATE TOPIC topic_name AS DATABASE db_name; | `auto.offset.reset` | enum | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` | | `enable.auto.commit` | boolean | 是否启用消费位点自动提交 | 合法值：`true`, `false`。 | | `auto.commit.interval.ms` | integer | 以毫秒为单位的消费记录自动提交消费位点时间间 | 默认 5000 m | -| `enable.heartbeat.background` | boolean | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 默认开启 | | `experimental.snapshot.enable` | boolean | 是否允许从 TSDB 消费数据 | 实验功能，默认关闭 | | `msg.with.table.name` | boolean | 是否允许从消息中解析表名, 不适用于列订阅（列订阅时可将 tbname 作为列写入 subquery 语句） | | @@ -366,7 +365,6 @@ conf := &tmq.ConfigMap{ "td.connect.port": "6030", "client.id": "test_tmq_c", "enable.auto.commit": "false", - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "true", } @@ -418,7 +416,6 @@ consumer = Consumer({"group.id": "local", "td.connect.ip": "127.0.0.1"}) | `auto.commit.interval.ms` | string | 以毫秒为单位的自动提交时间间隔 | 默认值：5000 ms | | `auto.offset.reset` | string | 消费组订阅的初始位置 | 可选：`earliest`(default), `latest`, `none` | | `experimental.snapshot.enable` | string | 是否允许从 TSDB 消费数据 | 合法值：`true`, `false` | -| `enable.heartbeat.background` | string | 启用后台心跳，启用后即使长时间不 poll 消息也不会造成离线 | 合法值：`true`, `false` | diff --git a/tests/system-test/0-others/tmqBasic.json b/tests/system-test/0-others/tmqBasic.json index 24e815708a..d716bff3ac 100644 --- a/tests/system-test/0-others/tmqBasic.json +++ b/tests/system-test/0-others/tmqBasic.json @@ -14,7 +14,6 @@ "auto.offset.reset": "earliest", "enable.auto.commit": "true", "auto.commit.interval.ms": 1000, - "enable.heartbeat.background": "true", "experimental.snapshot.enable": "true", "msg.with.table.name": "false", "topic_list": [ diff --git a/utils/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c index 1f25eae366..6661d6d14a 100644 --- a/utils/test/c/tmq_taosx_ci.c +++ b/utils/test/c/tmq_taosx_ci.c @@ -542,7 +542,6 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); - tmq_conf_set(conf, "enable.heartbeat.background", "true"); if (g_conf.snapShot) { tmq_conf_set(conf, "experimental.snapshot.enable", "true"); From 10979f6fc28199c6f6af9906599a8a140a8fef35 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 30 Mar 2023 13:34:35 +0800 Subject: [PATCH 37/47] fix: when the set operator statement is used for subquery, the outer layer filtering fails. --- source/libs/planner/src/planSpliter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 4a3d689f04..fd77261818 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -1366,7 +1366,7 @@ static int32_t unAllSplCreateExchangeNode(SSplitContext* pCxt, int32_t startGrou pExchange->node.precision = pProject->node.precision; pExchange->node.pTargets = nodesCloneList(pProject->node.pTargets); pExchange->node.pConditions = nodesCloneNode(pProject->node.pConditions); - if (NULL == pExchange->node.pTargets || NULL == pExchange->node.pConditions) { + if (NULL == pExchange->node.pTargets || (NULL != pProject->node.pConditions && NULL == pExchange->node.pConditions)) { return TSDB_CODE_OUT_OF_MEMORY; } TSWAP(pExchange->node.pLimit, pProject->node.pLimit); From 32c262acc41a0d0d2cf0268237b8c1e559ef3ad8 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Mar 2023 16:37:26 +0800 Subject: [PATCH 38/47] test: modify tmq test case --- tests/system-test/7-tmq/tmq3mnodeSwitch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index cdcdadbcbb..9ded8fc942 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -211,7 +211,7 @@ class TDTestCase: # init consume info, and start tmq_sim, then check consume result tdLog.info("insert consume info to consume processor") consumerId = 0 - expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] * 2 # because taosd switch, may be consume duplication data topicList = topicNameList[0] ifcheckdata = 1 ifManualCommit = 1 @@ -251,11 +251,12 @@ class TDTestCase: expectRows = 1 resultList = tmqCom.selectConsumeResult(expectRows) - if expectRowsList[0] != resultList[0]: - tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.info("expect consume rows: %d should less/equal than act consume rows: %d"%(expectRowsList[0], resultList[0])) + if expectRowsList[0] <= resultList[0]: tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + if expectRowsList[0] == resultList[0]: + self.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): From 8c9580f93e6f19108c7ef9f7a4b801d401718ab5 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Mar 2023 16:38:43 +0800 Subject: [PATCH 39/47] test: modify tmq test case --- tests/system-test/7-tmq/tmq3mnodeSwitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 9ded8fc942..7c95f7a3db 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -252,7 +252,7 @@ class TDTestCase: resultList = tmqCom.selectConsumeResult(expectRows) tdLog.info("expect consume rows: %d should less/equal than act consume rows: %d"%(expectRowsList[0], resultList[0])) - if expectRowsList[0] <= resultList[0]: + if expectRowsList[0] > resultList[0]: tdLog.exit("0 tmq consume rows error!") if expectRowsList[0] == resultList[0]: From ae98ad43a5b4ea5daf3ddf63e642fc612ce4a336 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 30 Mar 2023 16:35:56 +0800 Subject: [PATCH 40/47] fix:state window return wrong block type --- source/libs/executor/src/scanoperator.c | 10 ++++ tests/script/tsim/stream/state1.sim | 66 +++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 84317c825b..1e5c6c2168 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1829,6 +1829,15 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { printDataBlock(pInfo->pUpdateRes, "recover update"); return pInfo->pUpdateRes; } break; + case STREAM_SCAN_FROM_DELETE_DATA: { + generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + printDataBlock(pInfo->pDeleteDataRes, "recover delete"); + return pInfo->pDeleteDataRes; + } break; case STREAM_SCAN_FROM_DATAREADER_RANGE: { SSDataBlock* pSDB = doRangeScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB) { @@ -2021,6 +2030,7 @@ FETCH_NEXT_BLOCK: copyDataBlock(pInfo->pUpdateRes, pSup->pScanBlock); blockDataCleanup(pSup->pScanBlock); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; return pInfo->pUpdateRes; } diff --git a/tests/script/tsim/stream/state1.sim b/tests/script/tsim/stream/state1.sim index 2ae5739642..67e02c0890 100644 --- a/tests/script/tsim/stream/state1.sim +++ b/tests/script/tsim/stream/state1.sim @@ -4,6 +4,7 @@ system sh/exec.sh -n dnode1 -s start sleep 50 sql connect +print step 1 print =============== create database sql create database test vgroups 4; sql select * from information_schema.ins_databases; @@ -33,8 +34,8 @@ if $loop_count == 10 then endi sql select * from streamt1; -print data00 data01 -print data10 data11 +print $data00 $data01 +print $data10 $data11 if $rows != 0 then print =====rows=$rows @@ -52,8 +53,8 @@ if $loop_count == 10 then endi sql select * from streamt1; -print data00 data01 -print data10 data11 +print $data00 $data01 +print $data10 $data11 if $rows != 1 then print =====rows=$rows @@ -92,9 +93,64 @@ endi sql select * from streamt1; if $rows != 2 then print =====rows=$rows - goto loop2 + goto loop3 endi +print step 1 over +print step 2 + +sql create database test2 vgroups 1; +sql use test2; +sql create table t1(ts timestamp, a int, b int , c int, d double); +print create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a) +sql create stream streams2 trigger at_once watermark 1000s into streamt2 as select _wstart, count(*) c1, count(d) c2 from t1 partition by b state_window(a); + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791213010,1,2,3,1.1); + +$loop_count = 0 +loop4: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt2; +print $data00 $data01 +print $data10 $data11 + +if $rows != 1 then + print =====rows=$rows + goto loop4 +endi + +print insert into t1 values(1648791213005,2,2,3,1.1) +sql insert into t1 values(1648791213005,2,2,3,1.1); + +$loop_count = 0 +loop5: + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +print select * from streamt2 +sql select * from streamt2; +print $data00 $data01 +print $data10 $data11 +print $data20 $data21 +print $data30 $data31 + +if $rows != 3 then + print =====rows=$rows + goto loop5 +endi + +print step 2 over print state1 end From a3a3dc824355c4ff54a2f39d658181ffd4ebb3bb Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Thu, 30 Mar 2023 16:17:34 +0800 Subject: [PATCH 41/47] fix: add filter to exchange node for union-all subquery --- source/libs/executor/src/exchangeoperator.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 3bdecbe748..8caebd2a34 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -212,6 +212,11 @@ static SSDataBlock* loadRemoteData(SOperatorInfo* pOperator) { return NULL; } + doFilter(pBlock, pOperator->exprSupp.pFilterInfo, NULL); + if (blockDataGetNumOfRows(pBlock) == 0) { + continue; + } + SLimitInfo* pLimitInfo = &pExchangeInfo->limitInfo; if (hasLimitOffsetInfo(pLimitInfo)) { int32_t status = handleLimitOffset(pOperator, pLimitInfo, pBlock, false); @@ -303,6 +308,11 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pDummyBlock->pDataBlock); + code = filterInitFromNode((SNode*)pExNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, loadRemoteData, NULL, destroyExchangeOperatorInfo, optrDefaultBufFn, NULL); return pOperator; From b2ca68943c898c10b904e9c74abea1c9d7f077b7 Mon Sep 17 00:00:00 2001 From: slzhou Date: Thu, 30 Mar 2023 16:58:21 +0800 Subject: [PATCH 42/47] fix: add test case --- tests/parallel_test/cases.task | 1 + tests/script/tsim/query/unionall_as_table.sim | 28 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/script/tsim/query/unionall_as_table.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 4aa751ae7e..49f14191b1 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -874,6 +874,7 @@ ,,y,script,./test.sh -f tsim/query/udf.sim ,,y,script,./test.sh -f tsim/query/udf_with_const.sim ,,y,script,./test.sh -f tsim/query/join_interval.sim +,,y,script,./test.sh -f tsim/query/unionall_as_table.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/event.sim diff --git a/tests/script/tsim/query/unionall_as_table.sim b/tests/script/tsim/query/unionall_as_table.sim new file mode 100644 index 0000000000..dc3d2cbec4 --- /dev/null +++ b/tests/script/tsim/query/unionall_as_table.sim @@ -0,0 +1,28 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; +sql CREATE STABLE bw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql CREATE STABLE aw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql CREATE STABLE dw_yc_h_substation_mea (ts TIMESTAMP, create_date VARCHAR(50), create_time VARCHAR(30), load_time TIMESTAMP, sum_p_value FLOAT, sum_sz_value FLOAT, sum_gl_ys FLOAT, sum_g_value FLOAT) TAGS (id VARCHAR(50), name NCHAR(200), datasource VARCHAR(50), sys_flag VARCHAR(50)); +sql insert into t1 using dw_yc_h_substation_mea tags('1234567890','testa','0021001','abc01') values(now,'2023-03-27','00:01:00',now,2.3,3.3,4.4,5.5); +sql insert into t2 using dw_yc_h_substation_mea tags('2234567890','testb','0022001','abc02') values(now,'2023-03-27','00:01:00',now,2.3,2.3,2.4,2.5); +sql insert into t3 using aw_yc_h_substation_mea tags('2234567890','testc','0023001','abc03') values(now,'2023-03-27','00:15:00',now,2.3,2.3,2.4,2.5); +sql insert into t4 using bw_yc_h_substation_mea tags('4234567890','testd','0021001','abc03') values(now,'2023-03-27','00:45:00',now,2.3,2.3,2.4,2.5); +sql insert into t5 using bw_yc_h_substation_mea tags('5234567890','testd','0021001','abc03') values(now,'2023-03-27','00:00:00',now,2.3,2.3,2.4,2.5); +sql select t.ts,t.id,t.name,t.create_date,t.create_time,t.datasource,t.sum_p_value from (select ts,id,name,create_date,create_time,datasource,sum_p_value from bw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from aw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45') union all select ts,id,name,create_date,create_time,datasource,sum_p_value from dw_yc_h_substation_mea where create_date='2023-03-27' and substr(create_time,4,2) in ('00','15','30','45')) t where t.datasource='0021001' and t.id='4234567890' order by t.create_time; + +if $rows != 1 then + return -1 +endi +if $data01 != @4234567890@ then + return -1 +endi +if $data05 != @0021001@ then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From a8da9f31e70ec91cae7878501593f24d51bd4925 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 30 Mar 2023 18:04:32 +0800 Subject: [PATCH 43/47] fix: join query error --- source/libs/function/src/builtins.c | 4 ++-- source/libs/parser/src/parTranslater.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 629e846dae..1a039ebda6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -2457,7 +2457,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "interp", .type = FUNCTION_TYPE_INTERP, .classification = FUNC_MGT_TIMELINE_FUNC | FUNC_MGT_INTERVAL_INTERPO_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC | - FUNC_MGT_FORBID_STREAM_FUNC, + FUNC_MGT_FORBID_STREAM_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateInterp, .getEnvFunc = getSelectivityFuncEnv, .initFunc = functionSetup, @@ -3278,7 +3278,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_irowts", .type = FUNCTION_TYPE_IROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7fb5b0ba40..f58a66af18 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -757,7 +757,7 @@ static bool isPrimaryKeyImpl(SNode* pExpr) { if (FUNCTION_TYPE_SELECT_VALUE == pFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pFunc->funcType || FUNCTION_TYPE_FIRST == pFunc->funcType || FUNCTION_TYPE_LAST == pFunc->funcType) { return isPrimaryKeyImpl(nodesListGetNode(pFunc->pParameterList, 0)); - } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType) { + } else if (FUNCTION_TYPE_WSTART == pFunc->funcType || FUNCTION_TYPE_WEND == pFunc->funcType || FUNCTION_TYPE_IROWTS == pFunc->funcType) { return true; } } From 627cfb230bc0313f1d7799983ccee474b6e7ebfb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 30 Mar 2023 18:35:49 +0800 Subject: [PATCH 44/47] chore: update taos-tools d194dc9 for main (#20713) --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 3f7a43ab2d..aef89a2d42 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG e82b9fc + GIT_TAG d194dc9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 7e30f8619c9acb87c7aac837fdfbf5e83a3d3f43 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Mar 2023 18:39:50 +0800 Subject: [PATCH 45/47] fix(query): fix group_key is processed in selectivity twice --- source/libs/function/src/builtinsimpl.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3c9f2fe8ca..f8714a53c0 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -871,6 +871,12 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + // group_key function has its own process function + // do not process there + if (fmIsGroupKeyFunc(pc->functionId)) { + continue; + } + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); if (nullList[j]) { colDataSetNULL(pDstCol, rowIndex); @@ -3091,6 +3097,12 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid for (int32_t i = 0; i < pSubsidiaryies->num; ++i) { SqlFunctionCtx* pc = pSubsidiaryies->pCtx[i]; + // group_key function has its own process function + // do not process there + if (fmIsGroupKeyFunc(pc->functionId)) { + continue; + } + SFunctParam* pFuncParam = &pc->pExpr->base.pParam[0]; int32_t srcSlotId = pFuncParam->pCol->slotId; From a9cc66ba8f32380f9f9a23382f125dcf62a56815 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Mar 2023 19:48:21 +0800 Subject: [PATCH 46/47] fix:int64 parse error in schemaless --- source/client/src/clientSml.c | 42 +++++++++++++---------------------- utils/test/c/sml_test.c | 4 ++-- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 1719759822..a4ecc3c3df 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -257,19 +257,14 @@ cleanup: kvVal->f = (float)result; #define SET_BIGINT \ - if (smlDoubleToInt64OverFlow(result)) { \ - errno = 0; \ - int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ - if (errno == ERANGE) { \ - smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = tmp; \ - return true; \ - } \ - kvVal->type = TSDB_DATA_TYPE_BIGINT; \ - kvVal->i = (int64_t)result; + errno = 0; \ + int64_t tmp = taosStr2Int64(pVal, &endptr, 10); \ + if (errno == ERANGE) { \ + smlBuildInvalidDataMsg(msg, "big int out of range[-9223372036854775808,9223372036854775807]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_BIGINT; \ + kvVal->i = tmp; #define SET_INT \ if (!IS_VALID_INT(result)) { \ @@ -288,19 +283,14 @@ cleanup: kvVal->i = result; #define SET_UBIGINT \ - if (result >= (double)UINT64_MAX || result < 0) { \ - errno = 0; \ - uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ - if (errno == ERANGE || result < 0) { \ - smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ - return false; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = tmp; \ - return true; \ - } \ - kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ - kvVal->u = result; + errno = 0; \ + uint64_t tmp = taosStr2UInt64(pVal, &endptr, 10); \ + if (errno == ERANGE || result < 0) { \ + smlBuildInvalidDataMsg(msg, "unsigned big int out of range[0,18446744073709551615]", pVal); \ + return false; \ + } \ + kvVal->type = TSDB_DATA_TYPE_UBIGINT; \ + kvVal->u = tmp; #define SET_UINT \ if (!IS_VALID_UINT(result)) { \ diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index 873946121b..44fbd060be 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -850,7 +850,7 @@ int smlProcess_18784_Test() { taos_free_result(pRes); const char *sql[] = { - "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000", + "disk,device=sdc inodes_used=176059i,total=1076048383523889174i 1661943960000000000", "disk,device=sdc inodes_free=66932805i 1661943960000000000", }; pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); @@ -875,7 +875,7 @@ int smlProcess_18784_Test() { if (rowIndex == 0) { ASSERT(ts == 1661943960000); ASSERT(used == 176059); - ASSERT(total == 1081101176832); + ASSERT(total == 1076048383523889174); ASSERT(freed == 66932805); // ASSERT_EQ(latitude, 24.5208); // ASSERT_EQ(longitude, 28.09377); From 79784491139ea068bedfa9f9275ce796d5eb3f48 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Fri, 31 Mar 2023 09:41:20 +0800 Subject: [PATCH 47/47] enh(taosAdapter): TMQ parameter adjustment (#20710) --- cmake/taosadapter_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 1c401ae80e..b2f335e1f7 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG d8059ff + GIT_TAG cb1e89c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE