From 5c729cc836f2d9eac23bceb83c9b26c43dd541a4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 25 Mar 2024 17:57:59 +0800 Subject: [PATCH 1/9] refactor: do some internal refactor. --- source/dnode/vnode/src/inc/tsdb.h | 6 +- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 37 ++++++++++-- source/dnode/vnode/src/tsdb/tsdbRead2.c | 62 ++++++++++++++------- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 47 +++++++++++----- source/dnode/vnode/src/tsdb/tsdbReadUtil.h | 5 +- 5 files changed, 117 insertions(+), 40 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2b806e95a5..761e74b9cf 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -836,6 +836,8 @@ struct SLDataIter { STimeWindow timeWindow; SVersionRange verRange; SSttBlockLoadInfo *pBlockLoadInfo; + SRowKey startRowKey; // current row key + __compar_fn_t comparFn; bool ignoreEarlierTs; struct SSttFileReader *pReader; }; @@ -846,7 +848,7 @@ struct SSttFileReader; typedef int32_t (*_load_tomb_fn)(STsdbReader *pReader, struct SSttFileReader *pSttFileReader, SSttBlockLoadInfo *pLoadInfo); -typedef struct { +typedef struct SMergeTreeConf { int8_t backward; STsdb *pTsdb; uint64_t suid; @@ -859,7 +861,9 @@ typedef struct { STSchema *pSchema; int16_t *pCols; int32_t numOfCols; + SRowKey *pCurRowKey; _load_tomb_fn loadTombFn; + __compar_fn_t comparFn; void *pReader; void *idstr; bool rspRows; // response the rows in stt-file, if possible diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 61c610538e..97194acb00 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -479,6 +479,9 @@ int32_t tLDataIterOpen2(SLDataIter *pIter, SSttFileReader *pSttFileReader, int32 pIter->verRange.maxVer = pConf->verRange.maxVer; pIter->timeWindow.skey = pConf->timewindow.skey; pIter->timeWindow.ekey = pConf->timewindow.ekey; + pIter->comparFn = pConf->comparFn; + + tRowKeyAssign(&pIter->startRowKey, pConf->pCurRowKey); pIter->pReader = pSttFileReader; pIter->pBlockLoadInfo = pBlockLoadInfo; @@ -618,17 +621,39 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) { } int64_t ts = pData->aTSKEY[i]; - if (!pIter->backward) { // asc + if (!pIter->backward) { // asc if (ts > pIter->timeWindow.ekey) { // no more data break; - } else if (ts < pIter->timeWindow.skey) { - continue; + } else { + if (ts < pIter->timeWindow.skey) { + continue; + } + + if (ts == pIter->timeWindow.skey && pIter->startRowKey.numOfPKs > 0) { + SRowKey key; + tColRowGetKey(pData, i, &key); + int32_t ret = pkCompEx(pIter->comparFn, &key, &pIter->startRowKey); + if (ret < 0) { + continue; + } + } } } else { if (ts < pIter->timeWindow.skey) { break; - } else if (ts > pIter->timeWindow.ekey) { - continue; + } else { + if (ts > pIter->timeWindow.ekey) { + continue; + } + + if (ts == pIter->timeWindow.ekey && pIter->startRowKey.numOfPKs > 0) { + SRowKey key; + tColRowGetKey(pData, i, &key); + int32_t ret = pkCompEx(pIter->comparFn, &key, &pIter->startRowKey); + if (ret > 0) { + continue; + } + } } } @@ -802,8 +827,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF STimeWindow w = {0}; int64_t numOfRows = 0; + int64_t cid = pSttLevel->fobjArr->data[i]->f->cid; - int64_t cid = pSttLevel->fobjArr->data[i]->f->cid; code = tLDataIterOpen2(pIter, pSttFileReader, cid, pMTree->backward, pConf, pLoadInfo, &w, &numOfRows, pMTree->idStr); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 891fd823e7..80c4514579 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -89,7 +89,7 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi static void resetPreFilesetMemTableListIndex(SReaderStatus* pStatus); -static int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2) { +int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2) { if (p2 == NULL) { return 1; } @@ -118,12 +118,13 @@ static void tColRowGetKeyDeepCopy(SBlockData* pBlock, int32_t irow, int32_t slot return; } - pKey->numOfPKs = 1; - SColData* pColData = &pBlock->aColData[slotId]; SColVal cv; tColDataGetValue(pColData, irow, &cv); + pKey->numOfPKs = 1; + pKey->pks[0].type = cv.value.type; + if (IS_NUMERIC_TYPE(cv.value.type)) { pKey->pks[0].val = cv.value.val; } else { @@ -1394,7 +1395,7 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanI int64_t key = 0; if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { - int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; + int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts; key = ascScan ? TMIN(pBlock->firstKey, keyInStt) : TMAX(pBlock->lastKey, keyInStt); } else { key = ascScan ? pBlock->firstKey : pBlock->lastKey; @@ -1437,9 +1438,10 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->hasDupTs = (pBlockInfo->numRow > pBlockInfo->count) || (pBlockInfo->count <= 0); pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pRecord, pReader->info.order); + // todo handle the primary key overlap case ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { - int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey; + int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts; pInfo->overlapWithSttBlock = !(pBlockInfo->lastKey < nextProcKeyInStt || pBlockInfo->firstKey > nextProcKeyInStt); } @@ -1536,8 +1538,9 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, int32_t pkSrcSlot, SVersionRange* pVerRange) { - int32_t order = pSttBlockReader->order; - int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; + int32_t order = pSttBlockReader->order; + int32_t step = ASCENDING_TRAVERSE(order) ? 1 : -1; + SRowKey* pNextProc = &pScanInfo->sttKeyInfo.nextProcKey; while (1) { bool hasVal = tMergeTreeNext(&pSttBlockReader->mergeTree); @@ -1545,7 +1548,14 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; // next file, the timestamps in the next file must be greater than those in current - pScanInfo->sttKeyInfo.nextProcKey += step; + pNextProc->ts += step; + if (pSttBlockReader->numOfPks > 0) { + if (IS_NUMERIC_TYPE(pNextProc->pks[0].type)) { + pNextProc->pks[0].val = INT64_MIN; + } else { + memset(pNextProc->pks[0].pData, 0, pNextProc->pks[0].nData); + } + } return false; } @@ -1559,7 +1569,8 @@ static bool nextRowFromSttBlocks(SSttBlockReader* pSttBlockReader, STableBlockSc tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, &pSttBlockReader->currentKey); } - pScanInfo->sttKeyInfo.nextProcKey = key; + tColRowGetKeyDeepCopy(pRow->pBlockData, pRow->iRow, pkSrcSlot, pNextProc); + if (pScanInfo->delSkyline != NULL && TARRAY_SIZE(pScanInfo->delSkyline) > 0) { if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, order, pVerRange)) { pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; @@ -2042,7 +2053,9 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* d = NULL; STbData* di = NULL; bool asc = ASCENDING_TRAVERSE(pReader->info.order); + bool forward = true; STsdbReadSnap* pSnap = pReader->pReadSnap; + STimeWindow* pWindow = &pReader->info.window; if (pBlockScanInfo->iterInit) { return TSDB_CODE_SUCCESS; @@ -2051,6 +2064,10 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STsdbRowKey startKey = {0}; tRowKeyAssign(&startKey.key, &pBlockScanInfo->lastProcKey); startKey.version = asc ? pReader->info.verRange.minVer : pReader->info.verRange.maxVer; + if ((asc && (startKey.key.ts < pWindow->skey)) || ((!asc) && startKey.key.ts > pWindow->ekey)) { + startKey.key.ts = asc? pWindow->skey:pWindow->ekey; + forward = false; + } int32_t code = doInitMemDataIter(pReader, &d, pBlockScanInfo, &startKey, pSnap->pMem, &pBlockScanInfo->iter, "mem"); if (code != TSDB_CODE_SUCCESS) { @@ -2063,7 +2080,10 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea } loadMemTombData(&pBlockScanInfo->pMemDelData, d, di, pReader->info.verRange.maxVer); - forwardDataIter(&startKey.key, pBlockScanInfo, pReader); + + if (forward) { + forwardDataIter(&startKey.key, pBlockScanInfo, pReader); + } pBlockScanInfo->iterInit = true; return TSDB_CODE_SUCCESS; @@ -2115,6 +2135,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, int32_t rowIndex, STable static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { bool hasData = true; + bool asc = ASCENDING_TRAVERSE(pReader->info.order); // the stt block reader has been initialized for this table. if (pSttBlockReader->uid == pScanInfo->uid) { @@ -2133,10 +2154,10 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan } STimeWindow w = pSttBlockReader->window; - if (ASCENDING_TRAVERSE(pSttBlockReader->order)) { - w.skey = pScanInfo->sttKeyInfo.nextProcKey; + if (asc) { + w.skey = pScanInfo->sttKeyInfo.nextProcKey.ts; } else { - w.ekey = pScanInfo->sttKeyInfo.nextProcKey; + w.ekey = pScanInfo->sttKeyInfo.nextProcKey.ts; } int64_t st = taosGetTimestampUs(); @@ -2157,6 +2178,8 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan .pCols = pReader->suppInfo.colId, .numOfCols = pReader->suppInfo.numOfCols, .loadTombFn = loadSttTombDataForAll, + .pCurRowKey = &pScanInfo->sttKeyInfo.nextProcKey, + .comparFn = pReader->pkComparFn, .pReader = pReader, .idstr = pReader->idStr, .rspRows = (pReader->info.execMode == READER_EXEC_ROWS), @@ -2193,8 +2216,9 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan } pScanInfo->sttKeyInfo.status = taosArrayGetSize(info.pTimeWindowList) ? STT_FILE_HAS_DATA : STT_FILE_NO_DATA; - pScanInfo->sttKeyInfo.nextProcKey = - ASCENDING_TRAVERSE(pReader->info.order) ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; + + // todo set the primary key value + pScanInfo->sttKeyInfo.nextProcKey.ts = asc ? pScanInfo->sttWindow.skey : pScanInfo->sttWindow.ekey; hasData = (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); } else { // not clean stt blocks INIT_TIMEWINDOW(&pScanInfo->sttWindow); //reset the time window @@ -2755,7 +2779,7 @@ static void buildCleanBlockFromSttFiles(STsdbReader* pReader, STableBlockScanInf setComposedBlockFlag(pReader, true); - pScanInfo->sttKeyInfo.nextProcKey = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1; + pScanInfo->sttKeyInfo.nextProcKey.ts = asc ? pScanInfo->sttWindow.ekey + 1 : pScanInfo->sttWindow.skey - 1; pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; pScanInfo->lastProcKey.ts = asc ? pScanInfo->sttWindow.ekey : pScanInfo->sttWindow.skey; @@ -2915,7 +2939,7 @@ static bool notOverlapWithFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanI if ((!hasDataInSttBlock(pScanInfo)) || (pScanInfo->cleanSttBlocks == true)) { return true; } else { - int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; + int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts; return (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt); } } @@ -2963,7 +2987,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (notOverlapWithFiles(pBlockInfo, pScanInfo, asc)) { - int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; + int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey.ts; if ((!hasDataInSttBlock(pScanInfo)) || (asc && pBlockInfo->lastKey < keyInStt) || (!asc && pBlockInfo->firstKey > keyInStt)) { @@ -3649,7 +3673,7 @@ int32_t doMergeRowsInSttBlock(SSttBlockReader* pSttBlockReader, STableBlockScanI } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), - pScanInfo->sttKeyInfo.nextProcKey, idStr); + pScanInfo->sttKeyInfo.nextProcKey.ts, idStr); break; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 93bc5ca573..99f0e42261 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -130,25 +130,46 @@ STableBlockScanInfo* getTableBlockScanInfo(SSHashObj* pTableMap, uint64_t uid, c return *p; } +static int32_t initSRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, int32_t len) { + pKey->numOfPKs = numOfPks; + pKey->ts = ts; + + if (numOfPks > 0) { + pKey->pks[0].type = type; + if (IS_NUMERIC_TYPE(pKey->pks[0].type)) { + pKey->pks[0].val = INT64_MIN; + } else { + pKey->pks[0].pData = taosMemoryCalloc(1, len); + pKey->pks[0].nData = 0; + + if (pKey->pks[0].pData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + } + } + + return TSDB_CODE_SUCCESS; +} + static void initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) { + int32_t numOfPks = pReader->suppInfo.numOfPks; + SRowKey* pRowKey = &pScanInfo->lastProcKey; if (ASCENDING_TRAVERSE(pReader->info.order)) { int64_t skey = pReader->info.window.skey; - pRowKey->ts = (skey > INT64_MIN) ? (skey - 1) : skey; - pScanInfo->sttKeyInfo.nextProcKey = skey; + int64_t ts = (skey > INT64_MIN) ? (skey - 1) : skey; + + initSRowKey(pRowKey, ts, numOfPks, pReader->suppInfo.pk.type, pReader->suppInfo.pk.bytes); + initSRowKey(&pScanInfo->sttKeyInfo.nextProcKey, skey, numOfPks, pReader->suppInfo.pk.type, + pReader->suppInfo.pk.bytes); } else { int64_t ekey = pReader->info.window.ekey; - pRowKey->ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey; - pScanInfo->sttKeyInfo.nextProcKey = ekey; - } + int64_t ts = (ekey < INT64_MAX) ? (ekey + 1) : ekey; - // only handle the first primary key. - pRowKey->numOfPKs = pReader->suppInfo.numOfPks; - if (pReader->suppInfo.numOfPks > 0) { - if (IS_VAR_DATA_TYPE(pReader->suppInfo.pk.type)) { - pRowKey->pks[0].pData = taosMemoryCalloc(1, pReader->suppInfo.pk.bytes); - } - pRowKey->pks[0].type = pReader->suppInfo.pk.type; + initSRowKey(pRowKey, ts, numOfPks, pReader->suppInfo.pk.type, pReader->suppInfo.pk.bytes); + initSRowKey(&pScanInfo->sttKeyInfo.nextProcKey, ekey, numOfPks, pReader->suppInfo.pk.type, + pReader->suppInfo.pk.bytes); } } @@ -230,7 +251,7 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); pInfo->lastProcKey.ts = ts; // todo check the nextProcKey info - pInfo->sttKeyInfo.nextProcKey = ts + step; + pInfo->sttKeyInfo.nextProcKey.ts = ts + step; } } diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 4131bb1b86..9cdb36e648 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -77,7 +77,9 @@ typedef enum ESttKeyStatus { typedef struct SSttKeyInfo { ESttKeyStatus status; // this value should be updated when switch to the next fileset - int64_t nextProcKey; // todo remove this attribute, since it is impossible to set correct nextProcKey value + SRowKey nextProcKey; + // int64_t nextProcKey; // todo remove this attribute, since it is impossible to set correct nextProcKey + // value } SSttKeyInfo; // clean stt file blocks: @@ -333,6 +335,7 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra const char* pstr); bool isCleanSttBlock(SArray* pTimewindowList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order); bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBrinRecord* pRecord, int32_t order); +int32_t pkCompEx(__compar_fn_t comparFn, SRowKey* p1, SRowKey* p2); typedef struct { SArray* pTombData; From 9638f8d7fcb345d563f605c7d04cf03acc7f2afb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Mar 2024 00:14:46 +0800 Subject: [PATCH 2/9] fix(tsdb): set the correct sttkey --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 80c4514579..27e7da11df 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -1861,9 +1861,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); - SRowKey* pSttKey = NULL; + SRowKey* pSttKey = &(SRowKey){0}; if (hasDataInSttBlock(pBlockScanInfo) && (!pBlockScanInfo->cleanSttBlocks)) { - pSttKey = getCurrentKeyInSttBlock(pSttBlockReader); + tRowKeyAssign(pSttKey, getCurrentKeyInSttBlock(pSttBlockReader)); + } else { + pSttKey = NULL; } SRowKey* pfKey = &(SRowKey){0}; @@ -1902,7 +1904,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } } - SRowKey minKey; + SRowKey minKey = {0}; if (ASCENDING_TRAVERSE(pReader->info.order)) { minKey = k; // let's find the minimum @@ -1910,11 +1912,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = ik; } - if (pfKey != NULL && (pkCompEx(compFn, pfKey, &minKey) < 0)) { + if ((pfKey != NULL) && (pkCompEx(compFn, pfKey, &minKey) < 0)) { minKey = *pfKey; } - if (pSttKey != NULL && (pkCompEx(compFn, pSttKey, &minKey) < 0)) { + if ((pSttKey != NULL) && (pkCompEx(compFn, pSttKey, &minKey) < 0)) { minKey = *pSttKey; } } else { @@ -1923,11 +1925,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = ik; } - if (pfKey != NULL && (pkCompEx(compFn, pfKey, &minKey) > 0)) { + if ((pfKey != NULL) && (pkCompEx(compFn, pfKey, &minKey) > 0)) { minKey = *pfKey; } - if (pSttKey != NULL && (pkCompEx(compFn, pSttKey, &minKey) > 0)) { + if ((pSttKey != NULL) && (pkCompEx(compFn, pSttKey, &minKey) > 0)) { minKey = *pSttKey; } } From 5fe03fe3f4985ba7b9fdbd5fe48f6ea6a8212b26 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 26 Mar 2024 09:18:45 +0800 Subject: [PATCH 3/9] feat: irate support pk --- source/libs/function/inc/builtinsimpl.h | 2 +- source/libs/function/src/builtins.c | 3 +- source/libs/function/src/builtinsimpl.c | 86 ++++++++++++++++++------- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 54bcd26bc3..ecc64fcd00 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -144,7 +144,7 @@ int32_t irateFunction(SqlFunctionCtx* pCtx); int32_t irateFunctionMerge(SqlFunctionCtx* pCtx); int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); int32_t iratePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); -int32_t getIrateInfoSize(); +int32_t getIrateInfoSize(int32_t pkBytes); int32_t cachedLastRowFunction(SqlFunctionCtx* pCtx); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index cb2a979909..0f88572dbc 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1638,7 +1638,8 @@ static int32_t translateIrateImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t l if (!IS_NUMERIC_TYPE(colType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType){.bytes = getIrateInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; + int32_t pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0; + pFunc->node.resType = (SDataType){.bytes = getIrateInfoSize(pkBytes) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY}; } else { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 2907c096fe..70fe2b19df 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -269,6 +269,12 @@ typedef struct SRateInfo { double lastValue; TSKEY lastKey; int8_t hasResult; // flag to denote has value + + char* firstPk; + char* lastPk; + int8_t pkType; + int32_t pkBytes; + char pkData[]; } SRateInfo; typedef struct SGroupKeyInfo { @@ -6165,10 +6171,11 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t getIrateInfoSize() { return (int32_t)sizeof(SRateInfo); } +int32_t getIrateInfoSize(int32_t pkBytes) { return (int32_t)sizeof(SRateInfo) + 2 * pkBytes; } bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { - pEnv->calcMemSize = sizeof(SRateInfo); + int32_t pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0; + pEnv->calcMemSize = getIrateInfoSize(pkBytes); return true; } @@ -6188,6 +6195,36 @@ bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { return true; } +static void doSaveRateInfo(SRateInfo* pRateInfo, bool isFirst, int64_t ts, char* pk, double v) { + if (isFirst) { + pRateInfo->firstValue = v; + pRateInfo->firstKey = ts; + if (pRateInfo->firstPk) { + int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; + memcpy(pRateInfo->firstPk, pk, pkBytes); + } + } else { + pRateInfo->lastValue = v; + pRateInfo->lastKey = ts; + if (pRateInfo->lastPk) { + int32_t pkBytes = IS_VAR_DATA_TYPE(pRateInfo->pkType) ? varDataTLen(pk) : pRateInfo->pkBytes; + memcpy(pRateInfo->lastPk, pk, pkBytes); + } + } +} + +static void initializeRateInfo(SqlFunctionCtx* pCtx, SRateInfo* pRateInfo) { + if (pCtx->hasPrimaryKey) { + pRateInfo->pkType = pCtx->input.pPrimaryKey->info.type; + pRateInfo->pkBytes = pCtx->input.pPrimaryKey->info.bytes; + pRateInfo->firstPk = pRateInfo->pkData; + pRateInfo->lastPk = pRateInfo->pkData + pRateInfo->pkBytes; + } else { + pRateInfo->firstPk = NULL; + pRateInfo->lastPk = NULL; + } +} + int32_t irateFunction(SqlFunctionCtx* pCtx) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SRateInfo* pRateInfo = GET_ROWCELL_INTERBUF(pResInfo); @@ -6198,6 +6235,8 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) { SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; funcInputUpdate(pCtx); + + initializeRateInfo(pCtx, pRateInfo); int32_t numOfElems = 0; int32_t type = pInputCol->info.type; @@ -6212,21 +6251,16 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) { GET_TYPED_DATA(v, double, type, data); if (INT64_MIN == pRateInfo->lastKey) { - pRateInfo->lastValue = v; - pRateInfo->lastKey = row.ts; + doSaveRateInfo(pRateInfo, false, row.ts, row.pPk, v); pRateInfo->hasResult = 1; continue; } if (row.ts > pRateInfo->lastKey) { if ((INT64_MIN == pRateInfo->firstKey) || pRateInfo->lastKey > pRateInfo->firstKey) { - pRateInfo->firstValue = pRateInfo->lastValue; - pRateInfo->firstKey = pRateInfo->lastKey; + doSaveRateInfo(pRateInfo, true, pRateInfo->lastKey, pRateInfo->lastPk, pRateInfo->lastValue); } - - pRateInfo->lastValue = v; - pRateInfo->lastKey = row.ts; - + doSaveRateInfo(pRateInfo, false, row.ts, row.pPk, v); continue; } else if (row.ts == pRateInfo->lastKey) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; @@ -6234,8 +6268,7 @@ int32_t irateFunction(SqlFunctionCtx* pCtx) { if ((INT64_MIN == pRateInfo->firstKey) || row.ts > pRateInfo->firstKey) { - pRateInfo->firstValue = v; - pRateInfo->firstKey = row.ts; + doSaveRateInfo(pRateInfo, true, row.ts, row.pPk, v); } else if (row.ts == pRateInfo->firstKey) { return TSDB_CODE_FUNC_DUP_TIMESTAMP; } @@ -6271,25 +6304,26 @@ static double doCalcRate(const SRateInfo* pRateInfo, double tickPerSec) { static void irateTransferInfoImpl(TSKEY inputKey, SRateInfo* pInput, SRateInfo* pOutput, bool isFirstKey) { if (inputKey > pOutput->lastKey) { - pOutput->firstKey = pOutput->lastKey; - pOutput->firstValue = pOutput->lastValue; - - pOutput->lastKey = isFirstKey ? pInput->firstKey : pInput->lastKey; - pOutput->lastValue = isFirstKey ? pInput->firstValue : pInput->lastValue; + doSaveRateInfo(pOutput, true, pOutput->lastKey, pOutput->lastPk, pOutput->lastValue); + if (isFirstKey) { + doSaveRateInfo(pOutput, false, pInput->firstKey, pInput->firstPk, pInput->firstValue); + } else { + doSaveRateInfo(pOutput, false, pInput->lastKey, pInput->lastPk, pInput->lastValue); + } } else if ((inputKey < pOutput->lastKey) && (inputKey > pOutput->firstKey)) { - pOutput->firstKey = isFirstKey ? pInput->firstKey : pInput->lastKey; - pOutput->firstValue = isFirstKey ? pInput->firstValue : pInput->lastValue; + if (isFirstKey) { + doSaveRateInfo(pOutput, true, pInput->firstKey, pInput->firstPk, pInput->firstValue); + } else { + doSaveRateInfo(pOutput, true, pInput->lastKey, pInput->lastPk, pInput->lastValue); + } } else { // inputKey < pOutput->firstKey } } static void irateCopyInfo(SRateInfo* pInput, SRateInfo* pOutput) { - pOutput->firstKey = pInput->firstKey; - pOutput->lastKey = pInput->lastKey; - - pOutput->firstValue = pInput->firstValue; - pOutput->lastValue = pInput->lastValue; + doSaveRateInfo(pOutput, true, pInput->firstKey, pInput->firstPk, pInput->firstValue); + doSaveRateInfo(pOutput, false, pInput->lastKey, pInput->lastPk, pInput->lastValue); } static int32_t irateTransferInfo(SRateInfo* pInput, SRateInfo* pOutput) { @@ -6324,11 +6358,13 @@ int32_t irateFunctionMerge(SqlFunctionCtx* pCtx) { } SRateInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); + initializeRateInfo(pCtx, pInfo); int32_t start = pInput->startRowIndex; for (int32_t i = start; i < start + pInput->numOfRows; ++i) { char* data = colDataGetData(pCol, i); SRateInfo* pInputInfo = (SRateInfo*)varDataVal(data); + initializeRateInfo(pCtx, pInfo); if (pInputInfo->hasResult) { int32_t code = irateTransferInfo(pInputInfo, pInfo); if (code != TSDB_CODE_SUCCESS) { @@ -6347,7 +6383,7 @@ int32_t irateFunctionMerge(SqlFunctionCtx* pCtx) { int32_t iratePartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SRateInfo* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); - int32_t resultBytes = getIrateInfoSize(); + int32_t resultBytes = getIrateInfoSize(pInfo->pkBytes); char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char)); memcpy(varDataVal(res), pInfo, resultBytes); From a70d5a49476a19e45f3717e5f9188ccf13f75c36 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Mar 2024 09:50:06 +0800 Subject: [PATCH 4/9] fix(tsdb): fix bug during filtering brin records. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 27e7da11df..7505230eca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -650,7 +650,6 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN int32_t k = 0; int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); bool asc = ASCENDING_TRAVERSE(pReader->info.order); - int32_t step = asc ? 1 : -1; STimeWindow w = pReader->info.window; SBrinRecord* pRecord = NULL; @@ -710,7 +709,8 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN continue; } - if (pkCompEx(pReader->pkComparFn, &pRecord->lastKey.key, &pScanInfo->lastProcKey) <= 0) { + int32_t ret = pkCompEx(pReader->pkComparFn, &pRecord->lastKey.key, &pScanInfo->lastProcKey); + if ((asc && ret <= 0) || (!asc && ret >= 0)) { continue; } From 52a04cd00822576b90dc38ed80c40211285cbd3b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 26 Mar 2024 10:55:56 +0800 Subject: [PATCH 5/9] fix(tsdb): fix bug for filter brin record. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 7505230eca..c7164d43de 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -709,9 +709,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN continue; } - int32_t ret = pkCompEx(pReader->pkComparFn, &pRecord->lastKey.key, &pScanInfo->lastProcKey); - if ((asc && ret <= 0) || (!asc && ret >= 0)) { - continue; + if (asc) { + if (pkCompEx(pReader->pkComparFn, &pRecord->lastKey.key, &pScanInfo->lastProcKey) <= 0) { + continue; + } + } else { + if (pkCompEx(pReader->pkComparFn, &pRecord->firstKey.key, &pScanInfo->lastProcKey) >= 0) { + continue; + } } // 2. version range check, version range is an CLOSED interval From f778dce52d04220e375c384cf0ce4bcf19e3f202 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 26 Mar 2024 14:13:00 +0800 Subject: [PATCH 6/9] fix: add pk for merge generated during merge scan node split --- source/libs/planner/src/planSpliter.c | 59 ++++++++++++++++++++------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 34be489333..dd0cb5a964 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -690,19 +690,23 @@ static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent return code; } +static int32_t stbSplCreateMergeKeysByExpr(SNode* pExpr, EOrder order, SNodeList** pMergeKeys) { + SOrderByExprNode* pOrderByExpr = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); + if (NULL == pOrderByExpr) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pOrderByExpr->pExpr = nodesCloneNode(pExpr); + if (NULL == pOrderByExpr->pExpr) { + nodesDestroyNode((SNode*)pOrderByExpr); + return TSDB_CODE_OUT_OF_MEMORY; + } + pOrderByExpr->order = order; + pOrderByExpr->nullOrder = (order == ORDER_ASC) ? NULL_ORDER_FIRST : NULL_ORDER_LAST; + return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pOrderByExpr); +} + static int32_t stbSplCreateMergeKeysByPrimaryKey(SNode* pPrimaryKey, EOrder order, SNodeList** pMergeKeys) { - SOrderByExprNode* pMergeKey = (SOrderByExprNode*)nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR); - if (NULL == pMergeKey) { - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->pExpr = nodesCloneNode(pPrimaryKey); - if (NULL == pMergeKey->pExpr) { - nodesDestroyNode((SNode*)pMergeKey); - return TSDB_CODE_OUT_OF_MEMORY; - } - pMergeKey->order = order; - pMergeKey->nullOrder = NULL_ORDER_FIRST; - return nodesListMakeStrictAppend(pMergeKeys, (SNode*)pMergeKey); + return stbSplCreateMergeKeysByExpr(pPrimaryKey, order, pMergeKeys); } static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) { @@ -1357,6 +1361,28 @@ static SNode* stbSplFindPrimaryKeyFromScan(SScanLogicNode* pScan) { return pCol; } +static SNode* stbSplFindPkFromScan(SScanLogicNode* pScan) { + bool find = false; + SNode* pCol = NULL; + FOREACH(pCol, pScan->pScanCols) { + if (((SColumnNode*)pCol)->isPk) { + find = true; + break; + } + } + if (!find) { + return NULL; + } + SNode* pTarget = NULL; + FOREACH(pTarget, pScan->node.pTargets) { + if (nodesEqualNode(pTarget, pCol)) { + return pCol; + } + } + nodesListStrictAppend(pScan->node.pTargets, nodesCloneNode(pCol)); + return pCol; +} + static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOutputMergeScan, SNodeList** pOutputMergeKeys) { SNodeList* pChildren = pScan->node.pChildren; @@ -1374,8 +1400,13 @@ static int32_t stbSplCreateMergeScanNode(SScanLogicNode* pScan, SLogicNode** pOu pMergeScan->filesetDelimited = true; pMergeScan->node.pChildren = pChildren; splSetParent((SLogicNode*)pMergeScan); - code = stbSplCreateMergeKeysByPrimaryKey(stbSplFindPrimaryKeyFromScan(pMergeScan), - pMergeScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, &pMergeKeys); + + SNode* pTs = stbSplFindPrimaryKeyFromScan(pMergeScan); + code = stbSplCreateMergeKeysByPrimaryKey(pTs, pMergeScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, &pMergeKeys); + SNode* pPk = stbSplFindPkFromScan(pMergeScan); + if (TSDB_CODE_SUCCESS == code && NULL != pPk) { + code = stbSplCreateMergeKeysByExpr(pPk, pMergeScan->scanSeq[0] > 0 ? ORDER_ASC : ORDER_DESC, &pMergeKeys); + } } if (TSDB_CODE_SUCCESS == code) { From ee4dda56c12d43c8a9b2c9a539fa1575a2884843 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 26 Mar 2024 14:43:28 +0800 Subject: [PATCH 7/9] more code --- source/common/src/tdataformat.c | 22 ++++++++-------- source/dnode/vnode/src/vnd/vnodeSvr.c | 36 ++++++++++++++++++++------- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index af124f7a02..cf8e28bf9a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -557,13 +557,10 @@ void tRowDestroy(SRow *pRow) { } static int32_t tRowPCmprFn(const void *p1, const void *p2) { - if ((*(SRow **)p1)->ts < (*(SRow **)p2)->ts) { - return -1; - } else if ((*(SRow **)p1)->ts > (*(SRow **)p2)->ts) { - return 1; - } - - return 0; + SRowKey key1, key2; + tRowGetKey(*(SRow **)p1, &key1); + tRowGetKey(*(SRow **)p2, &key2); + return tRowKeyCompare(&key1, &key2); } static void tRowPDestroy(SRow **ppRow) { tRowDestroy(*ppRow); } static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart, int32_t iEnd, int8_t flag) { @@ -645,13 +642,18 @@ int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag) { int32_t iStart = 0; while (iStart < aRowP->size) { - SRow *pRow = (SRow *)taosArrayGetP(aRowP, iStart); + SRowKey key1; + SRow *row1 = (SRow *)taosArrayGetP(aRowP, iStart); + + tRowGetKey(row1, &key1); int32_t iEnd = iStart + 1; while (iEnd < aRowP->size) { - SRow *pRowT = (SRow *)taosArrayGetP(aRowP, iEnd); + SRowKey key2; + SRow *row2 = (SRow *)taosArrayGetP(aRowP, iEnd); + tRowGetKey(row2, &key2); - if (pRow->ts != pRowT->ts) break; + if (tRowKeyCompare(&key1, &key2) != 0) break; iEnd++; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index a03adb164b..a4af782ad7 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -32,14 +32,14 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, SRpcMsg *pOriginRpc); static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg); + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessCreateTSmaReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterConfigReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg); + SRpcMsg *pOriginalMsg); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -1516,7 +1516,7 @@ static int32_t vnodeRebuildSubmitReqMsg(SSubmitReq2 *pSubmitReq, void **ppMsg) { } static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg) { + SRpcMsg *pOriginalMsg) { int32_t code = 0; terrno = 0; @@ -1586,12 +1586,26 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in } else { int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP); SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); + SRowKey lastRowKey; for (int32_t iRow = 0; iRow < nRow; ++iRow) { - if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) { + if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey) { code = TSDB_CODE_INVALID_MSG; vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver); goto _exit; } + if (iRow == 0) { + tRowGetKey(aRow[iRow], &lastRowKey); + } else { + SRowKey rowKey; + tRowGetKey(aRow[iRow], &rowKey); + + if (tRowKeyCompare(&lastRowKey, &rowKey) >= 0) { + code = TSDB_CODE_INVALID_MSG; + vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver); + goto _exit; + } + lastRowKey = rowKey; + } } } } @@ -1735,10 +1749,14 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nInsertSuccess, pSubmitRsp->affectedRows); atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); - if(tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0){ - const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, pVnode->monitor.strClusterId, - pVnode->monitor.strDnodeId, tsLocalEp, pVnode->monitor.strVgId, - pOriginalMsg->info.conn.user, "Success"}; + if (tsEnableMonitor && pSubmitRsp->affectedRows > 0 && strlen(pOriginalMsg->info.conn.user) > 0) { + const char *sample_labels[] = {VNODE_METRIC_TAG_VALUE_INSERT_AFFECTED_ROWS, + pVnode->monitor.strClusterId, + pVnode->monitor.strDnodeId, + tsLocalEp, + pVnode->monitor.strVgId, + pOriginalMsg->info.conn.user, + "Success"}; taos_counter_add(pVnode->monitor.insertCounter, pSubmitRsp->affectedRows, sample_labels); } @@ -2019,7 +2037,7 @@ static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t ver, void *pRe } static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp, - SRpcMsg *pOriginalMsg) { + SRpcMsg *pOriginalMsg) { int32_t code = 0; SDecoder *pCoder = &(SDecoder){0}; SDeleteRes *pRes = &(SDeleteRes){0}; From 93ca55954fcd6f9d209ea204af8030e09d6ef32f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 26 Mar 2024 15:11:15 +0800 Subject: [PATCH 8/9] more code change --- include/libs/qcom/query.h | 2 +- source/libs/parser/inc/parInsertUtil.h | 4 +- source/libs/parser/src/parInsertSml.c | 31 ++-- source/libs/parser/src/parInsertSql.c | 194 ++++++++++++------------- source/libs/parser/src/parInsertUtil.c | 60 ++++---- 5 files changed, 148 insertions(+), 143 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 8d6cd6a3c0..0b192d5593 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -197,7 +197,7 @@ typedef struct STableDataCxt { SBoundColInfo boundColsInfo; SArray* pValues; SSubmitTbData* pData; - TSKEY lastTs; + SRowKey lastKey; bool ordered; bool duplicateTs; } STableDataCxt; diff --git a/source/libs/parser/inc/parInsertUtil.h b/source/libs/parser/inc/parInsertUtil.h index b20587dd43..1988620539 100644 --- a/source/libs/parser/inc/parInsertUtil.h +++ b/source/libs/parser/inc/parInsertUtil.h @@ -45,8 +45,8 @@ int16_t insFindCol(struct SToken *pColname, int16_t start, int16_t end, SSchema void insBuildCreateTbReq(SVCreateTbReq *pTbReq, const char *tname, STag *pTag, int64_t suid, const char *sname, SArray *tagName, uint8_t tagNum, int32_t ttl); int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo *pInfo); -void insInitColValues(STableMeta* pTableMeta, SArray* aColValues); -void insCheckTableDataOrder(STableDataCxt *pTableCxt, TSKEY tsKey); +void insInitColValues(STableMeta *pTableMeta, SArray *aColValues); +void insCheckTableDataOrder(STableDataCxt *pTableCxt, SRowKey *rowKey); int32_t insGetTableDataCxt(SHashObj *pHash, void *id, int32_t idLen, STableMeta *pTableMeta, SVCreateTbReq **pCreateTbReq, STableDataCxt **pTableCxt, bool colMode, bool ignoreColVals); int32_t initTableColSubmitData(STableDataCxt *pTableCxt); diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index c2ce77f02b..fcb5588717 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -113,7 +113,8 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); - if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){ + if (kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || + kv->type != pTagSchema->type) { code = TSDB_CODE_SML_INVALID_DATA; uError("SML smlBuildTagRow error col not same %s", pTagSchema->name); goto end; @@ -200,7 +201,9 @@ int32_t smlBuildRow(STableDataCxt* pTableCxt) { if (TSDB_CODE_SUCCESS != ret) { return ret; } - insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + SRowKey key; + tRowGetKey(*pRow, &key); + insCheckTableDataOrder(pTableCxt, &key); return TSDB_CODE_SUCCESS; } @@ -209,15 +212,16 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 SSchema* pColSchema = schema + index; SColVal* pVal = taosArrayGet(pTableCxt->pValues, index); SSmlKv* kv = (SSmlKv*)data; - if(kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || kv->type != pColSchema->type){ + if (kv->keyLen != strlen(pColSchema->name) || memcmp(kv->key, pColSchema->name, kv->keyLen) != 0 || + kv->type != pColSchema->type) { ret = TSDB_CODE_SML_INVALID_DATA; char* tmp = taosMemoryCalloc(kv->keyLen + 1, 1); - if(tmp){ + if (tmp) { memcpy(tmp, kv->key, kv->keyLen); - uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", - tmp, tDataTypes[kv->type].name, pColSchema->name, tDataTypes[pColSchema->type].name); + uInfo("SML data(name:%s type:%s) is not same like the db data(name:%s type:%s)", tmp, tDataTypes[kv->type].name, + pColSchema->name, tDataTypes[pColSchema->type].name); taosMemoryFree(tmp); - }else{ + } else { uError("SML smlBuildCol out of memory"); } goto end; @@ -225,11 +229,11 @@ int32_t smlBuildCol(STableDataCxt* pTableCxt, SSchema* schema, void* data, int32 if (kv->type == TSDB_DATA_TYPE_NCHAR) { int32_t len = 0; int64_t size = pColSchema->bytes - VARSTR_HEADER_SIZE; - if(size <= 0){ + if (size <= 0) { ret = TSDB_CODE_SML_INVALID_DATA; goto end; } - char* pUcs4 = taosMemoryCalloc(1, size); + char* pUcs4 = taosMemoryCalloc(1, size); if (NULL == pUcs4) { ret = TSDB_CODE_OUT_OF_MEMORY; goto end; @@ -351,7 +355,7 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc continue; } SSmlKv* kv = *(SSmlKv**)p; - if(kv->type != pColSchema->type){ + if (kv->type != pColSchema->type) { ret = buildInvalidOperationMsg(&pBuf, "kv type not equal to col type"); goto end; } @@ -367,7 +371,8 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc } if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)pUcs4, pColSchema->bytes - VARSTR_HEADER_SIZE, &len)) { if (errno == E2BIG) { - uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, pColSchema->bytes, kv->value); + uError("sml bind taosMbsToUcs4 error, kv length:%d, bytes:%d, kv->value:%s", (int)kv->length, + pColSchema->bytes, kv->value); buildInvalidOperationMsg(&pBuf, "value too long"); ret = TSDB_CODE_PAR_VALUE_TOO_LONG; goto end; @@ -396,7 +401,9 @@ int32_t smlBindData(SQuery* query, bool dataFormat, SArray* tags, SArray* colsSc buildInvalidOperationMsg(&pBuf, "tRowBuild error"); goto end; } - insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + SRowKey key; + tRowGetKey(*pRow, &key); + insCheckTableDataOrder(pTableCxt, &key); clearColValArraySml(pTableCxt->pValues); } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 77c63c38d8..1a41ba5fb9 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -13,12 +13,12 @@ * along with this program. If not, see . */ +#include "geosWrapper.h" #include "parInsertUtil.h" #include "parToken.h" #include "scalar.h" #include "tglobal.h" #include "ttime.h" -#include "geosWrapper.h" typedef struct SInsertParseContext { SParseContext* pComCxt; @@ -29,7 +29,7 @@ typedef struct SInsertParseContext { bool usingDuplicateTable; bool forceUpdate; bool needTableTagVal; - bool needRequest; // whether or not request server + bool needRequest; // whether or not request server } SInsertParseContext; typedef int32_t (*_row_append_fn_t)(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param); @@ -154,19 +154,15 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModify return code; } -typedef enum { - BOUND_TAGS, - BOUND_COLUMNS, - BOUND_ALL_AND_TBNAME -} EBoundColumnsType; +typedef enum { BOUND_TAGS, BOUND_COLUMNS, BOUND_ALL_AND_TBNAME } EBoundColumnsType; static int32_t getTbnameSchemaIndex(STableMeta* pTableMeta) { return pTableMeta->tableInfo.numOfTags + pTableMeta->tableInfo.numOfColumns; } // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, EBoundColumnsType boundColsType, STableMeta* pTableMeta, - SBoundColInfo* pBoundInfo) { +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, EBoundColumnsType boundColsType, + STableMeta* pTableMeta, SBoundColInfo* pBoundInfo) { SSchema* pSchema = NULL; if (boundColsType == BOUND_TAGS) { pSchema = getTableTagSchema(pTableMeta); @@ -202,8 +198,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E token.z = tmpTokenBuf; token.n = strdequote(token.z); - if (boundColsType == BOUND_ALL_AND_TBNAME && - token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { + if (boundColsType == BOUND_ALL_AND_TBNAME && token.n == strlen("tbname") && (strcasecmp(token.z, "tbname") == 0)) { pBoundInfo->pColIndex[pBoundInfo->numOfBound] = tbnameSchemaIndex; pUseCols[tbnameSchemaIndex] = true; ++pBoundInfo->numOfBound; @@ -230,7 +225,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, E if (TSDB_CODE_SUCCESS == code && (BOUND_TAGS != boundColsType) && !pUseCols[0]) { code = buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); } - if (TSDB_CODE_SUCCESS == code && (BOUND_ALL_AND_TBNAME == boundColsType) &&!pUseCols[tbnameSchemaIndex]) { + if (TSDB_CODE_SUCCESS == code && (BOUND_ALL_AND_TBNAME == boundColsType) && !pUseCols[tbnameSchemaIndex]) { code = buildInvalidOperationMsg(&pCxt->msg, "tbname column can not be null"); } taosMemoryFree(pUseCols); @@ -285,10 +280,11 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t int32_t index = 0, i = 0; int64_t interval = 0, tempInterval = 0; int64_t ts = 0, tempTs = 0; - bool firstIsTS = false, secondIsTs = false; + bool firstIsTS = false, secondIsTs = false; const char* pTokenEnd = *end; - if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) { + if (TSDB_CODE_SUCCESS != + parseTimestampOrInterval(&pTokenEnd, pToken, timePrec, &ts, &interval, pMsgBuf, &firstIsTS)) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } @@ -330,8 +326,7 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t if (pTokenEnd[i] == ' ' || pTokenEnd[i] == '\t') { i++; continue; - } - else if (pTokenEnd[i] == ',' || pTokenEnd[i] == ')') { + } else if (pTokenEnd[i] == ',' || pTokenEnd[i] == ')') { *end = pTokenEnd + i; if (!firstIsTS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -362,7 +357,8 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t valueToken.n = len; } - if (TSDB_CODE_SUCCESS != parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) { + if (TSDB_CODE_SUCCESS != + parseTimestampOrInterval(&pTokenEnd, &valueToken, timePrec, &tempTs, &tempInterval, pMsgBuf, &secondIsTs)) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } @@ -376,7 +372,7 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); } ts = tempTs; - }else { + } else { // not support operator between tow interval, such as 2h + 3s if (!firstIsTS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp format", pToken->z); @@ -413,7 +409,7 @@ static int parseTime(const char** end, SToken* pToken, int16_t timePrec, int64_t } // need to call geosFreeBuffer(*output) later -static int parseGeometry(SToken *pToken, unsigned char **output, size_t *size) { +static int parseGeometry(SToken* pToken, unsigned char** output, size_t* size) { int32_t code = TSDB_CODE_FAILED; //[ToDo] support to parse WKB as well as WKT @@ -432,19 +428,19 @@ static int parseGeometry(SToken *pToken, unsigned char **output, size_t *size) { return code; } -static int32_t parseVarbinary(SToken* pToken, uint8_t **pData, uint32_t *nData, int32_t bytes){ - if(pToken->type != TK_NK_STRING){ +static int32_t parseVarbinary(SToken* pToken, uint8_t** pData, uint32_t* nData, int32_t bytes) { + if (pToken->type != TK_NK_STRING) { return TSDB_CODE_PAR_INVALID_VARBINARY; } - if(isHex(pToken->z + 1, pToken->n - 2)){ - if(!isValidateHex(pToken->z + 1, pToken->n - 2)){ + if (isHex(pToken->z + 1, pToken->n - 2)) { + if (!isValidateHex(pToken->z + 1, pToken->n - 2)) { return TSDB_CODE_PAR_INVALID_VARBINARY; } - void* data = NULL; + void* data = NULL; uint32_t size = 0; - if(taosHex2Ascii(pToken->z + 1, pToken->n - 2, &data, &size) < 0){ + if (taosHex2Ascii(pToken->z + 1, pToken->n - 2, &data, &size) < 0) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -454,7 +450,7 @@ static int32_t parseVarbinary(SToken* pToken, uint8_t **pData, uint32_t *nData, } *pData = data; *nData = size; - }else{ + } else { *pData = taosMemoryCalloc(1, pToken->n); int32_t len = trimString(pToken->z, pToken->n, *pData, pToken->n); *nData = len; @@ -633,7 +629,7 @@ static int32_t parseTagToken(const char** end, SToken* pToken, SSchema* pSchema, } case TSDB_DATA_TYPE_VARBINARY: { code = parseVarbinary(pToken, &val->pData, &val->nData, pSchema->bytes); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { return generateSyntaxErrMsg(pMsgBuf, code, pSchema->name); } break; @@ -759,8 +755,8 @@ static int32_t buildCreateTbReq(SVnodeModifyOpStmt* pStmt, STag* pTag, SArray* p int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf, int8_t type) { if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && - pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && - pToken->type != TK_NK_BIN && pToken->type != TK_NK_VARIABLE) || + pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && pToken->type != TK_NK_BIN && + pToken->type != TK_NK_VARIABLE) || (pToken->n == 0) || (pToken->type == TK_NK_RP)) { return buildSyntaxErrMsg(pMsgBuf, "invalid data or symbol", pToken->z); } @@ -1114,8 +1110,8 @@ static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache, return code; } -static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMeta** pTableMeta, - bool* pMissCache, bool bUsingTable) { +static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, STableMeta** pTableMeta, bool* pMissCache, + bool bUsingTable) { SParseContext* pComCxt = pCxt->pComCxt; int32_t code = TSDB_CODE_SUCCESS; if (pComCxt->async) { @@ -1220,7 +1216,7 @@ static int32_t getTargetTableSchema(SInsertParseContext* pCxt, SVnodeModifyOpStm pCxt->missCache = true; return TSDB_CODE_SUCCESS; } - SNode* pTagCond = NULL; + SNode* pTagCond = NULL; int32_t code = checkAuth(pCxt->pComCxt, &pStmt->targetTableName, &pCxt->missCache, &pTagCond); if (TSDB_CODE_SUCCESS == code && !pCxt->missCache) { code = getTargetTableMetaAndVgroup(pCxt, pStmt, &pCxt->missCache); @@ -1365,13 +1361,11 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifyOp return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_COLUMNS, pStmt->pTableMeta, - &pTableCxt->boundColsInfo); + return parseBoundColumns(pCxt, &pStmt->pSql, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, - &pTableCxt->boundColsInfo); + return parseBoundColumns(pCxt, &pStmt->pBoundCols, BOUND_COLUMNS, pStmt->pTableMeta, &pTableCxt->boundColsInfo); } return TSDB_CODE_SUCCESS; @@ -1679,34 +1673,35 @@ static void clearColValArray(SArray* pCols) { typedef struct SStbRowsDataContext { SName stbName; - STableMeta* pStbMeta; - SNode* pTagCond; + STableMeta* pStbMeta; + SNode* pTagCond; SBoundColInfo boundColsInfo; // the following fields are for each stb row - SArray* aTagVals; - SArray* aColVals; - SArray* aTagNames; - SName ctbName; - STag* pTag; - STableMeta* pCtbMeta; + SArray* aTagVals; + SArray* aColVals; + SArray* aTagNames; + SName ctbName; + STag* pTag; + STableMeta* pCtbMeta; SVCreateTbReq* pCreateCtbReq; bool hasTimestampTag; bool isJsonTag; } SStbRowsDataContext; -typedef union SRowsDataContext{ - STableDataCxt* pTableDataCxt; +typedef union SRowsDataContext { + STableDataCxt* pTableDataCxt; SStbRowsDataContext* pStbRowsCxt; } SRowsDataContext; -static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) { +static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, + bool* pFoundCtbName) { *pFoundCtbName = false; int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg, TSDB_DATA_TYPE_BINARY); if (TK_NK_VARIABLE == pToken->type) { code = buildInvalidOperationMsg(&pCxt->msg, "not expected tbname"); } - if (code == TSDB_CODE_SUCCESS){ + if (code == TSDB_CODE_SUCCESS) { if (isNullValue(TSDB_DATA_TYPE_BINARY, pToken)) { return buildInvalidOperationMsg(&pCxt->msg, "tbname can not be null value"); } @@ -1717,7 +1712,7 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* if (pToken->z[i] == '.') { return buildInvalidOperationMsg(&pCxt->msg, "tbname can not contain '.'"); } else { - pStbRowsCxt->ctbName.tname[i] = pToken->z[i]; + pStbRowsCxt->ctbName.tname[i] = pToken->z[i]; } } pStbRowsCxt->ctbName.tname[pToken->n] = '\0'; @@ -1733,9 +1728,8 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* } static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, - SStbRowsDataContext* pStbRowsCxt, bool ctbFirst, - const SToken* tagTokens, SSchema* const* tagSchemas, - int numOfTagTokens) { + SStbRowsDataContext* pStbRowsCxt, bool ctbFirst, const SToken* tagTokens, + SSchema* const* tagSchemas, int numOfTagTokens) { int32_t code = TSDB_CODE_SUCCESS; uint8_t precision = pStmt->pTableMeta->tableInfo.precision; @@ -1749,8 +1743,8 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif } if (code == TSDB_CODE_SUCCESS) { - code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, pStbRowsCxt->aTagVals, - &pStbRowsCxt->pTag); + code = parseTagValue(&pCxt->msg, NULL, precision, pTagSchema, pTagToken, pStbRowsCxt->aTagNames, + pStbRowsCxt->aTagVals, &pStbRowsCxt->pTag); } } if (code == TSDB_CODE_SUCCESS && !pStbRowsCxt->isJsonTag) { @@ -1765,9 +1759,9 @@ static int32_t processCtbTagsAfterCtbName(SInsertParseContext* pCxt, SVnodeModif } static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, - SStbRowsDataContext* pStbRowsCxt, SToken* pToken, - const SBoundColInfo* pCols, const SSchema* pSchemas, - SToken* tagTokens, SSchema** tagSchemas, int* pNumOfTagTokens, bool* bFoundTbName) { + SStbRowsDataContext* pStbRowsCxt, SToken* pToken, const SBoundColInfo* pCols, + const SSchema* pSchemas, SToken* tagTokens, SSchema** tagSchemas, int* pNumOfTagTokens, + bool* bFoundTbName) { int32_t code = TSDB_CODE_SUCCESS; SArray* pTagNames = pStbRowsCxt->aTagNames; SArray* pTagVals = pStbRowsCxt->aTagVals; @@ -1791,10 +1785,10 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } if (pCols->pColIndex[i] < numOfCols) { const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]]; - SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); + SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]); code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, precision, pVal); if (TK_NK_VARIABLE == pToken->type) { - code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); + code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); } } else if (pCols->pColIndex[i] < tbnameIdx) { const SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]]; @@ -1808,11 +1802,11 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* code = buildInvalidOperationMsg(&pCxt->msg, "not expected row value"); } if (code == TSDB_CODE_SUCCESS) { - code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, &pStbRowsCxt->pTag); + code = parseTagValue(&pCxt->msg, ppSql, precision, (SSchema*)pTagSchema, pToken, pTagNames, pTagVals, + &pStbRowsCxt->pTag); } } - } - else if (pCols->pColIndex[i] == tbnameIdx) { + } else if (pCols->pColIndex[i] == tbnameIdx) { code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, bFoundTbName); } @@ -1826,22 +1820,21 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* return code; } -static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, - SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, - SToken* pToken, bool *pCtbFirst) { +static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql, + SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken, bool* pCtbFirst) { SBoundColInfo* pCols = &pStbRowsCxt->boundColsInfo; SSchema* pSchemas = getTableColumnSchema(pStbRowsCxt->pStbMeta); - bool bFoundTbName = false; + bool bFoundTbName = false; const char* pOrigSql = *ppSql; - int32_t code = TSDB_CODE_SUCCESS; - SToken tagTokens[TSDB_MAX_TAGS] = {0}; + int32_t code = TSDB_CODE_SUCCESS; + SToken tagTokens[TSDB_MAX_TAGS] = {0}; SSchema* tagSchemas[TSDB_MAX_TAGS] = {0}; - int numOfTagTokens = 0; + int numOfTagTokens = 0; - code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, tagTokens, - tagSchemas, &numOfTagTokens, &bFoundTbName); + code = doGetStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pToken, pCols, pSchemas, tagTokens, tagSchemas, + &numOfTagTokens, &bFoundTbName); if (code == TSDB_CODE_SUCCESS && !bFoundTbName) { code = buildSyntaxErrMsg(&pCxt->msg, "tbname value expected", pOrigSql); @@ -1870,7 +1863,8 @@ static int32_t getStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pS return code; } -static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SStbRowsDataContext* pStbRowsCxt) { +static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, + SStbRowsDataContext* pStbRowsCxt) { int32_t code = TSDB_CODE_SUCCESS; pStbRowsCxt->pCreateCtbReq = taosMemoryCalloc(1, sizeof(SVCreateTbReq)); @@ -1878,16 +1872,16 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod code = TSDB_CODE_OUT_OF_MEMORY; } if (code == TSDB_CODE_SUCCESS) { - insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, pStbRowsCxt->pStbMeta->uid, - pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, getNumOfTags(pStbRowsCxt->pStbMeta), - TSDB_DEFAULT_TABLE_TTL); + insBuildCreateTbReq(pStbRowsCxt->pCreateCtbReq, pStbRowsCxt->ctbName.tname, pStbRowsCxt->pTag, + pStbRowsCxt->pStbMeta->uid, pStbRowsCxt->stbName.tname, pStbRowsCxt->aTagNames, + getNumOfTags(pStbRowsCxt->pStbMeta), TSDB_DEFAULT_TABLE_TTL); pStbRowsCxt->pTag = NULL; } if (code == TSDB_CODE_SUCCESS) { char ctbFName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(&pStbRowsCxt->ctbName, ctbFName); - SVgroupInfo vg; + SVgroupInfo vg; SRequestConnInfo conn = {.pTrans = pCxt->pComCxt->pTransporter, .requestId = pCxt->pComCxt->requestId, .requestObjRefId = pCxt->pComCxt->requestRid, @@ -1907,7 +1901,6 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod return code; } - static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) { if (pStbRowsCxt == NULL) return; @@ -1938,7 +1931,7 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } if (code == TSDB_CODE_SUCCESS && bFirstTable) { - code = processCtbAutoCreationAndCtbMeta(pCxt, pStmt, pStbRowsCxt); + code = processCtbAutoCreationAndCtbMeta(pCxt, pStmt, pStbRowsCxt); } code = insGetTableDataCxt(pStmt->pTableBlockHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), @@ -1948,7 +1941,9 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt SRow** pRow = taosArrayReserve((*ppTableDataCxt)->pData->aRowP, 1); code = tRowBuild(pStbRowsCxt->aColVals, (*ppTableDataCxt)->pSchema, pRow); if (TSDB_CODE_SUCCESS == code) { - insCheckTableDataOrder(*ppTableDataCxt, TD_ROW_KEY(*pRow)); + SRowKey key; + tRowGetKey(*pRow, &key); + insCheckTableDataOrder(*ppTableDataCxt, &key); } } @@ -1957,11 +1952,12 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt } clearStbRowsDataContext(pStbRowsCxt); - + return TSDB_CODE_SUCCESS; } -static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, SToken* pToken) { +static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataCxt* pTableCxt, bool* pGotRow, + SToken* pToken) { SBoundColInfo* pCols = &pTableCxt->boundColsInfo; bool isParseBindParam = false; SSchema* pSchemas = getTableColumnSchema(pTableCxt->pMeta); @@ -2014,7 +2010,9 @@ static int parseOneRow(SInsertParseContext* pCxt, const char** pSql, STableDataC SRow** pRow = taosArrayReserve(pTableCxt->pData->aRowP, 1); code = tRowBuild(pTableCxt->pValues, pTableCxt->pSchema, pRow); if (TSDB_CODE_SUCCESS == code) { - insCheckTableDataOrder(pTableCxt, TD_ROW_KEY(*pRow)); + SRowKey key; + tRowGetKey(*pRow, &key); + insCheckTableDataOrder(pTableCxt, &key); } } @@ -2112,10 +2110,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt code = parseOneRow(pCxt, (const char**)&pRow, rowsDataCxt.pTableDataCxt, &gotRow, &token); } else { STableDataCxt* pTableDataCxt = NULL; - code = parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token, &pTableDataCxt); + code = + parseOneStbRow(pCxt, pStmt, (const char**)&pRow, rowsDataCxt.pStbRowsCxt, &gotRow, &token, &pTableDataCxt); if (code == TSDB_CODE_SUCCESS) { SStbRowsDataContext* pStbRowsCxt = rowsDataCxt.pStbRowsCxt; - void* pData = pTableDataCxt; + void* pData = pTableDataCxt; taosHashPut(pStmt->pTableCxtHashObj, &pStbRowsCxt->pCtbMeta->uid, sizeof(pStbRowsCxt->pCtbMeta->uid), &pData, POINTER_BYTES); } @@ -2149,11 +2148,11 @@ static int32_t parseCsvFile(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt return code; } -static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, SRowsDataContext rowsDataCxt) { +static int32_t parseDataFromFileImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, + SRowsDataContext rowsDataCxt) { // init only for file if (NULL == pStmt->pTableCxtHashObj) { - pStmt->pTableCxtHashObj = - taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pStmt->pTableCxtHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } int32_t numOfRows = 0; int32_t code = parseCsvFile(pCxt, pStmt, rowsDataCxt, &numOfRows); @@ -2267,7 +2266,7 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa cloneTableMeta(pStbRowsCxt->pStbMeta, &pStbRowsCxt->pCtbMeta); pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE; pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid; - + pStbRowsCxt->aTagNames = taosArrayInit(8, TSDB_COL_NAME_LEN); pStbRowsCxt->aTagVals = taosArrayInit(8, sizeof(STagVal)); @@ -2285,7 +2284,8 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; if (!pStmt->pBoundCols) { - return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", pStmt->pSql); + return buildSyntaxErrMsg(&pCxt->msg, "(...tbname, ts...) bounded cols is expected for supertable insertion", + pStmt->pSql); } SStbRowsDataContext* pStbRowsCxt = NULL; @@ -2297,7 +2297,7 @@ static int32_t parseInsertStbClauseBottom(SInsertParseContext* pCxt, SVnodeModif pStbRowsCxt->hasTimestampTag = false; for (int32_t i = 0; i < pStbRowsCxt->boundColsInfo.numOfBound; ++i) { int16_t schemaIndex = pStbRowsCxt->boundColsInfo.pColIndex[i]; - if (schemaIndex != getTbnameSchemaIndex(pStmt->pTableMeta) && schemaIndex >= getNumOfColumns(pStmt->pTableMeta) ) { + if (schemaIndex != getTbnameSchemaIndex(pStmt->pTableMeta) && schemaIndex >= getNumOfColumns(pStmt->pTableMeta)) { if (pStmt->pTableMeta->schema[schemaIndex].type == TSDB_DATA_TYPE_TIMESTAMP) { pStbRowsCxt->hasTimestampTag = true; } @@ -2326,8 +2326,8 @@ static int32_t parseInsertTableClauseBottom(SInsertParseContext* pCxt, SVnodeMod return buildInvalidOperationMsg(&pCxt->msg, "insert into super table syntax is not supported for stmt"); } if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; - int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); + STableDataCxt* pTableCxt = NULL; + int32_t code = parseSchemaClauseBottom(pCxt, pStmt, &pTableCxt); SRowsDataContext rowsDataCxt; rowsDataCxt.pTableDataCxt = pTableCxt; if (TSDB_CODE_SUCCESS == code) { @@ -2442,7 +2442,7 @@ static int32_t parseInsertBodyBottom(SInsertParseContext* pCxt, SVnodeModifyOpSt bool fileOnly = (pStmt->insertType == TSDB_QUERY_TYPE_FILE_INSERT); if (fileOnly) { - // none data, skip merge & buildvgdata + // none data, skip merge & buildvgdata if (0 == taosHashGetSize(pStmt->pTableCxtHashObj)) { pCxt->needRequest = false; return TSDB_CODE_SUCCESS; @@ -2503,7 +2503,7 @@ static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, S pStmt->freeHashFunc = insDestroyTableDataCxtHashMap; pStmt->freeArrayFunc = insDestroyVgroupDataCxtList; pStmt->freeStbRowsCxtFunc = destroyStbRowsDataContext; - + if (!reentry) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); pStmt->pTableBlockHashObj = @@ -2625,7 +2625,7 @@ static int32_t checkSubtablePrivilegeForTable(const SArray* pTables, SVnodeModif } static int32_t processTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, - SVnodeModifyOpStmt* pStmt, bool isStb) { + SVnodeModifyOpStmt* pStmt, bool isStb) { int32_t code = TSDB_CODE_SUCCESS; if (!isStb && TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); @@ -2767,11 +2767,11 @@ static int32_t parseInsertSqlFromStart(SInsertParseContext* pCxt, SVnodeModifyOp } static int32_t parseInsertSqlFromCsv(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SRowsDataContext rowsDataCxt; if (!pStmt->stbSyntax) { - STableDataCxt* pTableCxt = NULL; + STableDataCxt* pTableCxt = NULL; code = getTableDataCxt(pCxt, pStmt, &pTableCxt); rowsDataCxt.pTableDataCxt = pTableCxt; } else { diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 3f87f79301..7e99867b2a 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -170,9 +170,7 @@ static void initColValues(STableMeta* pTableMeta, SArray* pValues) { } } -void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { - initColValues(pTableMeta, aColValues); -} +void insInitColValues(STableMeta* pTableMeta, SArray* aColValues) { initColValues(pTableMeta, aColValues); } int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { pInfo->numOfCols = numOfBound; @@ -187,21 +185,22 @@ int32_t insInitBoundColsInfo(int32_t numOfBound, SBoundColInfo* pInfo) { return TSDB_CODE_SUCCESS; } -void insCheckTableDataOrder(STableDataCxt* pTableCxt, TSKEY tsKey) { +void insCheckTableDataOrder(STableDataCxt* pTableCxt, SRowKey* rowKey) { // once the data block is disordered, we do NOT keep last timestamp any more if (!pTableCxt->ordered) { return; } - if (tsKey < pTableCxt->lastTs) { + if (tRowKeyCompare(rowKey, &pTableCxt->lastKey) < 0) { pTableCxt->ordered = false; } - if (tsKey == pTableCxt->lastTs) { + if (tRowKeyCompare(rowKey, &pTableCxt->lastKey) == 0) { pTableCxt->duplicateTs = true; } - pTableCxt->lastTs = tsKey; + // TODO: for variable length data type, we need to copy it out + pTableCxt->lastKey = *rowKey; return; } @@ -217,7 +216,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat int32_t code = TSDB_CODE_SUCCESS; - pTableCxt->lastTs = 0; + pTableCxt->lastKey = (SRowKey){0}; pTableCxt->ordered = true; pTableCxt->duplicateTs = false; @@ -254,7 +253,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat pTableCxt->pData->uid = pTableMeta->uid; pTableCxt->pData->sver = pTableMeta->sversion; pTableCxt->pData->pCreateTbReq = pCreateTbReq != NULL ? *pCreateTbReq : NULL; - if(pCreateTbReq != NULL) *pCreateTbReq = NULL; + if (pCreateTbReq != NULL) *pCreateTbReq = NULL; if (pTableCxt->pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { pTableCxt->pData->aCol = taosArrayInit(128, sizeof(SColData)); if (NULL == pTableCxt->pData->aCol) { @@ -276,7 +275,7 @@ static int32_t createTableDataCxt(STableMeta* pTableMeta, SVCreateTbReq** pCreat } static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SSubmitTbData* pTmp = taosMemoryCalloc(1, sizeof(SSubmitTbData)); if (NULL == pTmp) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -313,11 +312,10 @@ static int32_t rebuildTableData(SSubmitTbData* pSrc, SSubmitTbData** pDst) { if (TSDB_CODE_SUCCESS == code) { *pDst = pTmp; } - + return code; } - static void resetColValues(SArray* pValues) { int32_t num = taosArrayGetSize(pValues); for (int32_t i = 0; i < num; ++i) { @@ -463,7 +461,7 @@ static int32_t createVgroupDataCxt(STableDataCxt* pTableCxt, SHashObj* pVgroupHa int32_t code = taosHashPut(pVgroupHash, &pVgCxt->vgId, sizeof(pVgCxt->vgId), &pVgCxt, POINTER_BYTES); if (TSDB_CODE_SUCCESS == code) { taosArrayPush(pVgroupList, &pVgCxt); -// uDebug("td23101 2vgId:%d, uid:%" PRIu64, pVgCxt->vgId, pTableCxt->pMeta->uid); + // uDebug("td23101 2vgId:%d, uid:%" PRIu64, pVgCxt->vgId, pTableCxt->pMeta->uid); *pOutput = pVgCxt; } else { insDestroyVgroupDataCxt(pVgCxt); @@ -613,7 +611,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, if (TSDB_CODE_SUCCESS == code) { dst->numOfTables = taosArrayGetSize(src->pData->aSubmitTbData); code = taosHashGetDup(pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg); -// uError("td23101 3vgId:%d, numEps:%d", src->vgId, dst->vg.epSet.numOfEps); + // uError("td23101 3vgId:%d, numEps:%d", src->vgId, dst->vg.epSet.numOfEps); } if (TSDB_CODE_SUCCESS == code) { code = buildSubmitReq(src->vgId, src->pData, &dst->pData, &dst->size); @@ -634,7 +632,7 @@ int32_t insBuildVgDataBlocks(SHashObj* pVgroupsHashObj, SArray* pVgDataCxtList, static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { for (int i = 0; i < numFields; i++) { - if(strcmp(pSchema->name, fields[i].name) == 0){ + if (strcmp(pSchema->name, fields[i].name) == 0) { return true; } } @@ -644,7 +642,8 @@ static bool findFileds(SSchema* pSchema, TAOS_FIELD* fields, int numFields) { int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreateTbReq** pCreateTb, TAOS_FIELD* tFields, int numFields, bool needChangeLength) { - void* tmp = taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); + void* tmp = + taosHashGet(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid)); STableDataCxt* pTableCxt = NULL; int ret = insGetTableDataCxt(((SVnodeModifyOpStmt*)(query->pRoot))->pTableBlockHashObj, &pTableMeta->uid, sizeof(pTableMeta->uid), pTableMeta, pCreateTb, &pTableCxt, true, false); @@ -654,7 +653,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } pTableCxt->pData->flags |= TD_REQ_FROM_TAOX; - if(tmp == NULL){ + if (tmp == NULL) { ret = initTableColSubmitData(pTableCxt); if (ret != TSDB_CODE_SUCCESS) { uError("initTableColSubmitData error"); @@ -663,8 +662,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } char* p = (char*)data; - // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each column - // length | + // | version | total length | total rows | blankFill | total columns | flag seg| block group id | column schema | each + // column length | int32_t version = *(int32_t*)data; p += sizeof(int32_t); p += sizeof(int32_t); @@ -699,8 +698,8 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate ret = TSDB_CODE_INVALID_PARA; goto end; } - if(tFields == NULL){ - for (int j = 0; j < boundInfo->numOfBound; j++){ + if (tFields == NULL) { + for (int j = 0; j < boundInfo->numOfBound; j++) { SSchema* pColSchema = &pSchema[j]; SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { @@ -717,7 +716,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate } char* pData = pStart; ret = tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); - if(ret != 0){ + if (ret != 0) { goto end; } fields += sizeof(int8_t) + sizeof(int32_t); @@ -727,11 +726,11 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate pStart += colLength[j]; } } - }else{ + } else { for (int i = 0; i < numFields; i++) { - for (int j = 0; j < boundInfo->numOfBound; j++){ - SSchema* pColSchema = &pSchema[j]; - if(strcmp(pColSchema->name, tFields[i].name) == 0){ + for (int j = 0; j < boundInfo->numOfBound; j++) { + SSchema* pColSchema = &pSchema[j]; + if (strcmp(pColSchema->name, tFields[i].name) == 0) { if (*fields != pColSchema->type && *(int32_t*)(fields + sizeof(int8_t)) != pColSchema->bytes) { uError("type or bytes not equal"); ret = TSDB_CODE_INVALID_PARA; @@ -748,7 +747,7 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, j); ret = tColDataAddValueByDataBlock(pCol, pColSchema->type, pColSchema->bytes, numOfRows, offset, pData); - if(ret != 0){ + if (ret != 0) { goto end; } fields += sizeof(int8_t) + sizeof(int32_t); @@ -761,17 +760,16 @@ int rawBlockBindData(SQuery* query, STableMeta* pTableMeta, void* data, SVCreate break; } } - } for (int c = 0; c < boundInfo->numOfBound; ++c) { - if( boundInfo->pColIndex[c] != -1){ + if (boundInfo->pColIndex[c] != -1) { SColData* pCol = taosArrayGet(pTableCxt->pData->aCol, c); ret = tColDataAddValueByDataBlock(pCol, 0, 0, numOfRows, NULL, NULL); - if(ret != 0){ + if (ret != 0) { goto end; } - }else{ + } else { boundInfo->pColIndex[c] = c; // restore for next block } } From 47410db46aab6548de286368b1477addf95a6407 Mon Sep 17 00:00:00 2001 From: slzhou Date: Tue, 26 Mar 2024 16:30:53 +0800 Subject: [PATCH 9/9] fix: set hasPk and pkBytes for partial func and merge func --- source/libs/function/src/builtinsimpl.c | 10 ++++++++++ source/libs/function/src/functionMgt.c | 8 ++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 70fe2b19df..b63fa6a167 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2343,6 +2343,11 @@ EFuncDataRequired firstDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo) { SFirstLastRes* pResult = GET_ROWCELL_INTERBUF(pEntry); if (pResult->hasResult) { + if (pResult->pkBytes > 0) { + pResult->pkData = pResult->buf + pResult->bytes; + } else { + pResult->pkData = NULL; + } if (pResult->ts < pBlockInfo->window.skey) { return FUNC_DATA_REQUIRED_NOT_LOAD; } else if (pResult->ts == pBlockInfo->window.skey && pResult->pkData) { @@ -2366,6 +2371,11 @@ EFuncDataRequired lastDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo) { SFirstLastRes* pResult = GET_ROWCELL_INTERBUF(pEntry); if (pResult->hasResult) { + if (pResult->pkBytes > 0) { + pResult->pkData = pResult->buf + pResult->bytes; + } else { + pResult->pkData = NULL; + } if (pResult->ts > pBlockInfo->window.ekey) { return FUNC_DATA_REQUIRED_NOT_LOAD; } else if (pResult->ts == pBlockInfo->window.ekey && pResult->pkData) { diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 697efb63fd..03bc1b4e59 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -415,6 +415,8 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc); taosCreateMD5Hash(name, len); strncpy((*pPartialFunc)->node.aliasName, name, TSDB_COL_NAME_LEN - 1); + (*pPartialFunc)->hasPk = pSrcFunc->hasPk; + (*pPartialFunc)->pkBytes = pSrcFunc->pkBytes; return TSDB_CODE_SUCCESS; } @@ -453,7 +455,8 @@ static int32_t createMidFunction(const SFunctionNode* pSrcFunc, const SFunctionN } else { nodesDestroyList(pParameterList); } - + (*pMidFunc)->hasPk = pPartialFunc->hasPk; + (*pMidFunc)->pkBytes = pPartialFunc->pkBytes; return code; } @@ -482,7 +485,8 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio } else { nodesDestroyList(pParameterList); } - + (*pMergeFunc)->hasPk = pPartialFunc->hasPk; + (*pMergeFunc)->pkBytes = pPartialFunc->pkBytes; return code; }