diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 8a3b28a125..77b6ca1833 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -438,7 +438,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW return 0; } -STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); +STSchema* tdGetSTSChemaFromSSChema(SSchema* pSchema, int32_t nCols, int32_t sver); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 8611278550..3e733d291b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4941,14 +4941,14 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) { return 0; } -STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { +STSchema *tdGetSTSChemaFromSSChema(SSchema *pSchema, int32_t nCols, int32_t sver) { STSchemaBuilder schemaBuilder = {0}; - if (tdInitTSchemaBuilder(&schemaBuilder, 1) < 0) { + if (tdInitTSchemaBuilder(&schemaBuilder, sver) < 0) { return NULL; } for (int i = 0; i < nCols; i++) { - SSchema *schema = *pSchema + i; + SSchema *schema = pSchema + i; if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index df5bf64acf..754e142437 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -568,6 +568,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { int32_t maxVarDataLen = 0; int32_t iColVal = 0; void *varBuf = NULL; + bool isAlloc = false; ASSERT(nColVal > 1); @@ -610,8 +611,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { ++iColVal; } - *ppRow = (STSRow *)taosMemoryCalloc( - 1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1)); + if (!(*ppRow)) { + *ppRow = (STSRow *)taosMemoryCalloc( + 1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1)); + isAlloc = true; + } if (!(*ppRow)) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -621,7 +625,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { if (maxVarDataLen > 0) { varBuf = taosMemoryMalloc(maxVarDataLen); if (!varBuf) { - taosMemoryFreeClear(*ppRow); + if(isAlloc) { + taosMemoryFreeClear(*ppRow); + } terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -1323,12 +1329,11 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV SCellVal cv; SValue value; - ASSERT(iCol > 0); + ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0)); if (TD_IS_TP_ROW(pRow)) { tdSTpRowGetVal(pRow, pTColumn->colId, pTColumn->type, pTSchema->flen, pTColumn->offset, iCol - 1, &cv); } else if (TD_IS_KV_ROW(pRow)) { - ASSERT(iCol > 0); tdSKvRowGetVal(pRow, pTColumn->colId, iCol - 1, &cv); } else { ASSERT(0); diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index d16e35ff07..65f21bee40 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -116,7 +116,7 @@ STSchema *genSTSchema(int16_t nCols) { } STSchema *pResult = NULL; - pResult = tdGetSTSChemaFromSSChema(&pSchema, nCols); + pResult = tdGetSTSChemaFromSSChema(pSchema, nCols, 1); taosMemoryFree(pSchema); return pResult; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index b2679ee245..5469960ce6 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -634,7 +634,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); pTask->tbSink.pTSchema = - tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols); + tdGetSTSChemaFromSSChema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1); ASSERT(pTask->tbSink.pTSchema); } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 05e8c1094d..76e57a9a07 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1200,7 +1200,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, *gotRow = true; #ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&schema, spd->numOfCols); + STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); #endif @@ -1972,7 +1972,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in } } #ifdef TD_DEBUG_PRINT_ROW - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); + STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); #endif @@ -2057,7 +2057,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu #ifdef TD_DEBUG_PRINT_ROW if (rowEnd) { - STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); + STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1); tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); } diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index 9e1d8dba8b..ae123a3563 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -19,6 +19,7 @@ #include "parInt.h" #include "parUtil.h" #include "querynodes.h" +#include "tRealloc.h" #define IS_RAW_PAYLOAD(t) \ (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert @@ -34,6 +35,32 @@ typedef struct SBlockKeyInfo { SBlockKeyTuple* pKeyTuple; } SBlockKeyInfo; +typedef struct { + int32_t index; + SArray* rowArray; // array of merged rows(mem allocated by tRealloc/free by tFree) + STSchema* pSchema; + int64_t tbUid; // suid for child table, uid for normal table +} SBlockRowMerger; + +static FORCE_INLINE void tdResetSBlockRowMerger(SBlockRowMerger* pMerger) { + if (pMerger) { + pMerger->index = -1; + } +} + +static void tdFreeSBlockRowMerger(SBlockRowMerger* pMerger) { + if (pMerger) { + int32_t size = taosArrayGetSize(pMerger->rowArray); + for (int32_t i = 0; i < size; ++i) { + tFree(*(void**)taosArrayGet(pMerger->rowArray, i)); + } + taosArrayDestroy(pMerger->rowArray); + + taosMemoryFreeClear(pMerger->pSchema); + taosMemoryFree(pMerger); + } +} + static int32_t rowDataCompar(const void* lhs, const void* rhs) { TSKEY left = *(TSKEY*)lhs; TSKEY right = *(TSKEY*)rhs; @@ -328,7 +355,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) { } // data block is disordered, sort it in ascending order -int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { +static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; int16_t nRows = pBlocks->numOfRows; @@ -396,6 +423,201 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey return 0; } +static void* tdGetCurRowFromBlockMerger(SBlockRowMerger* pBlkRowMerger) { + if (pBlkRowMerger && (pBlkRowMerger->index >= 0)) { + ASSERT(pBlkRowMerger->index < taosArrayGetSize(pBlkRowMerger->rowArray)); + return *(void**)taosArrayGet(pBlkRowMerger->rowArray, pBlkRowMerger->index); + } + return NULL; +} + +static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp, int32_t nDupRows, + SBlockRowMerger** pBlkRowMerger, int32_t rowSize) { + ASSERT(nDupRows > 1); + SBlockKeyTuple* pStartKeyTp = pEndKeyTp - (nDupRows - 1); + ASSERT(pStartKeyTp->skey == pEndKeyTp->skey); + + // TODO: optimization if end row is all normal +#if 0 + STSRow* pEndRow = (STSRow*)pEndKeyTp->payloadAddr; + if(isNormal(pEndRow)) { // set the end row if it is normal and return directly + pStartKeyTp->payloadAddr = pEndKeyTp->payloadAddr; + return TSDB_CODE_SUCCESS; + } +#endif + + if (!(*pBlkRowMerger)) { + (*pBlkRowMerger) = taosMemoryCalloc(1, sizeof(**pBlkRowMerger)); + if (!(*pBlkRowMerger)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + (*pBlkRowMerger)->index = -1; + if (!(*pBlkRowMerger)->rowArray) { + (*pBlkRowMerger)->rowArray = taosArrayInit(1, sizeof(void*)); + if (!(*pBlkRowMerger)->rowArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + } + + if ((*pBlkRowMerger)->pSchema) { + if ((*pBlkRowMerger)->pSchema->version != pTableMeta->sversion) { + taosMemoryFreeClear((*pBlkRowMerger)->pSchema); + } else { + if ((*pBlkRowMerger)->tbUid != (pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid)) { + taosMemoryFreeClear((*pBlkRowMerger)->pSchema); + } + } + } + + if (!(*pBlkRowMerger)->pSchema) { + (*pBlkRowMerger)->pSchema = + tdGetSTSChemaFromSSChema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion); + + if (!(*pBlkRowMerger)->pSchema) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + (*pBlkRowMerger)->tbUid = pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid; + } + + void* pDestRow = NULL; + ++((*pBlkRowMerger)->index); + if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) { + void* pAlloc = *(void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index); + if (tRealloc((uint8_t**)&pAlloc, rowSize) != 0) { + return TSDB_CODE_FAILED; + } + pDestRow = pAlloc; + } else { + if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) { + return TSDB_CODE_FAILED; + } + taosArrayPush((*pBlkRowMerger)->rowArray, &pDestRow); + } + + // merge rows to pDestRow + STSchema* pSchema = (*pBlkRowMerger)->pSchema; + SArray* pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal)); + for (int32_t i = 0; i < pSchema->numOfCols; ++i) { + SColVal colVal = {0}; + for (int32_t j = 0; j < nDupRows; ++j) { + tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal); + if (!colVal.isNone) { + break; + } + } + taosArrayPush(pArray, &colVal); + } + if (tdSTSRowNew(pArray, pSchema, (STSRow**)&pDestRow) < 0) { + taosArrayDestroy(pArray); + return TSDB_CODE_FAILED; + } + + taosArrayDestroy(pArray); + return TSDB_CODE_SUCCESS; +} + +// data block is disordered, sort it in ascending order, and merge dup rows if exists +static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo, + SBlockRowMerger** ppBlkRowMerger) { + SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; + STableMeta* pTableMeta = dataBuf->pTableMeta; + int16_t nRows = pBlocks->numOfRows; + + // size is less than the total size, since duplicated rows may be removed. + + // allocate memory + size_t nAlloc = nRows * sizeof(SBlockKeyTuple); + if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { + char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc); + if (tmp == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp; + pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc; + } + memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); + + tdResetSBlockRowMerger(*ppBlkRowMerger); + + int32_t extendedRowSize = getExtendedRowSize(dataBuf); + SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + char* pBlockData = pBlocks->data + pBlocks->schemaLen; + int n = 0; + while (n < nRows) { + pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData); + pBlkKeyTuple->payloadAddr = pBlockData; + pBlkKeyTuple->index = n; + + // next loop + pBlockData += extendedRowSize; + ++pBlkKeyTuple; + ++n; + } + + if (!dataBuf->ordered) { + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + + taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable); + + pBlkKeyTuple = pBlkKeyInfo->pKeyTuple; + bool hasDup = false; + int32_t nextPos = 0; + int32_t i = 0; + int32_t j = 1; + + while (j < nRows) { + TSKEY ti = (pBlkKeyTuple + i)->skey; + TSKEY tj = (pBlkKeyTuple + j)->skey; + + if (ti == tj) { + ++j; + continue; + } + + if ((j - i) > 1) { + if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { + return TSDB_CODE_FAILED; + } + (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); + if (!hasDup) { + hasDup = true; + } + i = j; + } else { + if (hasDup) { + memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); + } + ++i; + } + + ++nextPos; + ++j; + } + + if ((j - i) > 1) { + ASSERT((pBlkKeyTuple + i)->skey == (pBlkKeyTuple + j - 1)->skey); + if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) { + return TSDB_CODE_FAILED; + } + (pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger); + } else if (hasDup) { + memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple)); + } + + dataBuf->ordered = true; + pBlocks->numOfRows = nextPos + 1; + } + + dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize; + dataBuf->prevTS = INT64_MIN; + + return TSDB_CODE_SUCCESS; +} + // Erase the empty space reserved for binary data static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, bool isRawPayload) { @@ -464,6 +686,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p STableDataBlocks** p = taosHashIterate(pHashObj, NULL); STableDataBlocks* pOneTableBlock = *p; SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock + SBlockRowMerger *pBlkRowMerger = NULL; + while (pOneTableBlock) { SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; if (pBlocks->numOfRows > 0) { @@ -473,6 +697,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); if (ret != TSDB_CODE_SUCCESS) { + tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); taosMemoryFreeClear(blkKeyInfo.pKeyTuple); @@ -490,6 +715,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p if (tmp != NULL) { dataBuf->pData = tmp; } else { // failed to allocate memory, free already allocated memory and return error code + tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); taosMemoryFreeClear(dataBuf->pData); @@ -501,7 +727,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p if (isRawPayload) { sortRemoveDataBlockDupRowsRaw(pOneTableBlock); } else { - if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { + if ((code = sortMergeDataBlockDupRows(pOneTableBlock, &blkKeyInfo, &pBlkRowMerger)) != 0) { + tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); destroyBlockArrayList(pVnodeDataBlockList); taosMemoryFreeClear(dataBuf->pData); @@ -529,6 +756,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p } // free the table data blocks; + tdFreeSBlockRowMerger(pBlkRowMerger); taosHashCleanup(pVnodeDataBlockHashList); taosMemoryFreeClear(blkKeyInfo.pKeyTuple); *pVgDataBlocks = pVnodeDataBlockList; diff --git a/tests/script/tsim/insert/update0.sim b/tests/script/tsim/insert/update0.sim index 09b8de08e6..c6843acb9d 100644 --- a/tests/script/tsim/insert/update0.sim +++ b/tests/script/tsim/insert/update0.sim @@ -79,8 +79,8 @@ if $rows != 3 then return -1 endi -if $data01 != 103 then - print data01 $data01 != 103 +if $data01 != 303 then + print data01 $data01 != 303 return -1 endi @@ -89,8 +89,8 @@ if $data11 != 80 then return -1 endi -if $data21 != 40 then - print data21 $data21 != 40 +if $data21 != 60 then + print data21 $data21 != 60 return -1 endi @@ -138,8 +138,8 @@ if $rows != 3 then return -1 endi -if $data01 != 103 then - print data01 $data01 != 103 +if $data01 != 303 then + print data01 $data01 != 303 return -1 endi @@ -148,8 +148,8 @@ if $data11 != 80 then return -1 endi -if $data21 != 40 then - print data21 $data21 != 40 +if $data21 != 60 then + print data21 $data21 != 60 return -1 endi @@ -208,8 +208,8 @@ if $data01 != 10 then return -1 endi -if $data11 != 103 then - print data11 $data11 != 103 +if $data11 != 303 then + print data11 $data11 != 303 return -1 endi @@ -218,8 +218,8 @@ if $data21 != NULL then return -1 endi -if $data31 != 40 then - print data31 $data31 != 40 +if $data31 != 60 then + print data31 $data31 != 60 return -1 endi diff --git a/tests/script/tsim/insert/update1_sort_merge.sim b/tests/script/tsim/insert/update1_sort_merge.sim new file mode 100644 index 0000000000..c4f7877220 --- /dev/null +++ b/tests/script/tsim/insert/update1_sort_merge.sim @@ -0,0 +1,818 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database +sql drop database if exists d0 +sql create database d0 keep 365000d,365000d,365000d +sql use d0 + +print =============== create super table +sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20)); + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table +sql create table ct1 using stb tags("BeiJing", "ChaoYang") +sql create table ct2 using stb tags("BeiJing", "HaiDian") +sql create table ct3 using stb tags("BeiJing", "PingGu") +sql create table ct4 using stb tags("BeiJing", "YanQing") + +sql show tables +if $rows != 4 then + print rows $rows != 4 + return -1 +endi + +print =============== step 1 insert records into ct1 - taosd merge +sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10, 20); +sql insert into ct1(ts,c1,c2,c3,c4) values('2022-05-03 16:59:00.011', 11, NULL, 'binary', 'nchar'); +sql insert into ct1 values('2022-05-03 16:59:00.016', 16, NULL, NULL, 'nchar', NULL); +sql insert into ct1 values('2022-05-03 16:59:00.016', 17, NULL, NULL, 'nchar', 170); +sql insert into ct1 values('2022-05-03 16:59:00.020', 20, NULL, NULL, 'nchar', 200); +sql insert into ct1 values('2022-05-03 16:59:00.016', 18, NULL, NULL, 'nchar', 180); +sql insert into ct1 values('2022-05-03 16:59:00.021', 21, NULL, NULL, 'nchar', 210); +sql insert into ct1 values('2022-05-03 16:59:00.022', 22, NULL, NULL, 'nchar', 220); + +print =============== step 2 insert records into ct1/ct2 - taosc merge for 2022-05-03 16:59:00.010 +sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10,10), ('2022-05-03 16:59:00.010',20,10.0), ('2022-05-03 16:59:00.010',30,NULL) ct2(ts,c1) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',20) ct1(ts,c2) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',100) ct1(ts,c3) values('2022-05-03 16:59:00.010','bin1'), ('2022-05-03 16:59:00.010','bin2') ct1(ts,c4,c5) values('2022-05-03 16:59:00.010',NULL,NULL), ('2022-05-03 16:59:00.010','nchar4',1000.01) ct2(ts,c2,c3,c4,c5) values('2022-05-03 16:59:00.010',20,'xkl','zxc',10); + +print =============== step 3 insert records into ct3 +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.020', 10,10); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.021', 10,10), ('2022-05-03 16:59:00.021',20,20.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.022', 30,30), ('2022-05-03 16:59:00.022',40,40.0),('2022-05-03 16:59:00.022',50,50.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.023', 60,60), ('2022-05-03 16:59:00.023',70,70.0),('2022-05-03 16:59:00.023',80,80.0), ('2022-05-03 16:59:00.023',90,90.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.024', 100,100), ('2022-05-03 16:59:00.025',110,110.0),('2022-05-03 16:59:00.025',120,120.0), ('2022-05-03 16:59:00.025',130,130.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.030', 140,140), ('2022-05-03 16:59:00.030',150,150.0),('2022-05-03 16:59:00.031',160,160.0), ('2022-05-03 16:59:00.030',170,170.0), ('2022-05-03 16:59:00.031',180,180.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.042', 190,190), ('2022-05-03 16:59:00.041',200,200.0),('2022-05-03 16:59:00.040',210,210.0); +sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.050', 220,220), ('2022-05-03 16:59:00.051',230,230.0),('2022-05-03 16:59:00.052',240,240.0); + +print =============== step 4 insert records into ct4 +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.020', 10,'b0','n0'); +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.021', 20,'b1','n1'), ('2022-05-03 16:59:00.021',30,'b2','n2'); +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.022', 40,'b3','n3'), ('2022-05-03 16:59:00.022',40,'b4','n4'),('2022-05-03 16:59:00.022',50,'b5','n5'); +sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.023', 60,'b6','n6'), ('2022-05-03 16:59:00.024',70,'b7','n7'),('2022-05-03 16:59:00.024',80,'b8','n8'), ('2022-05-03 16:59:00.023',90,'b9','n9'); + + + +print =============== step 5 query records of ct1 from memory(taosc and taosd merge) +sql select * from ct1; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 + +if $rows != 6 then + print rows $rows != 6 + return -1 +endi + +if $data01 != 30 then + print data01 $data01 != 30 + return -1 +endi + +if $data02 != 100.000000000 then + print data02 $data02 != 100.000000000 + return -1 +endi + +if $data03 != bin2 then + print data03 $data03 != bin2 + return -1 +endi + +if $data04 != nchar4 then + print data04 $data04 != nchar4 + return -1 +endi + +if $data05 != 1000.010000000 then + print data05 $data05 != 1000.010000000 + return -1 +endi + +if $data11 != 11 then + print data11 $data11 != 11 + return -1 +endi + +if $data12 != NULL then + print data12 $data12 != NULL + return -1 +endi + +if $data13 != binary then + print data13 $data13 != binary + return -1 +endi + +if $data14 != nchar then + print data14 $data14 != nchar + return -1 +endi + +if $data15 != NULL then + print data15 $data15 != NULL + return -1 +endi + +if $data51 != 22 then + print data51 $data51 != 22 + return -1 +endi + +if $data52 != NULL then + print data52 $data52 != NULL + return -1 +endi + +if $data53 != NULL then + print data53 $data53 != NULL + return -1 +endi + +if $data54 != nchar then + print data54 $data54 != nchar + return -1 +endi + +if $data55 != 220.000000000 then + print data55 $data55 != 220.000000000 + return -1 +endi + + +print =============== step 6 query records of ct2 from memory(taosc and taosd merge) +sql select * from ct2; +print $data00 $data01 $data02 $data03 $data04 $data05 + +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != 20 then + print data01 $data01 != 20 + return -1 +endi + +if $data02 != 20.000000000 then + print data02 $data02 != 20.000000000 + return -1 +endi + +if $data03 != xkl then + print data03 $data03 != xkl + return -1 +endi + +if $data04 != zxc then + print data04 $data04 != zxc + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +print =============== step 7 query records of ct3 from memory +sql select * from ct3; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 +print $data60 $data61 $data62 $data63 $data64 $data65 +print $data70 $data71 $data72 $data73 $data74 $data75 +print $data80 $data81 $data82 $data83 $data84 $data85 +print $data90 $data91 $data92 $data93 $data94 $data95 +print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] +print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] +print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] +print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] + +if $rows != 14 then + print rows $rows != 14 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 20 then + print data11 $data1 != 20 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 100 then + print data41 $data41 != 100 + return -1 +endi + +if $data51 != 130 then + print data51 $data51 != 130 + return -1 +endi + +if $data61 != 170 then + print data61 $data61 != 170 + return -1 +endi + +if $data71 != 180 then + print data71 $data71 != 180 + return -1 +endi + +if $data81 != 210 then + print data81 $data81 != 210 + return -1 +endi + +if $data91 != 200 then + print data91 $data91 != 200 + return -1 +endi + +if $data[10][1] != 190 then + print data[10][1] $data[10][1] != 190 + return -1 +endi + +if $data[11][1] != 220 then + print data[11][1] $data[11][1] != 220 + return -1 +endi + +if $data[12][1] != 230 then + print data[12][1] $data[12][1] != 230 + return -1 +endi + +if $data[13][1] != 240 then + print data[13][1] $data[13][1] != 240 + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +if $data15 != 20.000000000 then + print data15 $data5 != 20.000000000 + return -1 +endi + +if $data25 != 50.000000000 then + print data25 $data25 != 50.000000000 + return -1 +endi + +if $data35 != 90.000000000 then + print data35 $data35 != 90.000000000 + return -1 +endi + +if $data45 != 100.000000000 then + print data45 $data45 != 100.000000000 + return -1 +endi + +if $data55 != 130.000000000 then + print data55 $data55 != 130.000000000 + return -1 +endi + +if $data65 != 170.000000000 then + print data65 $data65 != 170.000000000 + return -1 +endi + +if $data75 != 180.000000000 then + print data75 $data75 != 180.000000000 + return -1 +endi + +if $data85 != 210.000000000 then + print data85 $data85 != 210.000000000 + return -1 +endi + +if $data95 != 200.000000000 then + print data95 $data95 != 200.000000000 + return -1 +endi + +if $data[10][5] != 190.000000000 then + print data[10][5] $data[10][5] != 190.000000000 + return -1 +endi + +if $data[11][5] != 220.000000000 then + print data[11][5] $data[11][5] != 220.000000000 + return -1 +endi + +if $data[12][5] != 230.000000000 then + print data[12][5] $data[12][5] != 230.000000000 + return -1 +endi + +if $data[13][5] != 240.000000000 then + print data[13][5] $data[13][5] != 240.000000000 + return -1 +endi + + +print =============== step 8 query records of ct4 from memory +sql select * from ct4; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 + + +if $rows != 5 then + print rows $rows != 5 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 30 then + print data11 $data11 != 30 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 80 then + print data41 $data41 != 80 + return -1 +endi + +if $data03 != b0 then + print data03 $data03 != b0 + return -1 +endi + +if $data13 != b2 then + print data13 $data13 != b2 + return -1 +endi + +if $data23 != b5 then + print data23 $data23 != b5 + return -1 +endi + +if $data33 != b9 then + print data33 $data33 != b9 + return -1 +endi + +if $data43 != b8 then + print data43 $data43 != b8 + return -1 +endi + +if $data04 != n0 then + print data04 $data04 != n0 + return -1 +endi + +if $data14 != n2 then + print data14 $data14 != n2 + return -1 +endi + +if $data24 != n5 then + print data24 $data24 != n5 + return -1 +endi + +if $data34 != n9 then + print data34 $data34 != n9 + return -1 +endi + +if $data44 != n8 then + print data44 $data44 != n8 + return -1 +endi + +#==================== reboot to trigger commit data to file +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode1 -s start + + + +print =============== step 9 query records of ct1 from file +sql select * from ct1; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 + +if $rows != 6 then + print rows $rows != 6 + return -1 +endi + +if $data01 != 30 then + print data01 $data01 != 30 + return -1 +endi + +if $data02 != 100.000000000 then + print data02 $data02 != 100.000000000 + return -1 +endi + +if $data03 != bin2 then + print data03 $data03 != bin2 + return -1 +endi + +if $data04 != nchar4 then + print data04 $data04 != nchar4 + return -1 +endi + +if $data05 != 1000.010000000 then + print data05 $data05 != 1000.010000000 + return -1 +endi + +if $data11 != 11 then + print data11 $data11 != 11 + return -1 +endi + +if $data12 != NULL then + print data12 $data12 != NULL + return -1 +endi + +if $data13 != binary then + print data13 $data13 != binary + return -1 +endi + +if $data14 != nchar then + print data14 $data14 != nchar + return -1 +endi + +if $data15 != NULL then + print data15 $data15 != NULL + return -1 +endi + +if $data51 != 22 then + print data51 $data51 != 22 + return -1 +endi + +if $data52 != NULL then + print data52 $data52 != NULL + return -1 +endi + +if $data53 != NULL then + print data53 $data53 != NULL + return -1 +endi + +if $data54 != nchar then + print data54 $data54 != nchar + return -1 +endi + +if $data55 != 220.000000000 then + print data55 $data55 != 220.000000000 + return -1 +endi + + +print =============== step 10 query records of ct2 from file +sql select * from ct2; +print $data00 $data01 $data02 $data03 $data04 $data05 + +if $rows != 1 then + print rows $rows != 1 + return -1 +endi + +if $data01 != 20 then + print data01 $data01 != 20 + return -1 +endi + +if $data02 != 20.000000000 then + print data02 $data02 != 20.000000000 + return -1 +endi + +if $data03 != xkl then + print data03 $data03 != xkl + return -1 +endi + +if $data04 != zxc then + print data04 $data04 != zxc + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +print =============== step 11 query records of ct3 from file +sql select * from ct3; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 +print $data50 $data51 $data52 $data53 $data54 $data55 +print $data60 $data61 $data62 $data63 $data64 $data65 +print $data70 $data71 $data72 $data73 $data74 $data75 +print $data80 $data81 $data82 $data83 $data84 $data85 +print $data90 $data91 $data92 $data93 $data94 $data95 +print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5] +print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5] +print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5] +print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5] + +if $rows != 14 then + print rows $rows != 14 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 20 then + print data11 $data1 != 20 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 100 then + print data41 $data41 != 100 + return -1 +endi + +if $data51 != 130 then + print data51 $data51 != 130 + return -1 +endi + +if $data61 != 170 then + print data61 $data61 != 170 + return -1 +endi + +if $data71 != 180 then + print data71 $data71 != 180 + return -1 +endi + +if $data81 != 210 then + print data81 $data81 != 210 + return -1 +endi + +if $data91 != 200 then + print data91 $data91 != 200 + return -1 +endi + +if $data[10][1] != 190 then + print data[10][1] $data[10][1] != 190 + return -1 +endi + +if $data[11][1] != 220 then + print data[11][1] $data[11][1] != 220 + return -1 +endi + +if $data[12][1] != 230 then + print data[12][1] $data[12][1] != 230 + return -1 +endi + +if $data[13][1] != 240 then + print data[13][1] $data[13][1] != 240 + return -1 +endi + +if $data05 != 10.000000000 then + print data05 $data05 != 10.000000000 + return -1 +endi + +if $data15 != 20.000000000 then + print data15 $data5 != 20.000000000 + return -1 +endi + +if $data25 != 50.000000000 then + print data25 $data25 != 50.000000000 + return -1 +endi + +if $data35 != 90.000000000 then + print data35 $data35 != 90.000000000 + return -1 +endi + +if $data45 != 100.000000000 then + print data45 $data45 != 100.000000000 + return -1 +endi + +if $data55 != 130.000000000 then + print data55 $data55 != 130.000000000 + return -1 +endi + +if $data65 != 170.000000000 then + print data65 $data65 != 170.000000000 + return -1 +endi + +if $data75 != 180.000000000 then + print data75 $data75 != 180.000000000 + return -1 +endi + +if $data85 != 210.000000000 then + print data85 $data85 != 210.000000000 + return -1 +endi + +if $data95 != 200.000000000 then + print data95 $data95 != 200.000000000 + return -1 +endi + +if $data[10][5] != 190.000000000 then + print data[10][5] $data[10][5] != 190.000000000 + return -1 +endi + +if $data[11][5] != 220.000000000 then + print data[11][5] $data[11][5] != 220.000000000 + return -1 +endi + +if $data[12][5] != 230.000000000 then + print data[12][5] $data[12][5] != 230.000000000 + return -1 +endi + +if $data[13][5] != 240.000000000 then + print data[13][5] $data[13][5] != 240.000000000 + return -1 +endi + + +print =============== step 12 query records of ct4 from file +sql select * from ct4; +print $data00 $data01 $data02 $data03 $data04 $data05 +print $data10 $data11 $data12 $data13 $data14 $data15 +print $data20 $data21 $data22 $data23 $data24 $data25 +print $data30 $data31 $data32 $data33 $data34 $data35 +print $data40 $data41 $data42 $data43 $data44 $data45 + + +if $rows != 5 then + print rows $rows != 5 + return -1 +endi + +if $data01 != 10 then + print data01 $data01 != 10 + return -1 +endi + +if $data11 != 30 then + print data11 $data11 != 30 + return -1 +endi + +if $data21 != 50 then + print data21 $data21 != 50 + return -1 +endi + +if $data31 != 90 then + print data31 $data31 != 90 + return -1 +endi + +if $data41 != 80 then + print data41 $data41 != 80 + return -1 +endi + +if $data03 != b0 then + print data03 $data03 != b0 + return -1 +endi + +if $data13 != b2 then + print data13 $data13 != b2 + return -1 +endi + +if $data23 != b5 then + print data23 $data23 != b5 + return -1 +endi + +if $data33 != b9 then + print data33 $data33 != b9 + return -1 +endi + +if $data43 != b8 then + print data43 $data43 != b8 + return -1 +endi + +if $data04 != n0 then + print data04 $data04 != n0 + return -1 +endi + +if $data14 != n2 then + print data14 $data14 != n2 + return -1 +endi + +if $data24 != n5 then + print data24 $data24 != n5 + return -1 +endi + +if $data34 != n9 then + print data34 $data34 != n9 + return -1 +endi + +if $data44 != n8 then + print data44 $data44 != n8 + return -1 +endi \ No newline at end of file diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index a6ee5951a0..d4e575801c 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -391,13 +391,13 @@ if $data02 != 4 then return -1 endi -if $data03 != 14 then - print ======$data03 +if $data03 != 50 then + print ======$data03 != 50 return -1 endi -if $data04 != 4 then - print ======$data04 +if $data04 != 20 then + print ======$data04 != 20 return -1 endi @@ -421,13 +421,13 @@ if $data12 != 4 then return -1 endi -if $data13 != 10 then - print ======$data13 +if $data13 != 46 then + print ======$data13 != 46 return -1 endi -if $data14 != 3 then - print ======$data14 +if $data14 != 20 then + print ======$data14 != 20 return -1 endi