xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #15260 from taosdata/feature/TD-11274-3.0 

fix: merge dup rows in client
This commit is contained in:
Cary Xu 2022-07-21 19:21:26 +08:00 committed by GitHub
parent 3a48066a76 87c6f5805e
commit f2a3f1591b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1087 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/common/tmsg.h
View File

@ -438,7 +438,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapperEx(SDecoder* pDecoder, SSchemaW
return 0; return 0;
} }
STSchema* tdGetSTSChemaFromSSChema(SSchema** pSchema, int32_t nCols); STSchema* tdGetSTSChemaFromSSChema(SSchema* pSchema, int32_t nCols, int32_t sver);
typedef struct { typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; char name[TSDB_TABLE_FNAME_LEN];

6
source/common/src/tmsg.c
View File

@ -4941,14 +4941,14 @@ int tDecodeSVCreateStbReq(SDecoder *pCoder, SVCreateStbReq *pReq) {
return 0; return 0;
} }
STSchema *tdGetSTSChemaFromSSChema(SSchema **pSchema, int32_t nCols) { STSchema *tdGetSTSChemaFromSSChema(SSchema *pSchema, int32_t nCols, int32_t sver) {
STSchemaBuilder schemaBuilder = {0}; STSchemaBuilder schemaBuilder = {0};
if (tdInitTSchemaBuilder(&schemaBuilder, 1) < 0) { if (tdInitTSchemaBuilder(&schemaBuilder, sver) < 0) {
return NULL; return NULL;
} }
for (int i = 0; i < nCols; i++) { for (int i = 0; i < nCols; i++) {
SSchema *schema = *pSchema + i; SSchema *schema = pSchema + i;
if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) { if (tdAddColToSchema(&schemaBuilder, schema->type, schema->flags, schema->colId, schema->bytes) < 0) {
tdDestroyTSchemaBuilder(&schemaBuilder); tdDestroyTSchemaBuilder(&schemaBuilder);
return NULL; return NULL;

9
source/common/src/trow.c
View File

@ -568,6 +568,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
int32_t maxVarDataLen = 0; int32_t maxVarDataLen = 0;
int32_t iColVal = 0; int32_t iColVal = 0;
void *varBuf = NULL; void *varBuf = NULL;
bool isAlloc = false;
ASSERT(nColVal > 1); ASSERT(nColVal > 1);
@ -610,8 +611,11 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
++iColVal; ++iColVal;
} }
if (!(*ppRow)) {
*ppRow = (STSRow *)taosMemoryCalloc( *ppRow = (STSRow *)taosMemoryCalloc(
1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1)); 1, sizeof(STSRow) + pTSchema->flen + varDataLen + TD_BITMAP_BYTES(pTSchema->numOfCols - 1));
isAlloc = true;
}
if (!(*ppRow)) { if (!(*ppRow)) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -621,7 +625,9 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
if (maxVarDataLen > 0) { if (maxVarDataLen > 0) {
varBuf = taosMemoryMalloc(maxVarDataLen); varBuf = taosMemoryMalloc(maxVarDataLen);
if (!varBuf) { if (!varBuf) {
if(isAlloc) {
taosMemoryFreeClear(*ppRow); taosMemoryFreeClear(*ppRow);
}
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1; return -1;
} }
@ -1323,12 +1329,11 @@ void tTSRowGetVal(STSRow *pRow, STSchema *pTSchema, int16_t iCol, SColVal *pColV
SCellVal cv; SCellVal cv;
SValue value; SValue value;
ASSERT(iCol > 0); ASSERT((pTColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) || (iCol > 0));
if (TD_IS_TP_ROW(pRow)) { if (TD_IS_TP_ROW(pRow)) {
tdSTpRowGetVal(pRow, pTColumn->colId, pTColumn->type, pTSchema->flen, pTColumn->offset, iCol - 1, &cv); tdSTpRowGetVal(pRow, pTColumn->colId, pTColumn->type, pTSchema->flen, pTColumn->offset, iCol - 1, &cv);
} else if (TD_IS_KV_ROW(pRow)) { } else if (TD_IS_KV_ROW(pRow)) {
ASSERT(iCol > 0);
tdSKvRowGetVal(pRow, pTColumn->colId, iCol - 1, &cv); tdSKvRowGetVal(pRow, pTColumn->colId, iCol - 1, &cv);
} else { } else {
ASSERT(0); ASSERT(0);

2
source/common/test/dataformatTest.cpp
View File

@ -116,7 +116,7 @@ STSchema *genSTSchema(int16_t nCols) {
} }
STSchema *pResult = NULL; STSchema *pResult = NULL;
pResult = tdGetSTSChemaFromSSChema(&pSchema, nCols); pResult = tdGetSTSChemaFromSSChema(pSchema, nCols, 1);
taosMemoryFree(pSchema); taosMemoryFree(pSchema);
return pResult; return pResult;

2
source/dnode/vnode/src/tq/tq.c
View File

@ -634,7 +634,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) {
ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); ASSERT(pTask->tbSink.pSchemaWrapper->pSchema);
pTask->tbSink.pTSchema = pTask->tbSink.pTSchema =
tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols); tdGetSTSChemaFromSSChema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, 1);
ASSERT(pTask->tbSink.pTSchema); ASSERT(pTask->tbSink.pTSchema);
} }

6
source/libs/parser/src/parInsert.c
View File

@ -1200,7 +1200,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
*gotRow = true; *gotRow = true;
#ifdef TD_DEBUG_PRINT_ROW #ifdef TD_DEBUG_PRINT_ROW
STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&schema, spd->numOfCols); STSchema* pSTSchema = tdGetSTSChemaFromSSChema(schema, spd->numOfCols, 1);
tdSRowPrint(row, pSTSchema, __func__); tdSRowPrint(row, pSTSchema, __func__);
taosMemoryFree(pSTSchema); taosMemoryFree(pSTSchema);
#endif #endif
@ -1972,7 +1972,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in
} }
} }
#ifdef TD_DEBUG_PRINT_ROW #ifdef TD_DEBUG_PRINT_ROW
STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1);
tdSRowPrint(row, pSTSchema, __func__); tdSRowPrint(row, pSTSchema, __func__);
taosMemoryFree(pSTSchema); taosMemoryFree(pSTSchema);
#endif #endif
@ -2057,7 +2057,7 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu
#ifdef TD_DEBUG_PRINT_ROW #ifdef TD_DEBUG_PRINT_ROW
if (rowEnd) { if (rowEnd) {
STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); STSchema* pSTSchema = tdGetSTSChemaFromSSChema(pSchema, spd->numOfCols, 1);
tdSRowPrint(row, pSTSchema, __func__); tdSRowPrint(row, pSTSchema, __func__);
taosMemoryFree(pSTSchema); taosMemoryFree(pSTSchema);
} }

232
source/libs/parser/src/parInsertData.c
View File

@ -19,6 +19,7 @@
#include "parInt.h" #include "parInt.h"
#include "parUtil.h" #include "parUtil.h"
#include "querynodes.h" #include "querynodes.h"
#include "tRealloc.h"
#define IS_RAW_PAYLOAD(t) \ #define IS_RAW_PAYLOAD(t) \
(((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert (((int)(t)) == PAYLOAD_TYPE_RAW) // 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert
@ -34,6 +35,32 @@ typedef struct SBlockKeyInfo {
SBlockKeyTuple* pKeyTuple; SBlockKeyTuple* pKeyTuple;
} SBlockKeyInfo; } SBlockKeyInfo;
typedef struct {
int32_t index;
SArray* rowArray; // array of merged rows(mem allocated by tRealloc/free by tFree)
STSchema* pSchema;
int64_t tbUid; // suid for child table, uid for normal table
} SBlockRowMerger;
static FORCE_INLINE void tdResetSBlockRowMerger(SBlockRowMerger* pMerger) {
if (pMerger) {
pMerger->index = -1;
}
}
static void tdFreeSBlockRowMerger(SBlockRowMerger* pMerger) {
if (pMerger) {
int32_t size = taosArrayGetSize(pMerger->rowArray);
for (int32_t i = 0; i < size; ++i) {
tFree(*(void**)taosArrayGet(pMerger->rowArray, i));
}
taosArrayDestroy(pMerger->rowArray);
taosMemoryFreeClear(pMerger->pSchema);
taosMemoryFree(pMerger);
}
}
static int32_t rowDataCompar(const void* lhs, const void* rhs) { static int32_t rowDataCompar(const void* lhs, const void* rhs) {
TSKEY left = *(TSKEY*)lhs; TSKEY left = *(TSKEY*)lhs;
TSKEY right = *(TSKEY*)rhs; TSKEY right = *(TSKEY*)rhs;
@ -328,7 +355,7 @@ void sortRemoveDataBlockDupRowsRaw(STableDataBlocks* dataBuf) {
} }
// data block is disordered, sort it in ascending order // data block is disordered, sort it in ascending order
int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) { static int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo) {
SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData; SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData;
int16_t nRows = pBlocks->numOfRows; int16_t nRows = pBlocks->numOfRows;
@ -396,6 +423,201 @@ int sortRemoveDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKey
return 0; return 0;
} }
static void* tdGetCurRowFromBlockMerger(SBlockRowMerger* pBlkRowMerger) {
if (pBlkRowMerger && (pBlkRowMerger->index >= 0)) {
ASSERT(pBlkRowMerger->index < taosArrayGetSize(pBlkRowMerger->rowArray));
return *(void**)taosArrayGet(pBlkRowMerger->rowArray, pBlkRowMerger->index);
}
return NULL;
}
static int32_t tdBlockRowMerge(STableMeta* pTableMeta, SBlockKeyTuple* pEndKeyTp, int32_t nDupRows,
SBlockRowMerger** pBlkRowMerger, int32_t rowSize) {
ASSERT(nDupRows > 1);
SBlockKeyTuple* pStartKeyTp = pEndKeyTp - (nDupRows - 1);
ASSERT(pStartKeyTp->skey == pEndKeyTp->skey);
// TODO: optimization if end row is all normal
#if 0
STSRow* pEndRow = (STSRow*)pEndKeyTp->payloadAddr;
if(isNormal(pEndRow)) { // set the end row if it is normal and return directly
pStartKeyTp->payloadAddr = pEndKeyTp->payloadAddr;
return TSDB_CODE_SUCCESS;
}
#endif
if (!(*pBlkRowMerger)) {
(*pBlkRowMerger) = taosMemoryCalloc(1, sizeof(**pBlkRowMerger));
if (!(*pBlkRowMerger)) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
(*pBlkRowMerger)->index = -1;
if (!(*pBlkRowMerger)->rowArray) {
(*pBlkRowMerger)->rowArray = taosArrayInit(1, sizeof(void*));
if (!(*pBlkRowMerger)->rowArray) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
}
}
if ((*pBlkRowMerger)->pSchema) {
if ((*pBlkRowMerger)->pSchema->version != pTableMeta->sversion) {
taosMemoryFreeClear((*pBlkRowMerger)->pSchema);
} else {
if ((*pBlkRowMerger)->tbUid != (pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid)) {
taosMemoryFreeClear((*pBlkRowMerger)->pSchema);
}
}
}
if (!(*pBlkRowMerger)->pSchema) {
(*pBlkRowMerger)->pSchema =
tdGetSTSChemaFromSSChema(pTableMeta->schema, pTableMeta->tableInfo.numOfColumns, pTableMeta->sversion);
if (!(*pBlkRowMerger)->pSchema) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_FAILED;
}
(*pBlkRowMerger)->tbUid = pTableMeta->suid > 0 ? pTableMeta->suid : pTableMeta->uid;
}
void* pDestRow = NULL;
++((*pBlkRowMerger)->index);
if ((*pBlkRowMerger)->index < taosArrayGetSize((*pBlkRowMerger)->rowArray)) {
void* pAlloc = *(void**)taosArrayGet((*pBlkRowMerger)->rowArray, (*pBlkRowMerger)->index);
if (tRealloc((uint8_t**)&pAlloc, rowSize) != 0) {
return TSDB_CODE_FAILED;
}
pDestRow = pAlloc;
} else {
if (tRealloc((uint8_t**)&pDestRow, rowSize) != 0) {
return TSDB_CODE_FAILED;
}
taosArrayPush((*pBlkRowMerger)->rowArray, &pDestRow);
}
// merge rows to pDestRow
STSchema* pSchema = (*pBlkRowMerger)->pSchema;
SArray* pArray = taosArrayInit(pSchema->numOfCols, sizeof(SColVal));
for (int32_t i = 0; i < pSchema->numOfCols; ++i) {
SColVal colVal = {0};
for (int32_t j = 0; j < nDupRows; ++j) {
tTSRowGetVal((pEndKeyTp - j)->payloadAddr, pSchema, i, &colVal);
if (!colVal.isNone) {
break;
}
}
taosArrayPush(pArray, &colVal);
}
if (tdSTSRowNew(pArray, pSchema, (STSRow**)&pDestRow) < 0) {
taosArrayDestroy(pArray);
return TSDB_CODE_FAILED;
}
taosArrayDestroy(pArray);
return TSDB_CODE_SUCCESS;
}
// data block is disordered, sort it in ascending order, and merge dup rows if exists
static int sortMergeDataBlockDupRows(STableDataBlocks* dataBuf, SBlockKeyInfo* pBlkKeyInfo,
SBlockRowMerger** ppBlkRowMerger) {
SSubmitBlk* pBlocks = (SSubmitBlk*)dataBuf->pData;
STableMeta* pTableMeta = dataBuf->pTableMeta;
int16_t nRows = pBlocks->numOfRows;
// size is less than the total size, since duplicated rows may be removed.
// allocate memory
size_t nAlloc = nRows * sizeof(SBlockKeyTuple);
if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) {
char* tmp = taosMemoryRealloc(pBlkKeyInfo->pKeyTuple, nAlloc);
if (tmp == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple*)tmp;
pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc;
}
memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc);
tdResetSBlockRowMerger(*ppBlkRowMerger);
int32_t extendedRowSize = getExtendedRowSize(dataBuf);
SBlockKeyTuple* pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
char* pBlockData = pBlocks->data + pBlocks->schemaLen;
int n = 0;
while (n < nRows) {
pBlkKeyTuple->skey = TD_ROW_KEY((STSRow*)pBlockData);
pBlkKeyTuple->payloadAddr = pBlockData;
pBlkKeyTuple->index = n;
// next loop
pBlockData += extendedRowSize;
++pBlkKeyTuple;
++n;
}
if (!dataBuf->ordered) {
pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
taosSort(pBlkKeyTuple, nRows, sizeof(SBlockKeyTuple), rowDataComparStable);
pBlkKeyTuple = pBlkKeyInfo->pKeyTuple;
bool hasDup = false;
int32_t nextPos = 0;
int32_t i = 0;
int32_t j = 1;
while (j < nRows) {
TSKEY ti = (pBlkKeyTuple + i)->skey;
TSKEY tj = (pBlkKeyTuple + j)->skey;
if (ti == tj) {
++j;
continue;
}
if ((j - i) > 1) {
if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) {
return TSDB_CODE_FAILED;
}
(pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger);
if (!hasDup) {
hasDup = true;
}
i = j;
} else {
if (hasDup) {
memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple));
}
++i;
}
++nextPos;
++j;
}
if ((j - i) > 1) {
ASSERT((pBlkKeyTuple + i)->skey == (pBlkKeyTuple + j - 1)->skey);
if (tdBlockRowMerge(pTableMeta, (pBlkKeyTuple + j - 1), j - i, ppBlkRowMerger, extendedRowSize) < 0) {
return TSDB_CODE_FAILED;
}
(pBlkKeyTuple + nextPos)->payloadAddr = tdGetCurRowFromBlockMerger(*ppBlkRowMerger);
} else if (hasDup) {
memmove(pBlkKeyTuple + nextPos, pBlkKeyTuple + i, sizeof(SBlockKeyTuple));
}
dataBuf->ordered = true;
pBlocks->numOfRows = nextPos + 1;
}
dataBuf->size = sizeof(SSubmitBlk) + pBlocks->numOfRows * extendedRowSize;
dataBuf->prevTS = INT64_MIN;
return TSDB_CODE_SUCCESS;
}
// Erase the empty space reserved for binary data // Erase the empty space reserved for binary data
static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple, static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SBlockKeyTuple* blkKeyTuple,
bool isRawPayload) { bool isRawPayload) {
@ -464,6 +686,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
STableDataBlocks** p = taosHashIterate(pHashObj, NULL); STableDataBlocks** p = taosHashIterate(pHashObj, NULL);
STableDataBlocks* pOneTableBlock = *p; STableDataBlocks* pOneTableBlock = *p;
SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock SBlockKeyInfo blkKeyInfo = {0}; // share by pOneTableBlock
SBlockRowMerger *pBlkRowMerger = NULL;
while (pOneTableBlock) { while (pOneTableBlock) {
SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData; SSubmitBlk* pBlocks = (SSubmitBlk*)pOneTableBlock->pData;
if (pBlocks->numOfRows > 0) { if (pBlocks->numOfRows > 0) {
@ -473,6 +697,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0, getDataBlockFromList(pVnodeDataBlockHashList, &pOneTableBlock->vgId, sizeof(pOneTableBlock->vgId), TSDB_PAYLOAD_SIZE, INSERT_HEAD_SIZE, 0,
pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL); pOneTableBlock->pTableMeta, &dataBuf, pVnodeDataBlockList, NULL);
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
tdFreeSBlockRowMerger(pBlkRowMerger);
taosHashCleanup(pVnodeDataBlockHashList); taosHashCleanup(pVnodeDataBlockHashList);
destroyBlockArrayList(pVnodeDataBlockList); destroyBlockArrayList(pVnodeDataBlockList);
taosMemoryFreeClear(blkKeyInfo.pKeyTuple); taosMemoryFreeClear(blkKeyInfo.pKeyTuple);
@ -490,6 +715,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
if (tmp != NULL) { if (tmp != NULL) {
dataBuf->pData = tmp; dataBuf->pData = tmp;
} else { // failed to allocate memory, free already allocated memory and return error code } else { // failed to allocate memory, free already allocated memory and return error code
tdFreeSBlockRowMerger(pBlkRowMerger);
taosHashCleanup(pVnodeDataBlockHashList); taosHashCleanup(pVnodeDataBlockHashList);
destroyBlockArrayList(pVnodeDataBlockList); destroyBlockArrayList(pVnodeDataBlockList);
taosMemoryFreeClear(dataBuf->pData); taosMemoryFreeClear(dataBuf->pData);
@ -501,7 +727,8 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
if (isRawPayload) { if (isRawPayload) {
sortRemoveDataBlockDupRowsRaw(pOneTableBlock); sortRemoveDataBlockDupRowsRaw(pOneTableBlock);
} else { } else {
if ((code = sortRemoveDataBlockDupRows(pOneTableBlock, &blkKeyInfo)) != 0) { if ((code = sortMergeDataBlockDupRows(pOneTableBlock, &blkKeyInfo, &pBlkRowMerger)) != 0) {
tdFreeSBlockRowMerger(pBlkRowMerger);
taosHashCleanup(pVnodeDataBlockHashList); taosHashCleanup(pVnodeDataBlockHashList);
destroyBlockArrayList(pVnodeDataBlockList); destroyBlockArrayList(pVnodeDataBlockList);
taosMemoryFreeClear(dataBuf->pData); taosMemoryFreeClear(dataBuf->pData);
@ -529,6 +756,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
} }
// free the table data blocks; // free the table data blocks;
tdFreeSBlockRowMerger(pBlkRowMerger);
taosHashCleanup(pVnodeDataBlockHashList); taosHashCleanup(pVnodeDataBlockHashList);
taosMemoryFreeClear(blkKeyInfo.pKeyTuple); taosMemoryFreeClear(blkKeyInfo.pKeyTuple);
*pVgDataBlocks = pVnodeDataBlockList; *pVgDataBlocks = pVnodeDataBlockList;

24
tests/script/tsim/insert/update0.sim
View File

@ -79,8 +79,8 @@ if $rows != 3 then
return -1 return -1
endi endi
if $data01 != 103 then if $data01 != 303 then
print data01 $data01 != 103 print data01 $data01 != 303
return -1 return -1
endi endi
@ -89,8 +89,8 @@ if $data11 != 80 then
return -1 return -1
endi endi
if $data21 != 40 then if $data21 != 60 then
print data21 $data21 != 40 print data21 $data21 != 60
return -1 return -1
endi endi
@ -138,8 +138,8 @@ if $rows != 3 then
return -1 return -1
endi endi
if $data01 != 103 then if $data01 != 303 then
print data01 $data01 != 103 print data01 $data01 != 303
return -1 return -1
endi endi
@ -148,8 +148,8 @@ if $data11 != 80 then
return -1 return -1
endi endi
if $data21 != 40 then if $data21 != 60 then
print data21 $data21 != 40 print data21 $data21 != 60
return -1 return -1
endi endi
@ -208,8 +208,8 @@ if $data01 != 10 then
return -1 return -1
endi endi
if $data11 != 103 then if $data11 != 303 then
print data11 $data11 != 103 print data11 $data11 != 303
return -1 return -1
endi endi
@ -218,8 +218,8 @@ if $data21 != NULL then
return -1 return -1
endi endi
if $data31 != 40 then if $data31 != 60 then
print data31 $data31 != 40 print data31 $data31 != 60
return -1 return -1
endi endi

818
tests/script/tsim/insert/update1_sort_merge.sim Normal file
View File

@ -0,0 +1,818 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
print =============== create database
sql drop database if exists d0
sql create database d0 keep 365000d,365000d,365000d
sql use d0
print =============== create super table
sql create table if not exists stb (ts timestamp, c1 int unsigned, c2 double, c3 binary(10), c4 nchar(10), c5 double) tags (city binary(20),district binary(20));
sql show stables
if $rows != 1 then
return -1
endi
print =============== create child table
sql create table ct1 using stb tags("BeiJing", "ChaoYang")
sql create table ct2 using stb tags("BeiJing", "HaiDian")
sql create table ct3 using stb tags("BeiJing", "PingGu")
sql create table ct4 using stb tags("BeiJing", "YanQing")
sql show tables
if $rows != 4 then
print rows $rows != 4
return -1
endi
print =============== step 1 insert records into ct1 - taosd merge
sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10, 20);
sql insert into ct1(ts,c1,c2,c3,c4) values('2022-05-03 16:59:00.011', 11, NULL, 'binary', 'nchar');
sql insert into ct1 values('2022-05-03 16:59:00.016', 16, NULL, NULL, 'nchar', NULL);
sql insert into ct1 values('2022-05-03 16:59:00.016', 17, NULL, NULL, 'nchar', 170);
sql insert into ct1 values('2022-05-03 16:59:00.020', 20, NULL, NULL, 'nchar', 200);
sql insert into ct1 values('2022-05-03 16:59:00.016', 18, NULL, NULL, 'nchar', 180);
sql insert into ct1 values('2022-05-03 16:59:00.021', 21, NULL, NULL, 'nchar', 210);
sql insert into ct1 values('2022-05-03 16:59:00.022', 22, NULL, NULL, 'nchar', 220);
print =============== step 2 insert records into ct1/ct2 - taosc merge for 2022-05-03 16:59:00.010
sql insert into ct1(ts,c1,c2) values('2022-05-03 16:59:00.010', 10,10), ('2022-05-03 16:59:00.010',20,10.0), ('2022-05-03 16:59:00.010',30,NULL) ct2(ts,c1) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',20) ct1(ts,c2) values('2022-05-03 16:59:00.010',10), ('2022-05-03 16:59:00.010',100) ct1(ts,c3) values('2022-05-03 16:59:00.010','bin1'), ('2022-05-03 16:59:00.010','bin2') ct1(ts,c4,c5) values('2022-05-03 16:59:00.010',NULL,NULL), ('2022-05-03 16:59:00.010','nchar4',1000.01) ct2(ts,c2,c3,c4,c5) values('2022-05-03 16:59:00.010',20,'xkl','zxc',10);
print =============== step 3 insert records into ct3
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.020', 10,10);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.021', 10,10), ('2022-05-03 16:59:00.021',20,20.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.022', 30,30), ('2022-05-03 16:59:00.022',40,40.0),('2022-05-03 16:59:00.022',50,50.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.023', 60,60), ('2022-05-03 16:59:00.023',70,70.0),('2022-05-03 16:59:00.023',80,80.0), ('2022-05-03 16:59:00.023',90,90.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.024', 100,100), ('2022-05-03 16:59:00.025',110,110.0),('2022-05-03 16:59:00.025',120,120.0), ('2022-05-03 16:59:00.025',130,130.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.030', 140,140), ('2022-05-03 16:59:00.030',150,150.0),('2022-05-03 16:59:00.031',160,160.0), ('2022-05-03 16:59:00.030',170,170.0), ('2022-05-03 16:59:00.031',180,180.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.042', 190,190), ('2022-05-03 16:59:00.041',200,200.0),('2022-05-03 16:59:00.040',210,210.0);
sql insert into ct3(ts,c1,c5) values('2022-05-03 16:59:00.050', 220,220), ('2022-05-03 16:59:00.051',230,230.0),('2022-05-03 16:59:00.052',240,240.0);
print =============== step 4 insert records into ct4
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.020', 10,'b0','n0');
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.021', 20,'b1','n1'), ('2022-05-03 16:59:00.021',30,'b2','n2');
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.022', 40,'b3','n3'), ('2022-05-03 16:59:00.022',40,'b4','n4'),('2022-05-03 16:59:00.022',50,'b5','n5');
sql insert into ct4(ts,c1,c3,c4) values('2022-05-03 16:59:00.023', 60,'b6','n6'), ('2022-05-03 16:59:00.024',70,'b7','n7'),('2022-05-03 16:59:00.024',80,'b8','n8'), ('2022-05-03 16:59:00.023',90,'b9','n9');
print =============== step 5 query records of ct1 from memory(taosc and taosd merge)
sql select * from ct1;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
if $rows != 6 then
print rows $rows != 6
return -1
endi
if $data01 != 30 then
print data01 $data01 != 30
return -1
endi
if $data02 != 100.000000000 then
print data02 $data02 != 100.000000000
return -1
endi
if $data03 != bin2 then
print data03 $data03 != bin2
return -1
endi
if $data04 != nchar4 then
print data04 $data04 != nchar4
return -1
endi
if $data05 != 1000.010000000 then
print data05 $data05 != 1000.010000000
return -1
endi
if $data11 != 11 then
print data11 $data11 != 11
return -1
endi
if $data12 != NULL then
print data12 $data12 != NULL
return -1
endi
if $data13 != binary then
print data13 $data13 != binary
return -1
endi
if $data14 != nchar then
print data14 $data14 != nchar
return -1
endi
if $data15 != NULL then
print data15 $data15 != NULL
return -1
endi
if $data51 != 22 then
print data51 $data51 != 22
return -1
endi
if $data52 != NULL then
print data52 $data52 != NULL
return -1
endi
if $data53 != NULL then
print data53 $data53 != NULL
return -1
endi
if $data54 != nchar then
print data54 $data54 != nchar
return -1
endi
if $data55 != 220.000000000 then
print data55 $data55 != 220.000000000
return -1
endi
print =============== step 6 query records of ct2 from memory(taosc and taosd merge)
sql select * from ct2;
print $data00 $data01 $data02 $data03 $data04 $data05
if $rows != 1 then
print rows $rows != 1
return -1
endi
if $data01 != 20 then
print data01 $data01 != 20
return -1
endi
if $data02 != 20.000000000 then
print data02 $data02 != 20.000000000
return -1
endi
if $data03 != xkl then
print data03 $data03 != xkl
return -1
endi
if $data04 != zxc then
print data04 $data04 != zxc
return -1
endi
if $data05 != 10.000000000 then
print data05 $data05 != 10.000000000
return -1
endi
print =============== step 7 query records of ct3 from memory
sql select * from ct3;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
print $data60 $data61 $data62 $data63 $data64 $data65
print $data70 $data71 $data72 $data73 $data74 $data75
print $data80 $data81 $data82 $data83 $data84 $data85
print $data90 $data91 $data92 $data93 $data94 $data95
print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5]
print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5]
print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5]
print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5]
if $rows != 14 then
print rows $rows != 14
return -1
endi
if $data01 != 10 then
print data01 $data01 != 10
return -1
endi
if $data11 != 20 then
print data11 $data1 != 20
return -1
endi
if $data21 != 50 then
print data21 $data21 != 50
return -1
endi
if $data31 != 90 then
print data31 $data31 != 90
return -1
endi
if $data41 != 100 then
print data41 $data41 != 100
return -1
endi
if $data51 != 130 then
print data51 $data51 != 130
return -1
endi
if $data61 != 170 then
print data61 $data61 != 170
return -1
endi
if $data71 != 180 then
print data71 $data71 != 180
return -1
endi
if $data81 != 210 then
print data81 $data81 != 210
return -1
endi
if $data91 != 200 then
print data91 $data91 != 200
return -1
endi
if $data[10][1] != 190 then
print data[10][1] $data[10][1] != 190
return -1
endi
if $data[11][1] != 220 then
print data[11][1] $data[11][1] != 220
return -1
endi
if $data[12][1] != 230 then
print data[12][1] $data[12][1] != 230
return -1
endi
if $data[13][1] != 240 then
print data[13][1] $data[13][1] != 240
return -1
endi
if $data05 != 10.000000000 then
print data05 $data05 != 10.000000000
return -1
endi
if $data15 != 20.000000000 then
print data15 $data5 != 20.000000000
return -1
endi
if $data25 != 50.000000000 then
print data25 $data25 != 50.000000000
return -1
endi
if $data35 != 90.000000000 then
print data35 $data35 != 90.000000000
return -1
endi
if $data45 != 100.000000000 then
print data45 $data45 != 100.000000000
return -1
endi
if $data55 != 130.000000000 then
print data55 $data55 != 130.000000000
return -1
endi
if $data65 != 170.000000000 then
print data65 $data65 != 170.000000000
return -1
endi
if $data75 != 180.000000000 then
print data75 $data75 != 180.000000000
return -1
endi
if $data85 != 210.000000000 then
print data85 $data85 != 210.000000000
return -1
endi
if $data95 != 200.000000000 then
print data95 $data95 != 200.000000000
return -1
endi
if $data[10][5] != 190.000000000 then
print data[10][5] $data[10][5] != 190.000000000
return -1
endi
if $data[11][5] != 220.000000000 then
print data[11][5] $data[11][5] != 220.000000000
return -1
endi
if $data[12][5] != 230.000000000 then
print data[12][5] $data[12][5] != 230.000000000
return -1
endi
if $data[13][5] != 240.000000000 then
print data[13][5] $data[13][5] != 240.000000000
return -1
endi
print =============== step 8 query records of ct4 from memory
sql select * from ct4;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 5 then
print rows $rows != 5
return -1
endi
if $data01 != 10 then
print data01 $data01 != 10
return -1
endi
if $data11 != 30 then
print data11 $data11 != 30
return -1
endi
if $data21 != 50 then
print data21 $data21 != 50
return -1
endi
if $data31 != 90 then
print data31 $data31 != 90
return -1
endi
if $data41 != 80 then
print data41 $data41 != 80
return -1
endi
if $data03 != b0 then
print data03 $data03 != b0
return -1
endi
if $data13 != b2 then
print data13 $data13 != b2
return -1
endi
if $data23 != b5 then
print data23 $data23 != b5
return -1
endi
if $data33 != b9 then
print data33 $data33 != b9
return -1
endi
if $data43 != b8 then
print data43 $data43 != b8
return -1
endi
if $data04 != n0 then
print data04 $data04 != n0
return -1
endi
if $data14 != n2 then
print data14 $data14 != n2
return -1
endi
if $data24 != n5 then
print data24 $data24 != n5
return -1
endi
if $data34 != n9 then
print data34 $data34 != n9
return -1
endi
if $data44 != n8 then
print data44 $data44 != n8
return -1
endi
#==================== reboot to trigger commit data to file
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode1 -s start
print =============== step 9 query records of ct1 from file
sql select * from ct1;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
if $rows != 6 then
print rows $rows != 6
return -1
endi
if $data01 != 30 then
print data01 $data01 != 30
return -1
endi
if $data02 != 100.000000000 then
print data02 $data02 != 100.000000000
return -1
endi
if $data03 != bin2 then
print data03 $data03 != bin2
return -1
endi
if $data04 != nchar4 then
print data04 $data04 != nchar4
return -1
endi
if $data05 != 1000.010000000 then
print data05 $data05 != 1000.010000000
return -1
endi
if $data11 != 11 then
print data11 $data11 != 11
return -1
endi
if $data12 != NULL then
print data12 $data12 != NULL
return -1
endi
if $data13 != binary then
print data13 $data13 != binary
return -1
endi
if $data14 != nchar then
print data14 $data14 != nchar
return -1
endi
if $data15 != NULL then
print data15 $data15 != NULL
return -1
endi
if $data51 != 22 then
print data51 $data51 != 22
return -1
endi
if $data52 != NULL then
print data52 $data52 != NULL
return -1
endi
if $data53 != NULL then
print data53 $data53 != NULL
return -1
endi
if $data54 != nchar then
print data54 $data54 != nchar
return -1
endi
if $data55 != 220.000000000 then
print data55 $data55 != 220.000000000
return -1
endi
print =============== step 10 query records of ct2 from file
sql select * from ct2;
print $data00 $data01 $data02 $data03 $data04 $data05
if $rows != 1 then
print rows $rows != 1
return -1
endi
if $data01 != 20 then
print data01 $data01 != 20
return -1
endi
if $data02 != 20.000000000 then
print data02 $data02 != 20.000000000
return -1
endi
if $data03 != xkl then
print data03 $data03 != xkl
return -1
endi
if $data04 != zxc then
print data04 $data04 != zxc
return -1
endi
if $data05 != 10.000000000 then
print data05 $data05 != 10.000000000
return -1
endi
print =============== step 11 query records of ct3 from file
sql select * from ct3;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
print $data50 $data51 $data52 $data53 $data54 $data55
print $data60 $data61 $data62 $data63 $data64 $data65
print $data70 $data71 $data72 $data73 $data74 $data75
print $data80 $data81 $data82 $data83 $data84 $data85
print $data90 $data91 $data92 $data93 $data94 $data95
print $data[10][0] $data[10][1] $data[10][2] $data[10][3] $data[10][4] $data[10][5]
print $data[11][0] $data[11][1] $data[11][2] $data[11][3] $data[11][4] $data[11][5]
print $data[12][0] $data[12][1] $data[12][2] $data[12][3] $data[12][4] $data[12][5]
print $data[13][0] $data[13][1] $data[13][2] $data[13][3] $data[13][4] $data[13][5]
if $rows != 14 then
print rows $rows != 14
return -1
endi
if $data01 != 10 then
print data01 $data01 != 10
return -1
endi
if $data11 != 20 then
print data11 $data1 != 20
return -1
endi
if $data21 != 50 then
print data21 $data21 != 50
return -1
endi
if $data31 != 90 then
print data31 $data31 != 90
return -1
endi
if $data41 != 100 then
print data41 $data41 != 100
return -1
endi
if $data51 != 130 then
print data51 $data51 != 130
return -1
endi
if $data61 != 170 then
print data61 $data61 != 170
return -1
endi
if $data71 != 180 then
print data71 $data71 != 180
return -1
endi
if $data81 != 210 then
print data81 $data81 != 210
return -1
endi
if $data91 != 200 then
print data91 $data91 != 200
return -1
endi
if $data[10][1] != 190 then
print data[10][1] $data[10][1] != 190
return -1
endi
if $data[11][1] != 220 then
print data[11][1] $data[11][1] != 220
return -1
endi
if $data[12][1] != 230 then
print data[12][1] $data[12][1] != 230
return -1
endi
if $data[13][1] != 240 then
print data[13][1] $data[13][1] != 240
return -1
endi
if $data05 != 10.000000000 then
print data05 $data05 != 10.000000000
return -1
endi
if $data15 != 20.000000000 then
print data15 $data5 != 20.000000000
return -1
endi
if $data25 != 50.000000000 then
print data25 $data25 != 50.000000000
return -1
endi
if $data35 != 90.000000000 then
print data35 $data35 != 90.000000000
return -1
endi
if $data45 != 100.000000000 then
print data45 $data45 != 100.000000000
return -1
endi
if $data55 != 130.000000000 then
print data55 $data55 != 130.000000000
return -1
endi
if $data65 != 170.000000000 then
print data65 $data65 != 170.000000000
return -1
endi
if $data75 != 180.000000000 then
print data75 $data75 != 180.000000000
return -1
endi
if $data85 != 210.000000000 then
print data85 $data85 != 210.000000000
return -1
endi
if $data95 != 200.000000000 then
print data95 $data95 != 200.000000000
return -1
endi
if $data[10][5] != 190.000000000 then
print data[10][5] $data[10][5] != 190.000000000
return -1
endi
if $data[11][5] != 220.000000000 then
print data[11][5] $data[11][5] != 220.000000000
return -1
endi
if $data[12][5] != 230.000000000 then
print data[12][5] $data[12][5] != 230.000000000
return -1
endi
if $data[13][5] != 240.000000000 then
print data[13][5] $data[13][5] != 240.000000000
return -1
endi
print =============== step 12 query records of ct4 from file
sql select * from ct4;
print $data00 $data01 $data02 $data03 $data04 $data05
print $data10 $data11 $data12 $data13 $data14 $data15
print $data20 $data21 $data22 $data23 $data24 $data25
print $data30 $data31 $data32 $data33 $data34 $data35
print $data40 $data41 $data42 $data43 $data44 $data45
if $rows != 5 then
print rows $rows != 5
return -1
endi
if $data01 != 10 then
print data01 $data01 != 10
return -1
endi
if $data11 != 30 then
print data11 $data11 != 30
return -1
endi
if $data21 != 50 then
print data21 $data21 != 50
return -1
endi
if $data31 != 90 then
print data31 $data31 != 90
return -1
endi
if $data41 != 80 then
print data41 $data41 != 80
return -1
endi
if $data03 != b0 then
print data03 $data03 != b0
return -1
endi
if $data13 != b2 then
print data13 $data13 != b2
return -1
endi
if $data23 != b5 then
print data23 $data23 != b5
return -1
endi
if $data33 != b9 then
print data33 $data33 != b9
return -1
endi
if $data43 != b8 then
print data43 $data43 != b8
return -1
endi
if $data04 != n0 then
print data04 $data04 != n0
return -1
endi
if $data14 != n2 then
print data14 $data14 != n2
return -1
endi
if $data24 != n5 then
print data24 $data24 != n5
return -1
endi
if $data34 != n9 then
print data34 $data34 != n9
return -1
endi
if $data44 != n8 then
print data44 $data44 != n8
return -1
endi

16
tests/script/tsim/stream/basic1.sim
View File

@ -391,13 +391,13 @@ if $data02 != 4 then
return -1 return -1
endi endi
if $data03 != 14 then if $data03 != 50 then
print ======$data03 print ======$data03 != 50
return -1 return -1
endi endi
if $data04 != 4 then if $data04 != 20 then
print ======$data04 print ======$data04 != 20
return -1 return -1
endi endi
@ -421,13 +421,13 @@ if $data12 != 4 then
return -1 return -1
endi endi
if $data13 != 10 then if $data13 != 46 then
print ======$data13 print ======$data13 != 46
return -1 return -1
endi endi
if $data14 != 3 then if $data14 != 20 then
print ======$data14 print ======$data14 != 20
return -1 return -1
endi endi