From 9e2188b1961c696ba83f1921bcfff10db912ee7c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 5 Aug 2021 10:25:29 +0800 Subject: [PATCH 1/8] small improvements, fix potential bug for inserting --- src/tsdb/src/tsdbMemTable.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 8bb2d1c44e..4809f6303f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -702,11 +702,12 @@ static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg) { } //row1 has higher priority -static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, STSchema **ppSchema1, STSchema **ppSchema2, STable* pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) { +static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo, + STSchema **ppSchema1, STSchema **ppSchema2, + STable* pTable, int32_t* pPoints, SMemRow* pLastRow) { //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows! if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) { - (*pAffectedRows)++; (*pPoints)++; return NULL; } @@ -715,7 +716,6 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1)); if(pMem == NULL) return NULL; memRowCpy(pMem, row1); - (*pAffectedRows)++; (*pPoints)++; *pLastRow = pMem; return pMem; @@ -750,7 +750,6 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep if(pMem == NULL) return NULL; memRowCpy(pMem, tmp); - (*pAffectedRows)++; (*pPoints)++; *pLastRow = pMem; @@ -758,10 +757,10 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep } static void* tsdbInsertDupKeyMergePacked(void** args) { - return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7], args[8]); + return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]); } -static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pAffectedRows, int64_t* pPoints, SMemRow* pLastRow) { +static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) { if(pSkipList->insertHandleFn == NULL) { tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9); @@ -769,17 +768,16 @@ static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STa dupHandleSavedFunc->args[3] = NULL; dupHandleSavedFunc->args[4] = NULL; dupHandleSavedFunc->args[5] = pTable; - dupHandleSavedFunc->args[6] = pAffectedRows; - dupHandleSavedFunc->args[7] = pPoints; - dupHandleSavedFunc->args[8] = pLastRow; pSkipList->insertHandleFn = dupHandleSavedFunc; } + pSkipList->insertHandleFn->args[6] = pPoints; + pSkipList->insertHandleFn->args[7] = pLastRow; } static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t *pAffectedRows) { STsdbMeta *pMeta = pRepo->tsdbMeta; - int64_t points = 0; + int32_t points = 0; STable *pTable = NULL; SSubmitBlkIter blkIter = {0}; SMemTable *pMemTable = NULL; @@ -830,9 +828,10 @@ static int tsdbInsertDataToTable(STsdbRepo* pRepo, SSubmitBlk* pBlock, int32_t * SMemRow lastRow = NULL; int64_t osize = SL_SIZE(pTableData->pData); - tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, pAffectedRows, &points, &lastRow); + tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow); tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext); int64_t dsize = SL_SIZE(pTableData->pData) - osize; + (*pAffectedRows) += points; if(lastRow != NULL) { From c3413fd830c94e300fbbfec7e7ebfe55d7d46792 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 5 Aug 2021 10:34:24 +0800 Subject: [PATCH 2/8] fix realloc usage in tdataformat --- src/common/src/tdataformat.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 3f0ab7f93e..70911c88a6 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -357,8 +357,9 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { int oldMaxCols = pCols->maxCols; if (schemaNCols(pSchema) > oldMaxCols) { pCols->maxCols = schemaNCols(pSchema); - pCols->cols = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); - if (pCols->cols == NULL) return -1; + void* ptr = (SDataCol *)realloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); + if (ptr == NULL) return -1; + pCols->cols = ptr; for(i = oldMaxCols; i < pCols->maxCols; i++) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; From e11ae1f46e7cee40f1f795d8587fdb70ec4da622 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 5 Aug 2021 10:52:26 +0800 Subject: [PATCH 3/8] memory alloc may fail --- src/common/inc/tdataformat.h | 4 ++-- src/common/src/tdataformat.c | 26 +++++++++++++++++--------- src/tsdb/src/tsdbCommit.c | 2 ++ src/tsdb/src/tsdbMemTable.c | 1 - src/tsdb/src/tsdbReadImpl.c | 3 +++ 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index fb6bab0cf2..f49c80ad86 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -328,11 +328,11 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } int tdAllocMemForCol(SDataCol *pCol, int maxPoints); void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); -void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); +int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); -void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); +int dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 70911c88a6..42854704a5 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -22,7 +22,6 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows, bool forceSetNull); -//TODO: change caller to use return val int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { int spaceNeeded = pCol->bytes * maxPoints; if(IS_VAR_DATA_TYPE(pCol->type)) { @@ -31,7 +30,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { if(pCol->spaceSize < spaceNeeded) { void* ptr = realloc(pCol->pData, spaceNeeded); if(ptr == NULL) { - uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)pCol->spaceSize, + uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)spaceNeeded, strerror(errno)); return -1; } else { @@ -239,20 +238,22 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) { pDataCol->len = 0; } // value from timestamp should be TKEY here instead of TSKEY -void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { +int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); if (isAllRowsNull(pCol)) { if (isNull(value, pCol->type)) { // all null value yet, just return - return; + return 0; } if (numOfRows > 0) { // Find the first not null value, fill all previouse values as NULL - dataColSetNEleNull(pCol, numOfRows, maxPoints); + if(dataColSetNEleNull(pCol, numOfRows, maxPoints) < 0) + return -1; } else { - tdAllocMemForCol(pCol, maxPoints); + if(tdAllocMemForCol(pCol, maxPoints) < 0) + return -1; } } @@ -268,6 +269,7 @@ void dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxP memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; } + return -1; } bool isNEleNull(SDataCol *pCol, int nEle) { @@ -290,8 +292,10 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { } } -void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) { - tdAllocMemForCol(pCol, maxPoints); +int dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) { + if(tdAllocMemForCol(pCol, maxPoints)){ + return -1; + } if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; @@ -302,6 +306,7 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) { setNullN(pCol->pData, pCol->type, pCol->bytes, nEle); pCol->len = TYPE_BYTES[pCol->type] * nEle; } + return 0; } void dataColSetOffset(SDataCol *pCol, int nEle) { @@ -414,7 +419,10 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { if (keepData) { if (pDataCols->cols[i].len > 0) { - tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints); + if(tdAllocMemForCol(&pRet->cols[i], pRet->maxPoints) < 0) { + tdFreeDataCols(pRet); + return NULL; + } pRet->cols[i].len = pDataCols->cols[i].len; memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 6c98283189..9d6b0a3594 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -1277,6 +1277,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt if (key1 < key2) { for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } @@ -1308,6 +1309,7 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt ASSERT(!isRowDel); for (int i = 0; i < pDataCols->numOfCols; i++) { + //TODO: dataColAppendVal may fail dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 4809f6303f..e766d97a97 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -751,7 +751,6 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep memRowCpy(pMem, tmp); (*pPoints)++; - *pLastRow = pMem; return pMem; } diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index 711c32535b..c98f03b7de 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -463,6 +463,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat SDataCol *pDataCol = &(pDataCols->cols[dcol]); if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward + // TODO: dataColSetNEleNull may fail dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); dcol++; continue; @@ -503,6 +504,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat ccol++; } else { // Set current column as NULL and forward + // TODO: dataColSetNEleNull may fail dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); dcol++; } @@ -608,6 +610,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * } if (pBlockCol == NULL) { + // TODO: dataColSetNEleNull may fail dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); continue; } From fcd3b44533662e1ca5917016e92cb519a6d9cee7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Thu, 5 Aug 2021 13:50:12 +0800 Subject: [PATCH 4/8] refactor datacols --- src/common/inc/tdataformat.h | 16 ++++++------- src/common/src/tdataformat.c | 44 ++++++++++++++++-------------------- src/tsdb/src/tsdbCommit.c | 2 +- src/tsdb/src/tsdbCompact.c | 2 +- src/tsdb/src/tsdbMeta.c | 1 - src/tsdb/src/tsdbRead.c | 6 ++--- src/tsdb/src/tsdbReadImpl.c | 19 +++++++++------- 7 files changed, 42 insertions(+), 48 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index f49c80ad86..4e2974b090 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -330,9 +330,9 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints); void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); +void dataColSetNEleNull(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); -int dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) { @@ -357,13 +357,11 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { } typedef struct { - int maxRowSize; - int maxCols; // max number of columns - int maxPoints; // max number of points - - int numOfRows; - int numOfCols; // Total number of cols - int sversion; // TODO: set sversion + int maxCols; // max number of columns + int maxPoints; // max number of points + int numOfRows; + int numOfCols; // Total number of cols + int sversion; // TODO: set sversion SDataCol *cols; } SDataCols; @@ -407,7 +405,7 @@ static FORCE_INLINE TSKEY dataColsKeyLast(SDataCols *pCols) { } } -SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); +SDataCols *tdNewDataCols(int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 42854704a5..0234a44758 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -247,13 +247,10 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo return 0; } + if(tdAllocMemForCol(pCol, maxPoints) < 0) return -1; if (numOfRows > 0) { // Find the first not null value, fill all previouse values as NULL - if(dataColSetNEleNull(pCol, numOfRows, maxPoints) < 0) - return -1; - } else { - if(tdAllocMemForCol(pCol, maxPoints) < 0) - return -1; + dataColSetNEleNull(pCol, numOfRows); } } @@ -269,13 +266,21 @@ int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPo memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); pCol->len += pCol->bytes; } - return -1; + return 0; +} + +static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) { + if (IS_VAR_DATA_TYPE(pCol->type)) { + return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); + } } bool isNEleNull(SDataCol *pCol, int nEle) { if(isAllRowsNull(pCol)) return true; for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; + if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } return true; } @@ -292,11 +297,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { } } -int dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) { - if(tdAllocMemForCol(pCol, maxPoints)){ - return -1; - } - +void dataColSetNEleNull(SDataCol *pCol, int nEle) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; for (int i = 0; i < nEle; i++) { @@ -306,7 +307,6 @@ int dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints) { setNullN(pCol->pData, pCol->type, pCol->bytes, nEle); pCol->len = TYPE_BYTES[pCol->type] * nEle; } - return 0; } void dataColSetOffset(SDataCol *pCol, int nEle) { @@ -323,7 +323,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) { } } -SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { +SDataCols *tdNewDataCols(int maxCols, int maxRows) { SDataCols *pCols = (SDataCols *)calloc(1, sizeof(SDataCols)); if (pCols == NULL) { uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCols), strerror(errno)); @@ -331,6 +331,9 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { } pCols->maxPoints = maxRows; + pCols->maxCols = maxCols; + pCols->numOfRows = 0; + pCols->numOfCols = 0; if (maxCols > 0) { pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol)); @@ -347,13 +350,8 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; } - - pCols->maxCols = maxCols; } - pCols->maxRowSize = maxRowSize; - - return pCols; } @@ -372,10 +370,6 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { } } - if (schemaTLen(pSchema) > pCols->maxRowSize) { - pCols->maxRowSize = schemaTLen(pSchema); - } - tdResetDataCols(pCols); pCols->numOfCols = schemaNCols(pSchema); @@ -404,7 +398,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { } SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { - SDataCols *pRet = tdNewDataCols(pDataCols->maxRowSize, pDataCols->maxCols, pDataCols->maxPoints); + SDataCols *pRet = tdNewDataCols(pDataCols->maxCols, pDataCols->maxPoints); if (pRet == NULL) return NULL; pRet->numOfCols = pDataCols->numOfCols; @@ -593,7 +587,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0 && (forceSetNull || (!forceSetNull && !isNull(src2->cols[i].pData, src2->cols[i].type)))) { + if (src2->cols[i].len > 0 && (forceSetNull || (!isNull(src2->cols[i].pData, src2->cols[i].type)))) { dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, target->maxPoints); } diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 9d6b0a3594..3b0db8da6a 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -722,7 +722,7 @@ static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) { return -1; } - pCommith->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pCommith->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pCommith->pDataCols == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyCommitH(pCommith); diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 62f9e41119..5ccb9e90f2 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -296,7 +296,7 @@ static int tsdbCompactMeta(STsdbRepo *pRepo) { return -1; } - pComph->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pComph->pDataCols = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pComph->pDataCols == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyCompactH(pComph); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 619b32b3d9..21150c66e2 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -17,7 +17,6 @@ #define TSDB_SUPER_TABLE_SL_LEVEL 5 #define DEFAULT_TAG_INDEX_COLUMN 0 -static int tsdbCompareSchemaVersion(const void *key1, const void *key2); static char * getTagIndexKey(const void *pData); static STable *tsdbNewTable(); static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index c578555df2..716f82d154 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -466,7 +466,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC STsdbMeta* pMeta = tsdbGetMeta(tsdb); assert(pMeta != NULL); - pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); + pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); if (pQueryHandle->pDataCols == NULL) { tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1446,7 +1446,7 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { +static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { char* pData = NULL; int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; @@ -1481,7 +1481,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; } - if (pColInfo->info.colId == src->colId) { + if (!isAllRowsNull(src) && pColInfo->info.colId == src->colId) { if (pColInfo->info.type != TSDB_DATA_TYPE_BINARY && pColInfo->info.type != TSDB_DATA_TYPE_NCHAR) { memmove(pData, (char*)src->pData + bytes * start, bytes * num); } else { // handle the var-string diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index c98f03b7de..29f1385616 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -42,14 +42,14 @@ int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) { return -1; } - pReadh->pDCols[0] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pReadh->pDCols[0] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pReadh->pDCols[0] == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyReadH(pReadh); return -1; } - pReadh->pDCols[1] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock); + pReadh->pDCols[1] = tdNewDataCols(0, pCfg->maxRowsPerFileBlock); if (pReadh->pDCols[1] == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbDestroyReadH(pReadh); @@ -463,8 +463,9 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat SDataCol *pDataCol = &(pDataCols->cols[dcol]); if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward - // TODO: dataColSetNEleNull may fail - dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + // TODO: tdAllocMemForCol may fail + tdAllocMemForCol(pDataCol, pDataCols->maxPoints); + dataColSetNEleNull(pDataCol, pBlock->numOfRows); dcol++; continue; } @@ -504,8 +505,9 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat ccol++; } else { // Set current column as NULL and forward - // TODO: dataColSetNEleNull may fail - dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + // TODO: tdAllocMemForCol may fail + tdAllocMemForCol(pDataCol, pDataCols->maxPoints); + dataColSetNEleNull(pDataCol, pBlock->numOfRows); dcol++; } } @@ -610,8 +612,9 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * } if (pBlockCol == NULL) { - // TODO: dataColSetNEleNull may fail - dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints); + // TODO: tdAllocMemForCol may fail + tdAllocMemForCol(pDataCol, pDataCols->maxPoints); + dataColSetNEleNull(pDataCol, pBlock->numOfRows); continue; } From 347ff765cf220270dc4c76b132169730ad00db09 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 9 Aug 2021 03:24:43 +0800 Subject: [PATCH 5/8] fix merge --- src/common/src/tdataformat.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 0234a44758..b62c97bc2a 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -587,7 +587,10 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0 && (forceSetNull || (!isNull(src2->cols[i].pData, src2->cols[i].type)))) { + if(!forceSetNull && (isAllRowsNull(&src2->cols[i]) || isNull(src2->cols[i].pData, src2->cols[i].type)) && key1 == key2) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, + target->maxPoints); + } else if (src2->cols[i].len > 0) { dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, target->maxPoints); } From 855a76813ce1f84b3a870e0b4366a031c75cefc2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 9 Aug 2021 04:00:07 +0800 Subject: [PATCH 6/8] fix merge --- src/common/src/tdataformat.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index b62c97bc2a..3b48bf9977 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -587,12 +587,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); - if(!forceSetNull && (isAllRowsNull(&src2->cols[i]) || isNull(src2->cols[i].pData, src2->cols[i].type)) && key1 == key2) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, - target->maxPoints); - } else if (src2->cols[i].len > 0) { + if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) { dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, target->maxPoints); + } else if(!forceSetNull && key1 == key2 && src1->cols[i].len > 0) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src1->cols + i, *iter1), target->numOfRows, + target->maxPoints); } } target->numOfRows++; From 1d3a2d47f151b532123be6018eb8ef1cd3d2b2e0 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 9 Aug 2021 05:19:22 +0800 Subject: [PATCH 7/8] remove set null in tsdbread --- src/tsdb/src/tsdbReadImpl.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index 29f1385616..237025864e 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -464,8 +464,9 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward // TODO: tdAllocMemForCol may fail - tdAllocMemForCol(pDataCol, pDataCols->maxPoints); - dataColSetNEleNull(pDataCol, pBlock->numOfRows); + /*tdAllocMemForCol(pDataCol, pDataCols->maxPoints);*/ + /*dataColSetNEleNull(pDataCol, pBlock->numOfRows);*/ + dataColReset(pDataCol); dcol++; continue; } @@ -506,8 +507,9 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat } else { // Set current column as NULL and forward // TODO: tdAllocMemForCol may fail - tdAllocMemForCol(pDataCol, pDataCols->maxPoints); - dataColSetNEleNull(pDataCol, pBlock->numOfRows); + /*tdAllocMemForCol(pDataCol, pDataCols->maxPoints);*/ + /*dataColSetNEleNull(pDataCol, pBlock->numOfRows);*/ + dataColReset(pDataCol); dcol++; } } @@ -613,8 +615,9 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * if (pBlockCol == NULL) { // TODO: tdAllocMemForCol may fail - tdAllocMemForCol(pDataCol, pDataCols->maxPoints); - dataColSetNEleNull(pDataCol, pBlock->numOfRows); + /*tdAllocMemForCol(pDataCol, pDataCols->maxPoints);*/ + /*dataColSetNEleNull(pDataCol, pBlock->numOfRows);*/ + dataColReset(pDataCol); continue; } From 4521970f9932acbdb1d6606cd3aa3de055fdfff2 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Mon, 9 Aug 2021 14:20:36 +0800 Subject: [PATCH 8/8] refactor: dataColSetNEleNull should not be used by other modules and be set to static --- src/common/inc/tdataformat.h | 1 - src/common/src/tdataformat.c | 3 ++- src/tsdb/src/tsdbCommit.c | 1 - src/tsdb/src/tsdbReadImpl.c | 9 --------- 4 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 4e2974b090..1637c4832b 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -330,7 +330,6 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints); void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints); int dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints); void dataColSetOffset(SDataCol *pCol, int nEle); -void dataColSetNEleNull(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 3b48bf9977..c793d241f6 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -19,6 +19,7 @@ #include "wchar.h" #include "tarray.h" +static void dataColSetNEleNull(SDataCol *pCol, int nEle); static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows, bool forceSetNull); @@ -297,7 +298,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { } } -void dataColSetNEleNull(SDataCol *pCol, int nEle) { +static void dataColSetNEleNull(SDataCol *pCol, int nEle) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; for (int i = 0; i < nEle; i++) { diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 3b0db8da6a..8f5f885d69 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -920,7 +920,6 @@ int tsdbWriteBlockImpl(STsdbRepo *pRepo, STable *pTable, SDFile *pDFile, SDataCo SDataCol * pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; - // if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it continue; } diff --git a/src/tsdb/src/tsdbReadImpl.c b/src/tsdb/src/tsdbReadImpl.c index 237025864e..74d41cce19 100644 --- a/src/tsdb/src/tsdbReadImpl.c +++ b/src/tsdb/src/tsdbReadImpl.c @@ -463,9 +463,6 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat SDataCol *pDataCol = &(pDataCols->cols[dcol]); if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward - // TODO: tdAllocMemForCol may fail - /*tdAllocMemForCol(pDataCol, pDataCols->maxPoints);*/ - /*dataColSetNEleNull(pDataCol, pBlock->numOfRows);*/ dataColReset(pDataCol); dcol++; continue; @@ -506,9 +503,6 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat ccol++; } else { // Set current column as NULL and forward - // TODO: tdAllocMemForCol may fail - /*tdAllocMemForCol(pDataCol, pDataCols->maxPoints);*/ - /*dataColSetNEleNull(pDataCol, pBlock->numOfRows);*/ dataColReset(pDataCol); dcol++; } @@ -614,9 +608,6 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * } if (pBlockCol == NULL) { - // TODO: tdAllocMemForCol may fail - /*tdAllocMemForCol(pDataCol, pDataCols->maxPoints);*/ - /*dataColSetNEleNull(pDataCol, pBlock->numOfRows);*/ dataColReset(pDataCol); continue; }