From 11653686420b652b35a83d48d928bae7c245ee1b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 2 Apr 2022 15:39:20 +0800 Subject: [PATCH 1/6] retentions param bug fix and deliver to vnode --- source/common/src/tmsg.c | 6 +++--- source/dnode/mgmt/vm/src/vmMsg.c | 5 +++++ source/dnode/mnode/impl/src/mndDb.c | 3 ++- source/dnode/vnode/inc/tsdb.h | 3 ++- source/libs/parser/src/parTranslater.c | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 2a16b58565..988230bd3f 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -367,7 +367,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nCols)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.nBSmaCols)); pReq->stbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->stbCfg.nCols * sizeof(SSchemaEx)); - for (col_id_t i = 0; i < pReq->stbCfg.nCols; i++) { + for (col_id_t i = 0; i < pReq->stbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].type)); buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pSchema[i].sma)); buf = taosDecodeFixedI16(buf, &(pReq->stbCfg.pSchema[i].colId)); @@ -376,7 +376,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } buf = taosDecodeFixedI16(buf, &pReq->stbCfg.nTagCols); pReq->stbCfg.pTagSchema = (SSchema *)taosMemoryMalloc(pReq->stbCfg.nTagCols * sizeof(SSchema)); - for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; i++) { + for (col_id_t i = 0; i < pReq->stbCfg.nTagCols; ++i) { buf = taosDecodeFixedI8(buf, &(pReq->stbCfg.pTagSchema[i].type)); buf = taosDecodeFixedI16(buf, &pReq->stbCfg.pTagSchema[i].colId); buf = taosDecodeFixedI32(buf, &pReq->stbCfg.pTagSchema[i].bytes); @@ -408,7 +408,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.nCols); buf = taosDecodeFixedI16(buf, &(pReq->ntbCfg.nBSmaCols)); pReq->ntbCfg.pSchema = (SSchemaEx *)taosMemoryMalloc(pReq->ntbCfg.nCols * sizeof(SSchemaEx)); - for (col_id_t i = 0; i < pReq->ntbCfg.nCols; i++) { + for (col_id_t i = 0; i < pReq->ntbCfg.nCols; ++i) { buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].type); buf = taosDecodeFixedI8(buf, &pReq->ntbCfg.pSchema[i].sma); buf = taosDecodeFixedI16(buf, &pReq->ntbCfg.pSchema[i].colId); diff --git a/source/dnode/mgmt/vm/src/vmMsg.c b/source/dnode/mgmt/vm/src/vmMsg.c index c3ad74d246..a5e8fe6088 100644 --- a/source/dnode/mgmt/vm/src/vmMsg.c +++ b/source/dnode/mgmt/vm/src/vmMsg.c @@ -30,6 +30,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbCfg.keep1 = pCreate->daysToKeep2; pCfg->tsdbCfg.keep2 = pCreate->daysToKeep0; pCfg->tsdbCfg.lruCacheSize = pCreate->cacheBlockSize; + pCfg->tsdbCfg.retentions = pCreate->pRetensions; pCfg->metaCfg.lruSize = pCreate->cacheBlockSize; pCfg->walCfg.fsyncPeriod = pCreate->fsyncPeriod; pCfg->walCfg.level = pCreate->walLevel; @@ -70,6 +71,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { SVnodeObj *pVnode = vmAcquireVnode(pMgmt, createReq.vgId); if (pVnode != NULL) { + tFreeSCreateVnodeReq(&createReq); dDebug("vgId:%d, already exist", createReq.vgId); vmReleaseVnode(pMgmt, pVnode); terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; @@ -88,12 +90,14 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { + tFreeSCreateVnodeReq(&createReq); dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); return -1; } int32_t code = vmOpenVnode(pMgmt, &wrapperCfg, pImpl); if (code != 0) { + tFreeSCreateVnodeReq(&createReq); dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); @@ -103,6 +107,7 @@ int32_t vmProcessCreateVnodeReq(SVnodesMgmt *pMgmt, SNodeMsg *pMsg) { code = vmWriteVnodesToFile(pMgmt); if (code != 0) { + tFreeSCreateVnodeReq(&createReq); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); terrno = code; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 462f0eb85a..44c547bec3 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -69,7 +69,8 @@ void mndCleanupDb(SMnode *pMnode) {} static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, sizeof(SDbObj) + TSDB_DB_RESERVE_SIZE); + SSdbRaw *pRaw = sdbAllocRaw(SDB_DB, TSDB_DB_VER_NUMBER, + sizeof(SDbObj) + pDb->cfg.numOfRetensions * sizeof(SRetention) + TSDB_DB_RESERVE_SIZE); if (pRaw == NULL) goto DB_ENCODE_OVER; int32_t dataPos = 0; diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index 0d3fcffe7d..b287a1fe4c 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -55,13 +55,14 @@ typedef struct STsdbCfg { int8_t precision; int8_t update; int8_t compression; - uint64_t lruCacheSize; int32_t daysPerFile; int32_t minRowsPerFileBlock; int32_t maxRowsPerFileBlock; int32_t keep; int32_t keep1; int32_t keep2; + uint64_t lruCacheSize; + SArray *retentions; } STsdbCfg; // query condition to build multi-table data block iterator diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 773060beab..09f56f1657 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -938,7 +938,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe SNode* pNode = NULL; int32_t index = 0; FOREACH(pNode, pRetentions) { - if (0 == index % 2) { + if (0 == ((index++) & 1)) { pFreq = (SValueNode*)pNode; } else { pKeep = (SValueNode*)pNode; @@ -951,6 +951,7 @@ static int32_t buildCreateDbRetentions(const SNodeList* pRetentions, SCreateDbRe taosArrayPush(pReq->pRetensions, &retention); } } + pReq->numOfRetensions = taosArrayGetSize(pReq->pRetensions); } return TSDB_CODE_SUCCESS; } From b3bb652708c0e50d88d0821db9673914e8c6db42 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 18:02:30 +0800 Subject: [PATCH 2/6] data and bitmap compress/store/decompress separatedly --- include/common/tdataformat.h | 5 +- include/common/trow.h | 28 ++++----- source/common/src/tdataformat.c | 26 +++++--- source/common/src/trow.c | 2 +- source/dnode/vnode/src/inc/tsdbReadImpl.h | 8 +-- source/dnode/vnode/src/tsdb/tsdbCommit.c | 33 ++++++++-- source/dnode/vnode/src/tsdb/tsdbReadImpl.c | 73 +++++++++++++++------- source/util/src/tcompression.c | 4 +- 8 files changed, 120 insertions(+), 59 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index da724206d9..991cb55e50 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -135,8 +135,8 @@ typedef struct { #define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4 #define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder -#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS)) -#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8)) +#define TD_BITMAP_BYTES(cnt) (ceil((double)(cnt) / TD_VTYPE_PARTS)) +#define TD_BIT_TO_BYTES(cnt) (ceil((double)(cnt) / 8)) int32_t tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version); void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder); @@ -365,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche } #endif // ----------------- Data column structure +// SDataCol arrangement: data => bitmap => dataOffset typedef struct SDataCol { int8_t type; // column type uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM diff --git a/include/common/trow.h b/include/common/trow.h index 01f4076382..7cde8c50c5 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -147,20 +147,20 @@ typedef struct { typedef struct { // basic info - int8_t rowType; - int16_t sver; - STSRow *pBuf; + int8_t rowType; + schema_ver_t sver; + STSRow *pBuf; // extended info - int32_t flen; - int16_t nBoundCols; - int16_t nCols; - int16_t nBitmaps; - int16_t nBoundBitmaps; - int32_t offset; - void *pBitmap; - void *pOffset; - int32_t extendedRowSize; + int32_t flen; + col_id_t nBoundCols; + col_id_t nCols; + col_id_t nBitmaps; + col_id_t nBoundBitmaps; + int32_t offset; + void *pBitmap; + void *pOffset; + int32_t extendedRowSize; } SRowBuilder; #define TD_ROW_HEAD_LEN (sizeof(STSRow)) @@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t } #ifdef TD_SUPPORT_BITMAP // the primary TS key is stored separatedly - pBuilder->nBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); + pBuilder->nBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nCols - 1); if (nBoundCols > 0) { - pBuilder->nBoundBitmaps = (int16_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); + pBuilder->nBoundBitmaps = (col_id_t)TD_BITMAP_BYTES(pBuilder->nBoundCols - 1); } else { pBuilder->nBoundBitmaps = 0; } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1d7a4aeb8f..c1225012ac 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { spaceNeeded += (int)nBitmapBytes; // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered. - spaceNeeded += TYPE_BYTES[pCol->type]; + // spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove + // the additional space #endif if (pCol->spaceSize < spaceNeeded) { @@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { } } #ifdef TD_SUPPORT_BITMAP + if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints); pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes); @@ -306,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row) bool isNEleNull(SDataCol *pCol, int nEle) { if (isAllRowsNull(pCol)) return true; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { if (!isNull(tdGetColDataOfRowUnsafe(pCol, i), pCol->type)) return false; } return true; @@ -327,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) { static void dataColSetNEleNull(SDataCol *pCol, int nEle) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->len = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { dataColSetNullAt(pCol, i); } } else { @@ -343,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) { // char *tptr = (char *)(pCol->pData); VarDataOffsetT offset = 0; - for (int i = 0; i < nEle; i++) { + for (int i = 0; i < nEle; ++i) { pCol->dataOff[i] = offset; offset += varDataTLen(tptr); tptr = POINTER_SHIFT(tptr, varDataTLen(tptr)); @@ -371,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { tdFreeDataCols(pCols); return NULL; } +#if 0 // no need as calloc used int i; for (i = 0; i < maxCols; i++) { pCols->cols[i].spaceSize = 0; @@ -378,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; } +#endif } return pCols; @@ -391,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { void *ptr = (SDataCol *)taosMemoryRealloc(pCols->cols, sizeof(SDataCol) * pCols->maxCols); if (ptr == NULL) return -1; pCols->cols = ptr; - for (i = oldMaxCols; i < pCols->maxCols; i++) { + for (i = oldMaxCols; i < pCols->maxCols; ++i) { pCols->cols[i].pData = NULL; pCols->cols[i].dataOff = NULL; + pCols->cols[i].pBitmap = NULL; pCols->cols[i].spaceSize = 0; } } +#if 0 + tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...) +#endif - tdResetDataCols(pCols); + pCols->numOfRows = 0; pCols->numOfCols = schemaNCols(pSchema); - for (i = 0; i < schemaNCols(pSchema); i++) { + for (i = 0; i < schemaNCols(pSchema); ++i) { dataColInit(pCols->cols + i, schemaColAt(pSchema, i), pCols->maxPoints); } @@ -413,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) { if (pCols) { if (pCols->cols) { int maxCols = pCols->maxCols; - for (i = 0; i < maxCols; i++) { + for (i = 0; i < maxCols; ++i) { SDataCol *pCol = &pCols->cols[i]; taosMemoryFreeClear(pCol->pData); } @@ -464,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { void tdResetDataCols(SDataCols *pCols) { if (pCols != NULL) { pCols->numOfRows = 0; - for (int i = 0; i < pCols->maxCols; i++) { + for (int i = 0; i < pCols->maxCols; ++i) { dataColReset(pCols->cols + i); } } diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9ee2ec1300..0d5a874c1c 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, dataOffSize); } if (!TD_COL_ROWS_NORM(pRet->cols + i)) { - int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->maxPoints); + int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(pDataCols->numOfRows); memcpy(pRet->cols[i].pBitmap, pDataCols->cols[i].pBitmap, nBitmapBytes); } } diff --git a/source/dnode/vnode/src/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h index 17c220a35a..90a877bb83 100644 --- a/source/dnode/vnode/src/inc/tsdbReadImpl.h +++ b/source/dnode/vnode/src/inc/tsdbReadImpl.h @@ -112,10 +112,10 @@ typedef struct { #else typedef struct { int16_t colId; - uint8_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM - uint8_t reserve : 7; - uint8_t type; - int32_t len; + uint16_t type : 6; + uint16_t blen : 10; // bitmap length(TODO: full UT for the bitmap compress of various data input) + uint32_t bitmap : 1; // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM + uint32_t len : 31; // data length + bitmap length uint32_t offset; } SBlockColV0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index d5e9b55a71..497bae6ee1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1281,7 +1281,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ++ncol) { // All not NULL columns finish @@ -1297,7 +1297,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF #ifdef TD_SUPPORT_BITMAP int32_t tBitmaps = 0; + int32_t tBitmapsLen = 0; if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tBitmaps = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tlen += tBitmaps; @@ -1305,16 +1308,17 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tlen += tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif // move bitmap parts ahead // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove - memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); + // memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps); } #endif void *tptr; // Make room - if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { + if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { return -1; } pBlockData = (SBlockData *)(*ppBuf); @@ -1327,23 +1331,44 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF // Compress or just copy if (pCfg->compression) { +#if 0 flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr, tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tlen + COMP_OVERFLOW_BYTES); +#endif + flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, + tlen + COMP_OVERFLOW_BYTES); + if (tBitmaps > 0) { + tptr = POINTER_SHIFT(pBlockData, lsize + flen); + tBitmapsLen = + tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, rowsToWrite, tptr, tBitmaps + COMP_OVERFLOW_BYTES, + pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES); + TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES))); + flen += tBitmapsLen; + } } else { flen = tlen; memcpy(tptr, pDataCol->pData, flen); + if (tBitmaps > 0) { + tptr = POINTER_SHIFT(pBlockData, lsize + flen); + memcpy(tptr, pDataCol->pBitmap, tBitmaps); + tBitmapsLen = tBitmaps; + flen += tBitmapsLen; + } } // Add checksum ASSERT(flen > 0); + ASSERT(tBitmapsLen <= 1024); flen += sizeof(TSCKSUM); taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM))); if (ncol != 0) { tsdbSetBlockColOffset(pBlockCol, toffset); - pBlockCol->len = flen; + pBlockCol->len = flen; // data + bitmaps + pBlockCol->blen = tBitmapsLen; ++tcol; } else { keyLen = flen; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c index 304b3286fe..8cb90d76a6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadImpl.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c @@ -21,9 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh); static void tsdbResetReadFile(SReadH *pReadh); static int tsdbLoadBlockOffset(SReadH *pReadh, SBlock *pBlock); static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols); -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize); +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds, int numOfColIds); static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol); @@ -548,7 +547,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat if (dcol != 0 && ccol >= pBlockData->numOfCols) { // Set current column as NULL and forward dataColReset(pDataCol); - dcol++; + ++dcol; continue; } @@ -567,9 +566,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat TD_SET_COL_ROWS_NORM(pDataCol); } - int32_t tBitmaps = 0; + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -577,17 +578,18 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } if (tcolId == pDataCol->colId) { if (pBlock->algorithm == TWO_STAGE_COMP) { - int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES; + int zsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm, - pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh), - (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { + if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlockCol->blen, + pBlock->algorithm, pBlock->numOfRows, tLenBitmap, pDataCols->maxPoints, + TSDB_READ_COMP_BUF(pReadh), (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u", TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset); return -1; @@ -609,9 +611,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat return 0; } -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows, - int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer, - int bufferSize) { +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int32_t bitmapLen, int8_t comp, + int numOfRows, int numOfBitmaps, int maxPoints, char *buffer, int bufferSize) { if (!taosCheckChecksumWhole((uint8_t *)content, len)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; @@ -623,21 +624,41 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 if (comp) { // Need to decompress int tlen = - (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows + numOfBitmaps, + (*(tDataTypes[pDataCol->type].decompFunc))(content, len - bitmapLen - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize); if (tlen <= 0) { - tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", - len, comp, numOfRows, maxPoints, bufferSize); + tsdbError( + "Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d", + (int32_t)(len - bitmapLen - sizeof(TSCKSUM)), comp, numOfRows, maxPoints, bufferSize); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } pDataCol->len = tlen; + + if (numOfBitmaps > 0) { + tlen = tsDecompressTinyint(POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen, numOfBitmaps, + pDataCol->pBitmap, pDataCol->spaceSize, comp, buffer, bufferSize); + if (tlen <= 0) { + tsdbError( + "Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d " + "bufferSize:%d", + bitmapLen, comp, numOfBitmaps, maxPoints, bufferSize); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } + // pDataCol->blen = tlen; + } } else { // No need to decompress, just memcpy it - pDataCol->len = len - sizeof(TSCKSUM); + pDataCol->len = len - bitmapLen - sizeof(TSCKSUM); memcpy(pDataCol->pData, content, pDataCol->len); + if (numOfBitmaps > 0) { + // pDataCol->blen = bitmapLen; + memcpy(pDataCol->pBitmap, POINTER_SHIFT(content, len - bitmapLen - sizeof(TSCKSUM)), bitmapLen); + } } +#if 0 if (lenOfBitmaps > 0) { pDataCol->len -= lenOfBitmaps; @@ -653,7 +674,10 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32 } else if (IS_VAR_DATA_TYPE(pDataCol->type)) { dataColSetOffset(pDataCol, numOfRows); } - +#endif + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + dataColSetOffset(pDataCol, numOfRows); + } return 0; } @@ -740,14 +764,16 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols * static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) { ASSERT(pDataCol->colId == pBlockCol->colId); - STsdb * pRepo = TSDB_READ_REPO(pReadh); + STsdb *pRepo = TSDB_READ_REPO(pReadh); STsdbCfg *pCfg = REPO_CFG(pRepo); - int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); - int32_t tBitmaps = 0; + int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows); + // int32_t tBitmaps = 0; int32_t tLenBitmap = 0; if (!TD_COL_ROWS_NORM(pBlockCol)) { + tLenBitmap = nBitmaps; +#if 0 if (IS_VAR_DATA_TYPE(pDataCol->type)) { tBitmaps = nBitmaps; tLenBitmap = tBitmaps; @@ -755,9 +781,10 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]); tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type]; } +#endif } - int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES; + int tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + 2 * COMP_OVERFLOW_BYTES; if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1; if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1; @@ -785,8 +812,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc return -1; } - if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows, - tBitmaps, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, + if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlockCol->blen, pBlock->algorithm, + pBlock->numOfRows, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) { tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), pBlockCol->colId, offset); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b2332aded7..041aec4054 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -28,11 +28,11 @@ * * BOOLEAN Compression Algorithm: * We provide two methods for compress boolean types. Because boolean types in C - * code are char bytes with 0 and 1 values only, only one bit can used to discrimenate + * code are char bytes with 0 and 1 values only, only one bit can used to discriminate * the values. * 1. The first method is using only 1 bit to represent the boolean value with 1 for * true and 0 for false. Then the compression rate is 1/8. - * 2. The second method is using run length encoding (RLE) methods. This methos works + * 2. The second method is using run length encoding (RLE) methods. This method works * better when there are a lot of consecutive true values or false values. * * STRING Compression Algorithm: From 489f65f2ce95954d1cb8c189bef51a0e5d2afa37 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 18:10:21 +0800 Subject: [PATCH 3/6] trigger CI From 670554dcdd7ec8f63dcdb8c56e16c75079c5670b Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 18:56:44 +0800 Subject: [PATCH 4/6] bug fix --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 497bae6ee1..2a8875fefe 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1315,7 +1315,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } #endif - void *tptr; + void *tptr, *bptr; // Make room if (tsdbMakeRoom(ppBuf, lsize + tlen + tBitmaps + 2 * COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { @@ -1340,9 +1340,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tlen + COMP_OVERFLOW_BYTES); if (tBitmaps > 0) { - tptr = POINTER_SHIFT(pBlockData, lsize + flen); + bptr = POINTER_SHIFT(pBlockData, lsize + flen); tBitmapsLen = - tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, rowsToWrite, tptr, tBitmaps + COMP_OVERFLOW_BYTES, + tsCompressTinyint((char *)pDataCol->pBitmap, tBitmaps, tBitmaps, bptr, tBitmaps + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf, tBitmaps + COMP_OVERFLOW_BYTES); TASSERT((tBitmapsLen > 0) && (tBitmapsLen <= (tBitmaps + COMP_OVERFLOW_BYTES))); flen += tBitmapsLen; @@ -1351,8 +1351,8 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF flen = tlen; memcpy(tptr, pDataCol->pData, flen); if (tBitmaps > 0) { - tptr = POINTER_SHIFT(pBlockData, lsize + flen); - memcpy(tptr, pDataCol->pBitmap, tBitmaps); + bptr = POINTER_SHIFT(pBlockData, lsize + flen); + memcpy(bptr, pDataCol->pBitmap, tBitmaps); tBitmapsLen = tBitmaps; flen += tBitmapsLen; } From 3ce4784a74b5ee6448ca0e484707b86c243b374d Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 19:48:15 +0800 Subject: [PATCH 5/6] trigger CI From cea37a8e8f4880eb63865c15957b133f0a4f2705 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 3 Apr 2022 20:28:12 +0800 Subject: [PATCH 6/6] trigger CI