From e5d5782125104231d05d9369dc860e4d182b728d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 9 Jul 2020 15:03:30 +0800 Subject: [PATCH] move key column into SCompBlock --- src/tsdb/inc/tsdbMain.h | 4 +- src/tsdb/src/tsdbRWHelper.c | 120 ++++++++++++++++++++++-------------- 2 files changed, 77 insertions(+), 47 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 7d1a5a52de..97db3eb476 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -206,10 +206,10 @@ typedef struct { int64_t offset : 63; int32_t algorithm : 8; int32_t numOfRows : 24; - int32_t sversion; int32_t len; + int32_t keyLen; // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols int16_t numOfSubBlocks; - int16_t numOfCols; + int16_t numOfCols; // not including timestamp column TSKEY keyFirst; TSKEY keyLast; } SCompBlock; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 326da07a36..f961d144da 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -22,6 +22,7 @@ #include "tfile.h" #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) +#define TSDB_KEY_COL_OFFSET 0 static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, @@ -646,7 +647,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } int nColsNotAllNull = 0; - for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { + for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column SDataCol *pDataCol = pDataCols->cols + ncol; SCompCol *pCompCol = pCompData->cols + nColsNotAllNull; @@ -658,7 +659,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompCol->colId = pDataCol->colId; pCompCol->type = pDataCol->type; - if (tDataTypeDesc[pDataCol->type].getStatisFunc && ncol != 0) { + if (tDataTypeDesc[pDataCol->type].getStatisFunc) { (*tDataTypeDesc[pDataCol->type].getStatisFunc)( (TSKEY *)(pDataCols->cols[0].pData), pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max), &(pCompCol->sum), &(pCompCol->minIndex), &(pCompCol->maxIndex), &(pCompCol->numOfNull)); @@ -673,17 +674,17 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa int32_t toffset = 0; int32_t tsize = TSDB_GET_COMPCOL_LEN(nColsNotAllNull); int32_t lsize = tsize; + int32_t keyLen = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { if (tcol >= nColsNotAllNull) break; SDataCol *pDataCol = pDataCols->cols + ncol; SCompCol *pCompCol = pCompData->cols + tcol; - if (pDataCol->colId != pCompCol->colId) continue; - void *tptr = (void *)((char *)pCompData + lsize); - - pCompCol->offset = toffset; + if (ncol != 0 && (pDataCol->colId != pCompCol->colId)) continue; + void *tptr = POINTER_SHIFT(pCompData, lsize); + int32_t flen = 0; // final length int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); if (pCfg->compression) { @@ -695,22 +696,29 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } } - pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, - tsizeof(pHelper->pBuffer) - lsize, pCfg->compression, - pHelper->compBuffer, tsizeof(pHelper->compBuffer)); + flen = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, + tsizeof(pHelper->pBuffer) - lsize, pCfg->compression, + pHelper->compBuffer, tsizeof(pHelper->compBuffer)); } else { - pCompCol->len = tlen; - memcpy(tptr, pDataCol->pData, pCompCol->len); + flen = tlen; + memcpy(tptr, pDataCol->pData, flen); } // Add checksum - ASSERT(pCompCol->len > 0); - pCompCol->len += sizeof(TSCKSUM); - taosCalcChecksumAppend(0, (uint8_t *)tptr, pCompCol->len); + ASSERT(flen > 0); + flen += sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)tptr, flen); - toffset += pCompCol->len; - lsize += pCompCol->len; - tcol++; + if (ncol != 0) { + pCompCol->offset = toffset; + pCompCol->len = flen; + tcol++; + } else { + keyLen = flen; + } + + toffset += flen; + lsize += flen; } pCompData->delimiter = TSDB_FILE_DELIMITER; @@ -732,8 +740,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompBlock->offset = offset; pCompBlock->algorithm = pCfg->compression; pCompBlock->numOfRows = rowsToWrite; - pCompBlock->sversion = pHelper->tableInfo.sversion; - pCompBlock->len = (int32_t)lsize; + pCompBlock->len = lsize; + pCompBlock->keyLen = keyLen; pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0; pCompBlock->numOfCols = nColsNotAllNull; pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); @@ -1276,10 +1284,13 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, int numOfColIds) { ASSERT(pCompBlock->numOfSubBlocks <= 1); + ASSERT(colIds[0] == 0); - SFile * pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SFile * pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + SCompCol compCol = {0}; - if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) goto _err; + // If only load timestamp column, no need to load SCompData part + if (numOfColIds > 1 && tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) goto _err; int dcol = 0; int ccol = 0; @@ -1298,23 +1309,31 @@ static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, ASSERT(pDataCol->colId == colId); - while (ccol < pCompBlock->numOfCols) { - pCompCol = &pHelper->pCompData->cols[ccol]; - if (pCompCol->colId >= colId) break; - ccol++; - } + if (colId == 0) { // load the key row + compCol.colId = colId; + compCol.len = pCompBlock->keyLen; + compCol.type = pDataCol->type; + compCol.offset = TSDB_KEY_COL_OFFSET; + pCompCol = &compCol; + } else { // load non-key rows + while (ccol < pCompBlock->numOfCols) { + pCompCol = &pHelper->pCompData->cols[ccol]; + if (pCompCol->colId >= colId) break; + ccol++; + } - if (ccol >= pCompBlock->numOfCols || pCompCol->colId > colId) { - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; - continue; - } + if (ccol >= pCompBlock->numOfCols || pCompCol->colId > colId) { + dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); + dcol++; + continue; + } - ASSERT(pCompCol->colId == pDataCol->colId); + ASSERT(pCompCol->colId == pDataCol->colId); + } if (tsdbLoadColData(pHelper, pFile, pCompBlock, pCompCol, pDataCol) < 0) goto _err; dcol++; - ccol++; + if (colId != 0) ccol++; } return 0; @@ -1362,8 +1381,8 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa pDataCols->numOfRows = pCompBlock->numOfRows; // Recover the data - int ccol = 0; - int dcol = 0; + int ccol = 0; // loop iter for SCompCol object + int dcol = 0; // loop iter for SDataCols object while (dcol < pDataCols->numOfCols) { SDataCol *pDataCol = &(pDataCols->cols[dcol]); if (ccol >= pCompData->numOfCols) { @@ -1373,12 +1392,23 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa continue; } - SCompCol *pCompCol = &(pCompData->cols[ccol]); + int16_t tcolId = 0; + int32_t toffset = TSDB_KEY_COL_OFFSET; + int32_t tlen = pCompBlock->keyLen; - if (pCompCol->colId == pDataCol->colId) { + if (dcol != 0) { + SCompCol *pCompCol = &(pCompData->cols[ccol]); + tcolId = pCompCol->colId; + toffset = pCompCol->offset; + tlen = pCompCol->len; + } else { + ASSERT(pDataCol->colId == tcolId); + } + + if (tcolId == pDataCol->colId) { if (pCompBlock->algorithm == TWO_STAGE_COMP) { int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; - if (pCompCol->type == TSDB_DATA_TYPE_BINARY || pCompCol->type == TSDB_DATA_TYPE_NCHAR) { + if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows); } pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize); @@ -1387,16 +1417,16 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa goto _err; } } - if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len, - pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints, - pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) { - tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, - pCompCol->colId, (int64_t)pCompCol->offset); + if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + toffset, tlen, pCompBlock->algorithm, + pCompBlock->numOfRows, pDataCols->maxPoints, pHelper->compBuffer, + tsizeof(pHelper->compBuffer)) < 0) { + tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d", + REPO_ID(pHelper->pRepo), pFile->fname, tcolId, (int64_t)pCompBlock->offset, toffset); goto _err; } + if (dcol != 0) ccol++; dcol++; - ccol++; - } else if (pCompCol->colId < pDataCol->colId) { + } else if (tcolId < pDataCol->colId) { ccol++; } else { // Set current column as NULL and forward