From 678f9da4acf7ffad4fb803ebcde0e9b6b2dd5b6e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 10:11:55 +0800 Subject: [PATCH 01/59] refact compress code --- include/util/tcompression.h | 340 ++++----------------------------- source/util/src/tcompression.c | 275 +++++++++++++++++++++++++- 2 files changed, 313 insertions(+), 302 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index d7ba4dfa3f..5196c01fc6 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -51,287 +51,12 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 -extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type); -extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, - const char type); -extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize); -extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output, - int32_t outputSize); -extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output); -// lossy -extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output); -extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, - char *const output); -extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output); -extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, - char *const output); - #ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(); void tsCompressExit(); -#endif -static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressBoolImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressBoolImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - return tsCompressStringImp(input, inputSize, output, outputSize); -} - -static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - return tsDecompressStringImp(input, compressedSize, output, outputSize); -} - -static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { -#ifdef TD_TSZ - // lossy mode - if (lossyFloat) { - return tsCompressFloatLossyImp(input, nelements, output); - // lossless mode - } else { -#endif - if (algorithm == ONE_STAGE_COMP) { - return tsCompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressFloatImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { -#ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { - // decompress lossy - return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); - } else { -#endif - // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressFloatImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { -#ifdef TD_TSZ - if (lossyDouble) { - // lossy mode - return tsCompressDoubleLossyImp(input, nelements, output); - } else { -#endif - // lossless mode - if (algorithm == ONE_STAGE_COMP) { - return tsCompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressDoubleImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { -#ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { - // decompress lossy - return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); - } else { -#endif - // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressDoubleImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -#ifdef TD_TSZ -// -// lossy float double -// static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { @@ -358,33 +83,46 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int #endif -static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressTimestampImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} +int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressTimestampImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -} +int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize); +int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize); +int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize); +int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); #ifdef __cplusplus } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 62b9d87628..1128c51f9b 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1612,4 +1612,277 @@ int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); -} \ No newline at end of file +} + +/************************************************************************* + * REGULAR COMPRESSION + *************************************************************************/ +// Timestamp ===================================================== +int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressTimestampImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressTimestampImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressTimestampImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressTimestampImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +} + +// Float ===================================================== +int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + // lossy mode + if (lossyFloat) { + return tsCompressFloatLossyImp(input, nelements, output); + // lossless mode + } else { +#endif + if (algorithm == ONE_STAGE_COMP) { + return tsCompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressFloatImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { + // decompress lossy + return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); + } else { +#endif + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressFloatImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +// Double ===================================================== +int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (lossyDouble) { + // lossy mode + return tsCompressDoubleLossyImp(input, nelements, output); + } else { +#endif + // lossless mode + if (algorithm == ONE_STAGE_COMP) { + return tsCompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressDoubleImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { + // decompress lossy + return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); + } else { +#endif + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressDoubleImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +// Binary ===================================================== +int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + return tsCompressStringImp(input, inputSize, output, outputSize); +} + +int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + return tsDecompressStringImp(input, compressedSize, output, outputSize); +} + +// Bool ===================================================== +int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressBoolImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressBoolImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressBoolImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressBoolImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +} + +// Tinyint ===================================================== +int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else { + assert(0); + return -1; + } +} + +// Smallint ===================================================== +int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else { + assert(0); + return -1; + } +} + +// Int ===================================================== +int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); + } else { + assert(0); + return -1; + } +} + +// Bigint ===================================================== +int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else { + assert(0); + return -1; + } +} From 215883ea643d43392991380a24fcd4d6fd7e1607 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 11:08:46 +0800 Subject: [PATCH 02/59] refact code --- include/common/ttypes.h | 9 +- include/util/tcompression.h | 76 +++++----- source/util/src/tcompression.c | 249 ++++++++++++++++----------------- 3 files changed, 163 insertions(+), 171 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index a88f65f6ac..14e9a5af87 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -333,10 +333,10 @@ typedef struct tDataTypeDescriptor { char *name; int64_t minValue; int64_t maxValue; - int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); + int32_t (*compFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); + int32_t (*decompFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; @@ -356,7 +356,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void *getDataMin(int32_t type); void *getDataMax(int32_t type); - #ifdef __cplusplus } #endif diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 5196c01fc6..b620615ab1 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -83,46 +83,42 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int #endif -int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - -int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize); -int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize); -int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize); -int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); #ifdef __cplusplus } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 1128c51f9b..30b72f6fbe 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1618,27 +1618,26 @@ int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { * REGULAR COMPRESSION *************************************************************************/ // Timestamp ===================================================== -int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressTimestampImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressTimestampImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressTimestampImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressTimestampImp(buffer, nelements, output); +int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressTimestampImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressTimestampImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1646,20 +1645,20 @@ int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, c } // Float ===================================================== -int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ // lossy mode if (lossyFloat) { - return tsCompressFloatLossyImp(input, nelements, output); + return tsCompressFloatLossyImp(pIn, nEle, pOut); // lossless mode } else { #endif - if (algorithm == ONE_STAGE_COMP) { - return tsCompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressFloatImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressFloatImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressFloatImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; @@ -1669,20 +1668,20 @@ int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_ #endif } -int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { + if (HEAD_ALGO(pIn[0]) == ALGO_SZ_LOSSY) { // decompress lossy - return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); + return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); } else { #endif // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressFloatImp(buffer, nelements, output); + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressFloatImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressFloatImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1693,20 +1692,20 @@ int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const } // Double ===================================================== -int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ if (lossyDouble) { // lossy mode - return tsCompressDoubleLossyImp(input, nelements, output); + return tsCompressDoubleLossyImp(pIn, nEle, pOut); } else { #endif // lossless mode - if (algorithm == ONE_STAGE_COMP) { - return tsCompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressDoubleImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressDoubleImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressDoubleImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; @@ -1716,20 +1715,20 @@ int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32 #endif } -int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { // decompress lossy - return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); + return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); } else { #endif // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressDoubleImp(buffer, nelements, output); + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressDoubleImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressDoubleImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1740,37 +1739,37 @@ int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, cons } // Binary ===================================================== -int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - return tsCompressStringImp(input, inputSize, output, outputSize); +int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + return tsCompressStringImp(pIn, nIn, pOut, nOut); } -int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - return tsDecompressStringImp(input, compressedSize, output, outputSize); +int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + return tsDecompressStringImp(pIn, nIn, pOut, nOut); } // Bool ===================================================== -int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressBoolImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressBoolImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressBoolImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressBoolImp(buffer, nelements, output); +int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressBoolImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressBoolImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1778,27 +1777,26 @@ int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const } // Tinyint ===================================================== -int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_TINYINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); +int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_TINYINT); } else { assert(0); return -1; @@ -1806,27 +1804,26 @@ int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, con } // Smallint ===================================================== -int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_SMALLINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); +int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); } else { assert(0); return -1; @@ -1834,26 +1831,26 @@ int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, co } // Int ===================================================== -int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_INT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); +int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_INT); } else { assert(0); return -1; @@ -1861,26 +1858,26 @@ int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const i } // Bigint ===================================================== -int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_BIGINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); +int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_BIGINT); } else { assert(0); return -1; From fb45b8d5773049d629506b0f0b9dce9c07b954a0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 13:39:32 +0800 Subject: [PATCH 03/59] refact code --- include/util/tcompression.h | 14 +++++ source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 63 ++-------------------- source/util/src/tcompression.c | 1 - 4 files changed, 20 insertions(+), 62 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index b620615ab1..a884c4753f 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -83,6 +83,9 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int #endif +/************************************************************************* + * REGULAR COMPRESSION + *************************************************************************/ int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf); int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, @@ -120,6 +123,17 @@ int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf); +/************************************************************************* + * STREAM COMPRESSION + *************************************************************************/ +typedef struct SCompressor SCompressor; + +int32_t tCompressorCreate(SCompressor **ppCmprsor); +int32_t tCompressorDestroy(SCompressor *pCmprsor); +int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc); +int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData); +int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 817089f237..c2a31679d0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -285,8 +285,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char* id); -void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char* id); +int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char *id); +void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char *id); // tsdbMerge.c ============================================================================================== int32_t tsdbMerge(STsdb *pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 3bd71f0ea6..0a42edc94c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -15,70 +15,15 @@ #include "tsdb.h" -typedef struct SDiskColBuilder SDiskColBuilder; -struct SDiskColBuilder { - uint8_t flags; - uint8_t *pBitMap; - int32_t *aOffset; - int32_t nData; - uint8_t *pData; -}; +typedef struct SDiskData SDiskData; -int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - // TODO - return code; -} - -// ================================================================ -typedef struct SDiskDataBuilder SDiskDataBuilder; -struct SDiskDataBuilder { +struct SDiskData { SDiskDataHdr hdr; - SArray *aBlockCol; // SArray + /* data */ }; -int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { +int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow) { int32_t code = 0; // TODO return code; } - -void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { - // TODO -} - -void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) { - pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, // - .fmtVer = 0, - .suid = suid, - .uid = uid, - .cmprAlg = cmprAlg}; -} - -void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) { - // TODO -} - -int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { - int32_t code = 0; - - // uid (todo) - - // version (todo) - - // TSKEY (todo) - - SRowIter iter = {0}; - tRowIterInit(&iter, pRow, pTSchema); - - for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) { - } - - return code; -} - -int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) { - int32_t code = 0; - // TODO - return code; -} \ No newline at end of file diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 30b72f6fbe..3d4df28440 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1000,7 +1000,6 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co * STREAM COMPRESSION *************************************************************************/ #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) -typedef struct SCompressor SCompressor; static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); From 556a6fcd7fc27f9da69fb590add4b53b1ce55efb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 14:49:34 +0800 Subject: [PATCH 04/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 207 ++++++++++++++++++++- 1 file changed, 203 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 0a42edc94c..b7b03e8288 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -16,14 +16,213 @@ #include "tsdb.h" typedef struct SDiskData SDiskData; +typedef struct SDiskCol SDiskCol; -struct SDiskData { - SDiskDataHdr hdr; - /* data */ +struct SDiskCol { + int16_t cid; + int8_t type; + int8_t flag; + int32_t nVal; + SCompressor *pBitC; + SCompressor *pOffC; + SCompressor *pValC; }; -int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow) { +// SDiskCol ================================================ +static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { + int32_t code = 0; + + pDiskCol->cid = cid; + pDiskCol->type = type; + pDiskCol->flag = 0; + pDiskCol->nVal = 0; + + tCompressorReset(pDiskCol->pBitC, TSDB_DATA_TYPE_TINYINT, cmprAlg, 1); + tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg, 1); + tCompressorReset(pDiskCol->pValC, type, cmprAlg, 1); + + return code; +} + +static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + pDiskCol->flag = HAS_NONE; + } else if (pColVal->isNull) { + pDiskCol->flag = HAS_NULL; + } else { + pDiskCol->flag = HAS_VALUE; + code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + if (code) goto _exit; + } + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (!pColVal->isNone) { + } + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (!pColVal->isNull) { + } + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + } else if (pColVal->isNull) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone || pColVal->isNull) { + } else { + } + + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNull) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + } else if (pColVal->isNull) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { + tDiskColAddVal0, // 0 + tDiskColAddVal1, // HAS_NONE + tDiskColAddVal2, // HAS_NULL + tDiskColAddVal3, // HAS_NULL|HAS_NONE + tDiskColAddVal4, // HAS_VALUE + tDiskColAddVal5, // HAS_VALUE|HAS_NONE + tDiskColAddVal6, // HAS_VALUE|HAS_NULL + tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE +}; + +// SDiskData ================================================ +struct SDiskData { + int64_t suid; + int64_t uid; + uint8_t cmprAlg; + SCompressor *pUidC; + SCompressor *pVerC; + SCompressor *pKeyC; + int32_t nDiskCol; + SArray *aDiskCol; +}; + +int32_t tDiskDataCreate(SDiskData *pDiskData) { int32_t code = 0; // TODO return code; } + +int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tDiskDataReset(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { + int32_t code = 0; + + pDiskData->suid = pId->suid; + pDiskData->uid = pId->uid; + pDiskData->cmprAlg = cmprAlg; + + for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { + } + + return code; +} + +int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { + int32_t code = 0; + + ASSERT(pId->suid == pDiskData->suid); + + // uid + code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); + if (code) goto _exit; + + // version + int64_t version = TSDBROW_VERSION(pRow); + code = tCompress(pDiskData->pVerC, &version, sizeof(int64_t)); + if (code) goto _exit; + + // TSKEY + TSKEY ts = TSDBROW_TS(pRow); + code = tCompress(pDiskData->pVerC, &ts, sizeof(int64_t)); + if (code) goto _exit; + + SRowIter iter = {0}; + tRowIterInit(&iter, pRow, pTSchema); + + SColVal *pColVal = tRowIterNext(&iter); + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + while (pColVal && pColVal->cid < pDiskCol->cid) { + pColVal = tRowIterNext(&iter); + } + + if (pColVal == NULL || pColVal->cid > pDiskCol->cid) { + code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &COL_VAL_NONE(pDiskCol->cid, pDiskCol->type)); + if (code) goto _exit; + } else { + code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, pColVal); + if (code) goto _exit; + pColVal = tRowIterNext(&iter); + } + } + +_exit: + return code; +} From d197f5019c6bba6c52d8ada06452fac431842f6a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 16:18:12 +0800 Subject: [PATCH 05/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 163 +++++++++++++++++++-- 1 file changed, 154 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b7b03e8288..c688a64580 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -23,7 +23,9 @@ struct SDiskCol { int8_t type; int8_t flag; int32_t nVal; + uint8_t bit; SCompressor *pBitC; + int32_t offset; SCompressor *pOffC; SCompressor *pValC; }; @@ -44,7 +46,21 @@ static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8 return code; } -static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddValue(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColVal->type)) { + code = tCompress(pDiskCol->pOffC, &pDiskCol->offset, sizeof(int32_t)); + if (code) goto _exit; + pDiskCol->offset += pColVal->value.nData; + } + code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + if (code) goto _exit; + +_exit: + return code; +} +static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { // 0 int32_t code = 0; if (pColVal->isNone) { @@ -53,7 +69,7 @@ static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { pDiskCol->flag = HAS_NULL; } else { pDiskCol->flag = HAS_VALUE; - code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; } pDiskCol->nVal++; @@ -61,42 +77,144 @@ static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { _exit: return code; } -static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NONE int32_t code = 0; if (!pColVal->isNone) { + // bit map + pDiskCol->bit = 0; + for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } + SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); + if ((pDiskCol->nVal & 0x3) == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + pDiskCol->bit = 0; + } + + // value + if (pColVal->isNull) { + pDiskCol->flag |= HAS_NULL; + } else { + pDiskCol->flag |= HAS_VALUE; + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + code = tDiskColAddValue(pDiskCol, &cv); + if (code) goto _exit; + } + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; + } } pDiskCol->nVal++; _exit: return code; } -static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL int32_t code = 0; if (!pColVal->isNull) { + if (pColVal->isNone) { + pDiskCol->flag |= HAS_NONE; + + pDiskCol->bit = 1; + for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } + SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 0); + if ((pDiskCol->nVal & 0x3) == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + pDiskCol->bit = 0; + } + } else { + pDiskCol->flag |= HAS_VALUE; + + pDiskCol->bit = 0; + for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } + SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); + if ((pDiskCol->nVal & 0x3) == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + pDiskCol->bit = 0; + } + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + code = tDiskColAddValue(pDiskCol, &cv); + if (code) goto _exit; + } + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; + } } pDiskCol->nVal++; _exit: return code; } -static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; + uint8_t mod8 = (pDiskCol->nVal & 0x3); if (pColVal->isNone) { + SET_BIT1(&pDiskCol->bit, mod8, 0); + if (mod8 == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } } else if (pColVal->isNull) { + SET_BIT1(&pDiskCol->bit, mod8, 1); + if (mod8 == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } } else { + pDiskCol->flag |= HAS_VALUE; + + // convert from bit1 to bit2 and add a 2 (todo) + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + code = tDiskColAddValue(pDiskCol, &cv); + if (code) goto _exit; + } + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; } pDiskCol->nVal++; +_exit: return code; } -static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE int32_t code = 0; if (pColVal->isNone || pColVal->isNull) { + if (pColVal->isNone) { + pDiskCol->flag |= HAS_NONE; + } else { + pDiskCol->flag |= HAS_NULL; + } + + // set bitmap (todo) + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; } else { + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; } pDiskCol->nVal++; @@ -104,35 +222,62 @@ static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { _exit: return code; } -static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NONE int32_t code = 0; if (pColVal->isNull) { + pDiskCol->flag |= HAS_NULL; + + // convert bit1 to bit2 } else { + if (pColVal->isNone) { + // SET_BIT1(0); + } else { + // SET_BIT1(1); + } } + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; pDiskCol->nVal++; +_exit: return code; } -static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL int32_t code = 0; if (pColVal->isNone) { + pDiskCol->flag |= HAS_NONE; + // bit1 to bit2 } else { + if (pColVal->isNull) { + // SET_BIT1(0) + } else { + // SET_BIT1(1) + } } + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; pDiskCol->nVal++; +_exit: return code; } -static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE int32_t code = 0; if (pColVal->isNone) { + // set_bit2(0); } else if (pColVal->isNull) { + // set_bit2(1); } else { + // set_bit2(2); } + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; pDiskCol->nVal++; +_exit: return code; } static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { From 750a1a8d8679375e4fad3aca2091d61697da57d5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 18:56:36 +0800 Subject: [PATCH 06/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 139 ++++++++++++--------- 1 file changed, 80 insertions(+), 59 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index c688a64580..1266efcf8e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -23,8 +23,7 @@ struct SDiskCol { int8_t type; int8_t flag; int32_t nVal; - uint8_t bit; - SCompressor *pBitC; + uint8_t *pBitMap; int32_t offset; SCompressor *pOffC; SCompressor *pValC; @@ -39,7 +38,6 @@ static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8 pDiskCol->flag = 0; pDiskCol->nVal = 0; - tCompressorReset(pDiskCol->pBitC, TSDB_DATA_TYPE_TINYINT, cmprAlg, 1); tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg, 1); tCompressorReset(pDiskCol->pValC, type, cmprAlg, 1); @@ -82,17 +80,13 @@ static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_N if (!pColVal->isNone) { // bit map - pDiskCol->bit = 0; - for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } - SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); - if ((pDiskCol->nVal & 0x3) == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - pDiskCol->bit = 0; - } + int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + + code = tRealloc(&pDiskCol->pBitMap, nBit); + if (code) goto _exit; + + memset(pDiskCol->pBitMap, 0, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); // value if (pColVal->isNull) { @@ -119,34 +113,20 @@ static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_N int32_t code = 0; if (!pColVal->isNull) { + int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + code = tRealloc(&pDiskCol->pBitMap, nBit); + if (code) goto _exit; + if (pColVal->isNone) { pDiskCol->flag |= HAS_NONE; - pDiskCol->bit = 1; - for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } - SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 0); - if ((pDiskCol->nVal & 0x3) == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - pDiskCol->bit = 0; - } + memset(pDiskCol->pBitMap, 255, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else { pDiskCol->flag |= HAS_VALUE; - pDiskCol->bit = 0; - for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } - SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); - if ((pDiskCol->nVal & 0x3) == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - pDiskCol->bit = 0; - } + memset(pDiskCol->pBitMap, 0, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { @@ -166,23 +146,30 @@ _exit: static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; - uint8_t mod8 = (pDiskCol->nVal & 0x3); if (pColVal->isNone) { - SET_BIT1(&pDiskCol->bit, mod8, 0); - if (mod8 == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else if (pColVal->isNull) { - SET_BIT1(&pDiskCol->bit, mod8, 1); - if (mod8 == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); } else { pDiskCol->flag |= HAS_VALUE; - // convert from bit1 to bit2 and add a 2 (todo) + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pDiskCol->nVal, 2); + + tFree(pDiskCol->pBitMap); + pDiskCol->pBitMap = pBitMap; SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { @@ -208,7 +195,12 @@ static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V pDiskCol->flag |= HAS_NULL; } - // set bitmap (todo) + int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + code = tRealloc(&pDiskCol->pBitMap, nBit); + if (code) goto _exit; + + memset(pDiskCol->pBitMap, 255, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; @@ -216,7 +208,6 @@ static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; _exit: @@ -228,12 +219,25 @@ static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V if (pColVal->isNull) { pDiskCol->flag |= HAS_NULL; - // convert bit1 to bit2 + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pDiskCol->nVal, 1); + + tFree(pDiskCol->pBitMap); + pDiskCol->pBitMap = pBitMap; } else { + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + if (pColVal->isNone) { - // SET_BIT1(0); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else { - // SET_BIT1(1); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); } } code = tDiskColAddValue(pDiskCol, pColVal); @@ -248,12 +252,26 @@ static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V if (pColVal->isNone) { pDiskCol->flag |= HAS_NONE; - // bit1 to bit2 + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pDiskCol->nVal, 0); + + tFree(pDiskCol->pBitMap); + pDiskCol->pBitMap = pBitMap; } else { + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + if (pColVal->isNull) { - // SET_BIT1(0) + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else { - // SET_BIT1(1) + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); } } code = tDiskColAddValue(pDiskCol, pColVal); @@ -266,12 +284,15 @@ _exit: static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE int32_t code = 0; + code = tRealloc(&pDiskCol->pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + if (pColVal->isNone) { - // set_bit2(0); + SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else if (pColVal->isNull) { - // set_bit2(1); + SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 1); } else { - // set_bit2(2); + SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 2); } code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; From b4a43079fb04d2e9f71301a5ab7c1e184841345f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 11:03:08 +0800 Subject: [PATCH 07/59] more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 110 ++++++++++++++++++--- source/util/src/tcompression.c | 16 ++- 3 files changed, 102 insertions(+), 26 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index a884c4753f..fd01bcf1a3 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -130,7 +130,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc); +int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 1266efcf8e..c20a598b6c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -22,6 +22,7 @@ struct SDiskCol { int16_t cid; int8_t type; int8_t flag; + uint8_t cmprAlg; int32_t nVal; uint8_t *pBitMap; int32_t offset; @@ -30,16 +31,41 @@ struct SDiskCol { }; // SDiskCol ================================================ -static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { +static int32_t tDiskColInit(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; pDiskCol->cid = cid; pDiskCol->type = type; pDiskCol->flag = 0; + pDiskCol->cmprAlg = cmprAlg; pDiskCol->nVal = 0; + pDiskCol->offset = 0; - tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg, 1); - tCompressorReset(pDiskCol->pValC, type, cmprAlg, 1); + if (IS_VAR_DATA_TYPE(type)) { + if (pDiskCol->pOffC == NULL) { + code = tCompressorCreate(&pDiskCol->pOffC); + if (code) return code; + } + code = tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + if (code) return code; + } + + if (pDiskCol->pValC == NULL) { + code = tCompressorCreate(&pDiskCol->pValC); + if (code) return code; + } + code = tCompressorReset(pDiskCol->pValC, type, cmprAlg); + if (code) return code; + + return code; +} + +static int32_t tDiskColClear(SDiskCol *pDiskCol) { + int32_t code = 0; + + tFree(pDiskCol->pBitMap); + if (pDiskCol->pOffC) tCompressorDestroy(pDiskCol->pOffC); + if (pDiskCol->pValC) tCompressorDestroy(pDiskCol->pValC); return code; } @@ -324,26 +350,78 @@ struct SDiskData { SArray *aDiskCol; }; -int32_t tDiskDataCreate(SDiskData *pDiskData) { - int32_t code = 0; - // TODO - return code; -} - -int32_t tDiskDataDestroy(SDiskData *pDiskData) { - int32_t code = 0; - // TODO - return code; -} - -int32_t tDiskDataReset(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { +int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { int32_t code = 0; pDiskData->suid = pId->suid; pDiskData->uid = pId->uid; pDiskData->cmprAlg = cmprAlg; + if (pDiskData->pUidC == NULL) { + code = tCompressorCreate(&pDiskData->pUidC); + if (code) return code; + } + code = tCompressorReset(pDiskData->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (code) return code; + + if (pDiskData->pVerC == NULL) { + code = tCompressorCreate(&pDiskData->pVerC); + if (code) return code; + } + code = tCompressorReset(pDiskData->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (code) return code; + + if (pDiskData->pKeyC == NULL) { + code = tCompressorCreate(&pDiskData->pKeyC); + if (code) return code; + } + code = tCompressorReset(pDiskData->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + if (code) return code; + + if (pDiskData->aDiskCol == NULL) { + pDiskData->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); + if (pDiskData->aDiskCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + pDiskData->nDiskCol = 0; for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { + STColumn *pTColumn = &pTSchema->columns[iCol]; + + if (pDiskData->nDiskCol >= taosArrayGetSize(pDiskData->aDiskCol)) { + SDiskCol dc = (SDiskCol){0}; + if (taosArrayPush(pDiskData->aDiskCol, &dc) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, pDiskData->nDiskCol); + + code = tDiskColInit(pDiskCol, pTColumn->colId, pTColumn->type, cmprAlg); + if (code) return code; + + pDiskData->nDiskCol++; + } + + return code; +} + +int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + + if (pDiskData->pUidC) tCompressorDestroy(pDiskData->pUidC); + if (pDiskData->pVerC) tCompressorDestroy(pDiskData->pVerC); + if (pDiskData->pKeyC) tCompressorDestroy(pDiskData->pKeyC); + + if (pDiskData->aDiskCol) { + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + tDiskColClear(pDiskCol); + } + taosArrayDestroy(pDiskData->aDiskCol); } return code; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 3d4df28440..5d6e6d12ac 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1508,24 +1508,22 @@ _exit: int32_t tCompressorDestroy(SCompressor *pCmprsor) { int32_t code = 0; - if (pCmprsor) { - int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); - for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { - tFree(pCmprsor->aBuf[iBuf]); - } - - taosMemoryFree(pCmprsor); + int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); + for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { + tFree(pCmprsor->aBuf[iBuf]); } + taosMemoryFree(pCmprsor); + return code; } -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc) { +int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; pCmprsor->type = type; pCmprsor->cmprAlg = cmprAlg; - pCmprsor->autoAlloc = autoAlloc; + pCmprsor->autoAlloc = 1; pCmprsor->nVal = 0; switch (type) { From 8ffc7e9f3e42c3242f746d1d7a51e1f475718189 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 11:28:20 +0800 Subject: [PATCH 08/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 8 ++++++-- source/dnode/vnode/src/tsdb/tsdbUtil.c | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index c20a598b6c..3904012988 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -342,6 +342,7 @@ static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { struct SDiskData { int64_t suid; int64_t uid; + int32_t nRow; uint8_t cmprAlg; SCompressor *pUidC; SCompressor *pVerC; @@ -355,6 +356,7 @@ int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, ui pDiskData->suid = pId->suid; pDiskData->uid = pId->uid; + pDiskData->nRow = 0; pDiskData->cmprAlg = cmprAlg; if (pDiskData->pUidC == NULL) { @@ -443,7 +445,7 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, // TSKEY TSKEY ts = TSDBROW_TS(pRow); - code = tCompress(pDiskData->pVerC, &ts, sizeof(int64_t)); + code = tCompress(pDiskData->pKeyC, &ts, sizeof(int64_t)); if (code) goto _exit; SRowIter iter = {0}; @@ -458,7 +460,8 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, } if (pColVal == NULL || pColVal->cid > pDiskCol->cid) { - code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &COL_VAL_NONE(pDiskCol->cid, pDiskCol->type)); + SColVal cv = COL_VAL_NONE(pDiskCol->cid, pDiskCol->type); + code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &cv); if (code) goto _exit; } else { code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, pColVal); @@ -466,6 +469,7 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, pColVal = tRowIterNext(&iter); } } + pDiskData->nRow++; _exit: return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 64c150484b..4e6e1f2f7f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1082,9 +1082,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS pColVal = tRowIterNext(&rIter); } } - -_exit: pBlockData->nRow++; + return code; _err: From ce9afbae19c86ae3707cd95226c4bd583802edff Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 14:02:52 +0800 Subject: [PATCH 09/59] more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 101 ++++++++++++++++++++- source/util/src/tcompression.c | 2 +- 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index fd01bcf1a3..8d7c607e6b 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -131,7 +131,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData); +int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 3904012988..a5104dcfbd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -349,6 +349,7 @@ struct SDiskData { SCompressor *pKeyC; int32_t nDiskCol; SArray *aDiskCol; + uint8_t *aBuf[2]; }; int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { @@ -425,6 +426,9 @@ int32_t tDiskDataDestroy(SDiskData *pDiskData) { } taosArrayDestroy(pDiskData->aDiskCol); } + for (int32_t iBuf = 0; iBuf < sizeof(pDiskData->aBuf) / sizeof(pDiskData->aBuf[0]); iBuf++) { + tFree(pDiskData->aBuf[iBuf]); + } return code; } @@ -435,8 +439,17 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, ASSERT(pId->suid == pDiskData->suid); // uid - code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); - if (code) goto _exit; + if (pDiskData->uid && pDiskData->uid != pId->uid) { + for (int32_t iRow = 0; iRow < pDiskData->nRow; iRow++) { + code = tCompress(pDiskData->pUidC, &pDiskData->uid, sizeof(int64_t)); + if (code) goto _exit; + } + pDiskData->uid = 0; + } + if (pDiskData->uid == 0) { + code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); + if (code) goto _exit; + } // version int64_t version = TSDBROW_VERSION(pRow); @@ -474,3 +487,87 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, _exit: return code; } + +int32_t tDiskDataToBinary(SDiskData *pDiskData, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + ASSERT(pDiskData->nRow); + + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pDiskData->suid, + .uid = pDiskData->uid, + .szUid = 0, + .szVer = 0, + .szKey = 0, + .szBlkCol = 0, + .nRow = pDiskData->nRow, + .cmprAlg = pDiskData->cmprAlg}; + + // UID + const uint8_t *pUid = NULL; + if (pDiskData->uid == 0) { + code = tCompGen(pDiskData->pUidC, &pUid, &hdr.szUid); + if (code) return code; + } + + // VERSION + const uint8_t *pVer = NULL; + code = tCompGen(pDiskData->pVerC, &pVer, &hdr.szVer); + if (code) return code; + + // TSKEY + const uint8_t *pKey = NULL; + code = tCompGen(pDiskData->pKeyC, &pKey, &hdr.szKey); + if (code) return code; + + int32_t offset = 0; + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (pDiskCol->flag == HAS_NONE) continue; + + // code = tDiskColToBinary(pDiskCol, ); + // if (code) return code; + + SBlockCol bCol = {.cid = pDiskCol->cid, + .type = pDiskCol->type, + // .smaOn = , + .flag = pDiskCol->flag, + // .szOrigin = + // .szBitmap = + // .szOffset = + // .szValue = + .offset = offset}; + + hdr.szBlkCol += tPutBlockCol(NULL, &bCol); + offset = offset + bCol.szBitmap + bCol.szOffset + bCol.szValue; + } + + *nData = tPutDiskDataHdr(NULL, &hdr) + hdr.szUid + hdr.szVer + hdr.szKey + hdr.szBlkCol + offset; + code = tRealloc(&pDiskData->aBuf[0], *nData); + if (code) return code; + *ppData = pDiskData->aBuf[0]; + + int32_t n = 0; + n += tPutDiskDataHdr(pDiskData->aBuf[0] + n, &hdr); + if (hdr.szUid) { + memcpy(pDiskData->aBuf[0] + n, pUid, hdr.szUid); + n += hdr.szUid; + } + memcpy(pDiskData->aBuf[0] + n, pVer, hdr.szVer); + n += hdr.szVer; + memcpy(pDiskData->aBuf[0] + n, pKey, hdr.szKey); + n += hdr.szKey; + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + n += tPutBlockCol(pDiskData->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); + } + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + // memcpy(pDiskData->aBuf[0] + n, NULL, ); + // n += 0; + } + + return code; +} diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 5d6e6d12ac..cd9ff5e325 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1574,7 +1574,7 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { return code; } -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) { +int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; if (pCmprsor->nVal == 0) { From f51c86430a381a9bae6ec1c80726b3a0c09feb22 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 14:50:14 +0800 Subject: [PATCH 10/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 198 +++++++++++++-------- 1 file changed, 124 insertions(+), 74 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index a5104dcfbd..a0c1bb8934 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -15,8 +15,9 @@ #include "tsdb.h" -typedef struct SDiskData SDiskData; -typedef struct SDiskCol SDiskCol; +typedef struct SDiskDataBuilder SDiskDataBuilder; +typedef struct SDiskCol SDiskCol; +typedef struct SDiskData SDiskData; struct SDiskCol { int16_t cid; @@ -70,6 +71,34 @@ static int32_t tDiskColClear(SDiskCol *pDiskCol) { return code; } +static int32_t tDiskColToBinary(SDiskCol *pDiskCol, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + ASSERT(pDiskCol->flag && pDiskCol->flag != HAS_NONE); + + if (pDiskCol->flag == HAS_NULL) { + return code; + } + + // bitmap (todo) + if (pDiskCol->flag != HAS_VALUE) { + } + + // offset (todo) + if (IS_VAR_DATA_TYPE(pDiskCol->type)) { + code = tCompGen(pDiskCol->pOffC, NULL /* todo */, NULL /* todo */); + if (code) return code; + } + + // value (todo) + if (pDiskCol->flag != (HAS_NULL | HAS_NONE)) { + code = tCompGen(pDiskCol->pValC, NULL /* todo */, NULL /* todo */); + if (code) return code; + } + + return code; +} + static int32_t tDiskColAddValue(SDiskCol *pDiskCol, SColVal *pColVal) { int32_t code = 0; @@ -338,8 +367,8 @@ static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE }; -// SDiskData ================================================ -struct SDiskData { +// SDiskDataBuilder ================================================ +struct SDiskDataBuilder { int64_t suid; int64_t uid; int32_t nRow; @@ -352,121 +381,121 @@ struct SDiskData { uint8_t *aBuf[2]; }; -int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { int32_t code = 0; - pDiskData->suid = pId->suid; - pDiskData->uid = pId->uid; - pDiskData->nRow = 0; - pDiskData->cmprAlg = cmprAlg; + pBuilder->suid = pId->suid; + pBuilder->uid = pId->uid; + pBuilder->nRow = 0; + pBuilder->cmprAlg = cmprAlg; - if (pDiskData->pUidC == NULL) { - code = tCompressorCreate(&pDiskData->pUidC); + if (pBuilder->pUidC == NULL) { + code = tCompressorCreate(&pBuilder->pUidC); if (code) return code; } - code = tCompressorReset(pDiskData->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressorReset(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pDiskData->pVerC == NULL) { - code = tCompressorCreate(&pDiskData->pVerC); + if (pBuilder->pVerC == NULL) { + code = tCompressorCreate(&pBuilder->pVerC); if (code) return code; } - code = tCompressorReset(pDiskData->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressorReset(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pDiskData->pKeyC == NULL) { - code = tCompressorCreate(&pDiskData->pKeyC); + if (pBuilder->pKeyC == NULL) { + code = tCompressorCreate(&pBuilder->pKeyC); if (code) return code; } - code = tCompressorReset(pDiskData->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; - if (pDiskData->aDiskCol == NULL) { - pDiskData->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); - if (pDiskData->aDiskCol == NULL) { + if (pBuilder->aDiskCol == NULL) { + pBuilder->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); + if (pBuilder->aDiskCol == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - pDiskData->nDiskCol = 0; + pBuilder->nDiskCol = 0; for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { STColumn *pTColumn = &pTSchema->columns[iCol]; - if (pDiskData->nDiskCol >= taosArrayGetSize(pDiskData->aDiskCol)) { + if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskCol)) { SDiskCol dc = (SDiskCol){0}; - if (taosArrayPush(pDiskData->aDiskCol, &dc) == NULL) { + if (taosArrayPush(pBuilder->aDiskCol, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, pDiskData->nDiskCol); + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, pBuilder->nDiskCol); code = tDiskColInit(pDiskCol, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; - pDiskData->nDiskCol++; + pBuilder->nDiskCol++; } return code; } -int32_t tDiskDataDestroy(SDiskData *pDiskData) { +int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { int32_t code = 0; - if (pDiskData->pUidC) tCompressorDestroy(pDiskData->pUidC); - if (pDiskData->pVerC) tCompressorDestroy(pDiskData->pVerC); - if (pDiskData->pKeyC) tCompressorDestroy(pDiskData->pKeyC); + if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); + if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); + if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - if (pDiskData->aDiskCol) { - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + if (pBuilder->aDiskCol) { + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pBuilder->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); tDiskColClear(pDiskCol); } - taosArrayDestroy(pDiskData->aDiskCol); + taosArrayDestroy(pBuilder->aDiskCol); } - for (int32_t iBuf = 0; iBuf < sizeof(pDiskData->aBuf) / sizeof(pDiskData->aBuf[0]); iBuf++) { - tFree(pDiskData->aBuf[iBuf]); + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); } return code; } -int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { +int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { int32_t code = 0; - ASSERT(pId->suid == pDiskData->suid); + ASSERT(pId->suid == pBuilder->suid); // uid - if (pDiskData->uid && pDiskData->uid != pId->uid) { - for (int32_t iRow = 0; iRow < pDiskData->nRow; iRow++) { - code = tCompress(pDiskData->pUidC, &pDiskData->uid, sizeof(int64_t)); + if (pBuilder->uid && pBuilder->uid != pId->uid) { + for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { + code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); if (code) goto _exit; } - pDiskData->uid = 0; + pBuilder->uid = 0; } - if (pDiskData->uid == 0) { - code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); + if (pBuilder->uid == 0) { + code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); if (code) goto _exit; } // version int64_t version = TSDBROW_VERSION(pRow); - code = tCompress(pDiskData->pVerC, &version, sizeof(int64_t)); + code = tCompress(pBuilder->pVerC, &version, sizeof(int64_t)); if (code) goto _exit; // TSKEY TSKEY ts = TSDBROW_TS(pRow); - code = tCompress(pDiskData->pKeyC, &ts, sizeof(int64_t)); + code = tCompress(pBuilder->pKeyC, &ts, sizeof(int64_t)); if (code) goto _exit; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); while (pColVal && pColVal->cid < pDiskCol->cid) { pColVal = tRowIterNext(&iter); @@ -482,53 +511,53 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, pColVal = tRowIterNext(&iter); } } - pDiskData->nRow++; + pBuilder->nRow++; _exit: return code; } -int32_t tDiskDataToBinary(SDiskData *pDiskData, const uint8_t **ppData, int32_t *nData) { +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { int32_t code = 0; - ASSERT(pDiskData->nRow); + ASSERT(pBuilder->nRow); SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .fmtVer = 0, - .suid = pDiskData->suid, - .uid = pDiskData->uid, + .suid = pBuilder->suid, + .uid = pBuilder->uid, .szUid = 0, .szVer = 0, .szKey = 0, .szBlkCol = 0, - .nRow = pDiskData->nRow, - .cmprAlg = pDiskData->cmprAlg}; + .nRow = pBuilder->nRow, + .cmprAlg = pBuilder->cmprAlg}; // UID const uint8_t *pUid = NULL; - if (pDiskData->uid == 0) { - code = tCompGen(pDiskData->pUidC, &pUid, &hdr.szUid); + if (pBuilder->uid == 0) { + code = tCompGen(pBuilder->pUidC, &pUid, &hdr.szUid); if (code) return code; } // VERSION const uint8_t *pVer = NULL; - code = tCompGen(pDiskData->pVerC, &pVer, &hdr.szVer); + code = tCompGen(pBuilder->pVerC, &pVer, &hdr.szVer); if (code) return code; // TSKEY const uint8_t *pKey = NULL; - code = tCompGen(pDiskData->pKeyC, &pKey, &hdr.szKey); + code = tCompGen(pBuilder->pKeyC, &pKey, &hdr.szKey); if (code) return code; int32_t offset = 0; - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); if (pDiskCol->flag == HAS_NONE) continue; - // code = tDiskColToBinary(pDiskCol, ); - // if (code) return code; + code = tDiskColToBinary(pDiskCol, NULL, NULL); + if (code) return code; SBlockCol bCol = {.cid = pDiskCol->cid, .type = pDiskCol->type, @@ -544,30 +573,51 @@ int32_t tDiskDataToBinary(SDiskData *pDiskData, const uint8_t **ppData, int32_t offset = offset + bCol.szBitmap + bCol.szOffset + bCol.szValue; } +#if 0 *nData = tPutDiskDataHdr(NULL, &hdr) + hdr.szUid + hdr.szVer + hdr.szKey + hdr.szBlkCol + offset; - code = tRealloc(&pDiskData->aBuf[0], *nData); + code = tRealloc(&pBuilder->aBuf[0], *nData); if (code) return code; - *ppData = pDiskData->aBuf[0]; + *ppData = pBuilder->aBuf[0]; int32_t n = 0; - n += tPutDiskDataHdr(pDiskData->aBuf[0] + n, &hdr); + n += tPutDiskDataHdr(pBuilder->aBuf[0] + n, &hdr); if (hdr.szUid) { - memcpy(pDiskData->aBuf[0] + n, pUid, hdr.szUid); + memcpy(pBuilder->aBuf[0] + n, pUid, hdr.szUid); n += hdr.szUid; } - memcpy(pDiskData->aBuf[0] + n, pVer, hdr.szVer); + memcpy(pBuilder->aBuf[0] + n, pVer, hdr.szVer); n += hdr.szVer; - memcpy(pDiskData->aBuf[0] + n, pKey, hdr.szKey); + memcpy(pBuilder->aBuf[0] + n, pKey, hdr.szKey); n += hdr.szKey; - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - n += tPutBlockCol(pDiskData->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + n += tPutBlockCol(pBuilder->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); } - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); // memcpy(pDiskData->aBuf[0] + n, NULL, ); // n += 0; } +#endif + + return code; +} + +// SDiskData ================================================ +struct SDiskData { + SDiskDataHdr hdr; + const uint8_t *pUid; + const uint8_t *pVer; + const uint8_t *pKey; + SArray *aBlockCol; + SArray *aColData; +}; + +int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + + if (pDiskData->aBlockCol) taosArrayDestroy(pDiskData->aBlockCol); + if (pDiskData->aColData) taosArrayDestroy(pDiskData->aColData); return code; } From e12ac1dd6f9a0dd40bb7a8d1db3b649525803350 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 15:00:32 +0800 Subject: [PATCH 11/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 309 +++++++++++---------- 1 file changed, 155 insertions(+), 154 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index a0c1bb8934..a514bddabb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -16,10 +16,10 @@ #include "tsdb.h" typedef struct SDiskDataBuilder SDiskDataBuilder; -typedef struct SDiskCol SDiskCol; +typedef struct SDiskColBuilder SDiskColBuilder; typedef struct SDiskData SDiskData; -struct SDiskCol { +struct SDiskColBuilder { int16_t cid; int8_t type; int8_t flag; @@ -31,332 +31,332 @@ struct SDiskCol { SCompressor *pValC; }; -// SDiskCol ================================================ -static int32_t tDiskColInit(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { +// SDiskColBuilder ================================================ +static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; - pDiskCol->cid = cid; - pDiskCol->type = type; - pDiskCol->flag = 0; - pDiskCol->cmprAlg = cmprAlg; - pDiskCol->nVal = 0; - pDiskCol->offset = 0; + pBuilder->cid = cid; + pBuilder->type = type; + pBuilder->flag = 0; + pBuilder->cmprAlg = cmprAlg; + pBuilder->nVal = 0; + pBuilder->offset = 0; if (IS_VAR_DATA_TYPE(type)) { - if (pDiskCol->pOffC == NULL) { - code = tCompressorCreate(&pDiskCol->pOffC); + if (pBuilder->pOffC == NULL) { + code = tCompressorCreate(&pBuilder->pOffC); if (code) return code; } - code = tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + code = tCompressorReset(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } - if (pDiskCol->pValC == NULL) { - code = tCompressorCreate(&pDiskCol->pValC); + if (pBuilder->pValC == NULL) { + code = tCompressorCreate(&pBuilder->pValC); if (code) return code; } - code = tCompressorReset(pDiskCol->pValC, type, cmprAlg); + code = tCompressorReset(pBuilder->pValC, type, cmprAlg); if (code) return code; return code; } -static int32_t tDiskColClear(SDiskCol *pDiskCol) { +static int32_t tDiskColClear(SDiskColBuilder *pBuilder) { int32_t code = 0; - tFree(pDiskCol->pBitMap); - if (pDiskCol->pOffC) tCompressorDestroy(pDiskCol->pOffC); - if (pDiskCol->pValC) tCompressorDestroy(pDiskCol->pValC); + tFree(pBuilder->pBitMap); + if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); + if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); return code; } -static int32_t tDiskColToBinary(SDiskCol *pDiskCol, const uint8_t **ppData, int32_t *nData) { +static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SBlockCol *pBlockCol, const uint8_t **ppData) { int32_t code = 0; - ASSERT(pDiskCol->flag && pDiskCol->flag != HAS_NONE); + ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); - if (pDiskCol->flag == HAS_NULL) { + if (pBuilder->flag == HAS_NULL) { return code; } // bitmap (todo) - if (pDiskCol->flag != HAS_VALUE) { + if (pBuilder->flag != HAS_VALUE) { } // offset (todo) - if (IS_VAR_DATA_TYPE(pDiskCol->type)) { - code = tCompGen(pDiskCol->pOffC, NULL /* todo */, NULL /* todo */); + if (IS_VAR_DATA_TYPE(pBuilder->type)) { + code = tCompGen(pBuilder->pOffC, NULL /* todo */, NULL /* todo */); if (code) return code; } // value (todo) - if (pDiskCol->flag != (HAS_NULL | HAS_NONE)) { - code = tCompGen(pDiskCol->pValC, NULL /* todo */, NULL /* todo */); + if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { + code = tCompGen(pBuilder->pValC, NULL /* todo */, NULL /* todo */); if (code) return code; } return code; } -static int32_t tDiskColAddValue(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColVal->type)) { - code = tCompress(pDiskCol->pOffC, &pDiskCol->offset, sizeof(int32_t)); + code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); if (code) goto _exit; - pDiskCol->offset += pColVal->value.nData; + pBuilder->offset += pColVal->value.nData; } - code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); if (code) goto _exit; _exit: return code; } -static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { // 0 +static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 int32_t code = 0; if (pColVal->isNone) { - pDiskCol->flag = HAS_NONE; + pBuilder->flag = HAS_NONE; } else if (pColVal->isNull) { - pDiskCol->flag = HAS_NULL; + pBuilder->flag = HAS_NULL; } else { - pDiskCol->flag = HAS_VALUE; - code = tDiskColAddValue(pDiskCol, pColVal); + pBuilder->flag = HAS_VALUE; + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NONE +static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE int32_t code = 0; if (!pColVal->isNone) { // bit map - int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pDiskCol->pBitMap, nBit); + code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; - memset(pDiskCol->pBitMap, 0, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); // value if (pColVal->isNull) { - pDiskCol->flag |= HAS_NULL; + pBuilder->flag |= HAS_NULL; } else { - pDiskCol->flag |= HAS_VALUE; + pBuilder->flag |= HAS_VALUE; SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - code = tDiskColAddValue(pDiskCol, &cv); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColAddValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL +static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL int32_t code = 0; if (!pColVal->isNull) { - int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); - code = tRealloc(&pDiskCol->pBitMap, nBit); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; if (pColVal->isNone) { - pDiskCol->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; - memset(pDiskCol->pBitMap, 255, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { - pDiskCol->flag |= HAS_VALUE; + pBuilder->flag |= HAS_VALUE; - memset(pDiskCol->pBitMap, 0, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - code = tDiskColAddValue(pDiskCol, &cv); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColAddValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; if (pColVal->isNone) { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else if (pColVal->isNull) { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } else { - pDiskCol->flag |= HAS_VALUE; + pBuilder->flag |= HAS_VALUE; uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal)); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); } - SET_BIT2(pBitMap, pDiskCol->nVal, 2); + SET_BIT2(pBitMap, pBuilder->nVal, 2); - tFree(pDiskCol->pBitMap); - pDiskCol->pBitMap = pBitMap; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - code = tDiskColAddValue(pDiskCol, &cv); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColAddValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE +static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE int32_t code = 0; if (pColVal->isNone || pColVal->isNull) { if (pColVal->isNone) { - pDiskCol->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; } else { - pDiskCol->flag |= HAS_NULL; + pBuilder->flag |= HAS_NULL; } - int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); - code = tRealloc(&pDiskCol->pBitMap, nBit); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; - memset(pDiskCol->pBitMap, 255, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } else { - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NONE +static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NONE int32_t code = 0; if (pColVal->isNull) { - pDiskCol->flag |= HAS_NULL; + pBuilder->flag |= HAS_NULL; uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 0); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); } - SET_BIT2(pBitMap, pDiskCol->nVal, 1); + SET_BIT2(pBitMap, pBuilder->nVal, 1); - tFree(pDiskCol->pBitMap); - pDiskCol->pBitMap = pBitMap; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; } else { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; if (pColVal->isNone) { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL +static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL int32_t code = 0; if (pColVal->isNone) { - pDiskCol->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 1); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); } - SET_BIT2(pBitMap, pDiskCol->nVal, 0); + SET_BIT2(pBitMap, pBuilder->nVal, 0); - tFree(pDiskCol->pBitMap); - pDiskCol->pBitMap = pBitMap; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; } else { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; if (pColVal->isNull) { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE +static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE int32_t code = 0; - code = tRealloc(&pDiskCol->pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; if (pColVal->isNone) { - SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); } else if (pColVal->isNull) { - SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); } else { - SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 2); + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { +static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { tDiskColAddVal0, // 0 tDiskColAddVal1, // HAS_NONE tDiskColAddVal2, // HAS_NULL @@ -377,7 +377,7 @@ struct SDiskDataBuilder { SCompressor *pVerC; SCompressor *pKeyC; int32_t nDiskCol; - SArray *aDiskCol; + SArray *aDiskColBuilder; uint8_t *aBuf[2]; }; @@ -410,9 +410,9 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; - if (pBuilder->aDiskCol == NULL) { - pBuilder->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); - if (pBuilder->aDiskCol == NULL) { + if (pBuilder->aDiskColBuilder == NULL) { + pBuilder->aDiskColBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); + if (pBuilder->aDiskColBuilder == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } @@ -422,17 +422,17 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { STColumn *pTColumn = &pTSchema->columns[iCol]; - if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskCol)) { - SDiskCol dc = (SDiskCol){0}; - if (taosArrayPush(pBuilder->aDiskCol, &dc) == NULL) { + if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskColBuilder)) { + SDiskColBuilder dc = (SDiskColBuilder){0}; + if (taosArrayPush(pBuilder->aDiskColBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, pBuilder->nDiskCol); + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, pBuilder->nDiskCol); - code = tDiskColInit(pDiskCol, pTColumn->colId, pTColumn->type, cmprAlg); + code = tDiskColInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; pBuilder->nDiskCol++; @@ -448,12 +448,13 @@ int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - if (pBuilder->aDiskCol) { - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pBuilder->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); - tDiskColClear(pDiskCol); + if (pBuilder->aDiskColBuilder) { + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aDiskColBuilder); + iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + tDiskColClear(pDiskColBuilder); } - taosArrayDestroy(pBuilder->aDiskCol); + taosArrayDestroy(pBuilder->aDiskColBuilder); } for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { tFree(pBuilder->aBuf[iBuf]); @@ -494,19 +495,19 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - while (pColVal && pColVal->cid < pDiskCol->cid) { + while (pColVal && pColVal->cid < pDiskColBuilder->cid) { pColVal = tRowIterNext(&iter); } - if (pColVal == NULL || pColVal->cid > pDiskCol->cid) { - SColVal cv = COL_VAL_NONE(pDiskCol->cid, pDiskCol->type); - code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &cv); + if (pColVal == NULL || pColVal->cid > pDiskColBuilder->cid) { + SColVal cv = COL_VAL_NONE(pDiskColBuilder->cid, pDiskColBuilder->type); + code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, &cv); if (code) goto _exit; } else { - code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, pColVal); + code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, pColVal); if (code) goto _exit; pColVal = tRowIterNext(&iter); } @@ -551,18 +552,18 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { if (code) return code; int32_t offset = 0; - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - if (pDiskCol->flag == HAS_NONE) continue; + if (pDiskColBuilder->flag == HAS_NONE) continue; - code = tDiskColToBinary(pDiskCol, NULL, NULL); + code = tGnrtDiskCol(pDiskColBuilder, NULL, NULL); if (code) return code; - SBlockCol bCol = {.cid = pDiskCol->cid, - .type = pDiskCol->type, + SBlockCol bCol = {.cid = pDiskColBuilder->cid, + .type = pDiskColBuilder->type, // .smaOn = , - .flag = pDiskCol->flag, + .flag = pDiskColBuilder->flag, // .szOrigin = // .szBitmap = // .szOffset = @@ -589,12 +590,12 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { n += hdr.szVer; memcpy(pBuilder->aBuf[0] + n, pKey, hdr.szKey); n += hdr.szKey; - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); n += tPutBlockCol(pBuilder->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); } - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); // memcpy(pDiskData->aBuf[0] + n, NULL, ); // n += 0; } From f8619dfbe8859dec9e5dae09abb60f5166ad92dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 15:28:04 +0800 Subject: [PATCH 12/59] refact code for further refact --- include/common/tdataformat.h | 12 +-- source/common/src/tdataformat.c | 96 ++---------------- source/common/src/trow.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 127 +++++++++++++----------- 5 files changed, 80 insertions(+), 163 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 5b49560175..8e0fc2f680 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -154,17 +154,7 @@ struct STSRowBuilder { struct SValue { union { - int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT - uint8_t u8; // TSDB_DATA_TYPE_UTINYINT - int16_t i16; // TSDB_DATA_TYPE_SMALLINT - uint16_t u16; // TSDB_DATA_TYPE_USMALLINT - int32_t i32; // TSDB_DATA_TYPE_INT - uint32_t u32; // TSDB_DATA_TYPE_UINT - int64_t i64; // TSDB_DATA_TYPE_BIGINT - uint64_t u64; // TSDB_DATA_TYPE_UBIGINT - TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP - float f; // TSDB_DATA_TYPE_FLOAT - double d; // TSDB_DATA_TYPE_DOUBLE + int64_t val; struct { uint32_t nData; uint8_t *pData; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 15a369fe40..e37d587529 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -33,105 +33,21 @@ typedef struct { // SValue int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { - int32_t n = 0; - if (IS_VAR_DATA_TYPE(type)) { - n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData); + return tPutBinary(p, pValue->pData, pValue->nData); } else { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tPutI8(p ? p + n : p, pValue->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tPutI16(p ? p + n : p, pValue->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tPutI32(p ? p + n : p, pValue->i32); - break; - case TSDB_DATA_TYPE_BIGINT: - n += tPutI64(p ? p + n : p, pValue->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tPutFloat(p ? p + n : p, pValue->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tPutDouble(p ? p + n : p, pValue->d); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - n += tPutI64(p ? p + n : p, pValue->ts); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tPutU8(p ? p + n : p, pValue->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tPutU16(p ? p + n : p, pValue->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tPutU32(p ? p + n : p, pValue->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tPutU64(p ? p + n : p, pValue->u64); - break; - default: - ASSERT(0); - } + if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); + return tDataTypes[type].bytes; } - - return n; } int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - int32_t n = 0; - if (IS_VAR_DATA_TYPE(type)) { - n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); + return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); } else { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - n += tGetI8(p, &pValue->i8); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p, &pValue->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, &pValue->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tGetI32(p, &pValue->i32); - break; - case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, &pValue->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, &pValue->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, &pValue->d); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - n += tGetI64(p, &pValue->ts); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, &pValue->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, &pValue->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, &pValue->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, &pValue->u64); - break; - default: - ASSERT(0); - } + memcpy(&pValue->val, p, tDataTypes[type].bytes); + return tDataTypes[type].bytes; } - - return n; } int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9880fe362e..f2e060ae86 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -689,7 +689,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); val = varBuf; } else { - val = (const void *)&pColVal->value.i64; + val = (const void *)&pColVal->value.val; } } else { pColVal = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8452d14c86..185d836e26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -260,7 +260,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; - SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs}); + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal}); } @@ -1048,7 +1048,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo lastRowTs = TSDBROW_TS(pRow); STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); if (taosArrayPush(pColArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1147,7 +1147,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { lastRowTs = rowTs; STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4e6e1f2f7f..c77af951af 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -649,7 +649,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -690,7 +690,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { STColumn *pTColumn; int32_t iCol, jCol = 1; - ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { pTColumn = &pMerger->pTSchema->columns[iCol]; @@ -744,7 +744,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -770,7 +770,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; - ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal); @@ -1513,73 +1513,79 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { case TSDB_DATA_TYPE_BOOL: break; case TSDB_DATA_TYPE_TINYINT: { - pColAgg->sum += colVal.value.i8; - if (!minAssigned || pColAgg->min > colVal.value.i8) { - pColAgg->min = colVal.value.i8; + int8_t i8 = *(int8_t *)&colVal.value.val; + pColAgg->sum += i8; + if (!minAssigned || pColAgg->min > i8) { + pColAgg->min = i8; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i8) { - pColAgg->max = colVal.value.i8; + if (!maxAssigned || pColAgg->max < i8) { + pColAgg->max = i8; maxAssigned = true; } break; } case TSDB_DATA_TYPE_SMALLINT: { - pColAgg->sum += colVal.value.i16; - if (!minAssigned || pColAgg->min > colVal.value.i16) { - pColAgg->min = colVal.value.i16; + int16_t i16 = *(int16_t *)&colVal.value.val; + pColAgg->sum += i16; + if (!minAssigned || pColAgg->min > i16) { + pColAgg->min = i16; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i16) { - pColAgg->max = colVal.value.i16; + if (!maxAssigned || pColAgg->max < i16) { + pColAgg->max = i16; maxAssigned = true; } break; } case TSDB_DATA_TYPE_INT: { - pColAgg->sum += colVal.value.i32; - if (!minAssigned || pColAgg->min > colVal.value.i32) { - pColAgg->min = colVal.value.i32; + int32_t i32 = *(int32_t *)&colVal.value.val; + pColAgg->sum += i32; + if (!minAssigned || pColAgg->min > i32) { + pColAgg->min = i32; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i32) { - pColAgg->max = colVal.value.i32; + if (!maxAssigned || pColAgg->max < i32) { + pColAgg->max = i32; maxAssigned = true; } break; } case TSDB_DATA_TYPE_BIGINT: { - pColAgg->sum += colVal.value.i64; - if (!minAssigned || pColAgg->min > colVal.value.i64) { - pColAgg->min = colVal.value.i64; + int64_t i64 = *(int64_t *)&colVal.value.val; + pColAgg->sum += i64; + if (!minAssigned || pColAgg->min > i64) { + pColAgg->min = i64; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i64) { - pColAgg->max = colVal.value.i64; + if (!maxAssigned || pColAgg->max < i64) { + pColAgg->max = i64; maxAssigned = true; } break; } case TSDB_DATA_TYPE_FLOAT: { - *(double *)(&pColAgg->sum) += colVal.value.f; - if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) { - *(double *)(&pColAgg->min) = colVal.value.f; + float f = *(float *)&colVal.value.val; + *(double *)(&pColAgg->sum) += f; + if (!minAssigned || *(double *)(&pColAgg->min) > f) { + *(double *)(&pColAgg->min) = f; minAssigned = true; } - if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) { - *(double *)(&pColAgg->max) = colVal.value.f; + if (!maxAssigned || *(double *)(&pColAgg->max) < f) { + *(double *)(&pColAgg->max) = f; maxAssigned = true; } break; } case TSDB_DATA_TYPE_DOUBLE: { - *(double *)(&pColAgg->sum) += colVal.value.d; - if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) { - *(double *)(&pColAgg->min) = colVal.value.d; + double d = *(double *)&colVal.value.val; + *(double *)(&pColAgg->sum) += d; + if (!minAssigned || *(double *)(&pColAgg->min) > d) { + *(double *)(&pColAgg->min) = d; minAssigned = true; } - if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) { - *(double *)(&pColAgg->max) = colVal.value.d; + if (!maxAssigned || *(double *)(&pColAgg->max) < d) { + *(double *)(&pColAgg->max) = d; maxAssigned = true; } break; @@ -1587,12 +1593,13 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { case TSDB_DATA_TYPE_VARCHAR: break; case TSDB_DATA_TYPE_TIMESTAMP: { - if (!minAssigned || pColAgg->min > colVal.value.i64) { - pColAgg->min = colVal.value.i64; + int64_t ts = *(int64_t *)&colVal.value.val; + if (!minAssigned || pColAgg->min > ts) { + pColAgg->min = ts; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i64) { - pColAgg->max = colVal.value.i64; + if (!maxAssigned || pColAgg->max < ts) { + pColAgg->max = ts; maxAssigned = true; } break; @@ -1600,49 +1607,53 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { case TSDB_DATA_TYPE_NCHAR: break; case TSDB_DATA_TYPE_UTINYINT: { - pColAgg->sum += colVal.value.u8; - if (!minAssigned || pColAgg->min > colVal.value.u8) { - pColAgg->min = colVal.value.u8; + uint8_t u8 = *(uint8_t *)&colVal.value.val; + pColAgg->sum += u8; + if (!minAssigned || pColAgg->min > u8) { + pColAgg->min = u8; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.u8) { - pColAgg->max = colVal.value.u8; + if (!maxAssigned || pColAgg->max < u8) { + pColAgg->max = u8; maxAssigned = true; } break; } case TSDB_DATA_TYPE_USMALLINT: { - pColAgg->sum += colVal.value.u16; - if (!minAssigned || pColAgg->min > colVal.value.u16) { - pColAgg->min = colVal.value.u16; + uint16_t u16 = *(uint16_t *)&colVal.value.val; + pColAgg->sum += u16; + if (!minAssigned || pColAgg->min > u16) { + pColAgg->min = u16; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.u16) { - pColAgg->max = colVal.value.u16; + if (!maxAssigned || pColAgg->max < u16) { + pColAgg->max = u16; maxAssigned = true; } break; } case TSDB_DATA_TYPE_UINT: { - pColAgg->sum += colVal.value.u32; - if (!minAssigned || pColAgg->min > colVal.value.u32) { - pColAgg->min = colVal.value.u32; + uint32_t u32 = *(uint32_t *)&colVal.value.val; + pColAgg->sum += u32; + if (!minAssigned || pColAgg->min > u32) { + pColAgg->min = u32; minAssigned = true; } - if (!minAssigned || pColAgg->max < colVal.value.u32) { - pColAgg->max = colVal.value.u32; + if (!minAssigned || pColAgg->max < u32) { + pColAgg->max = u32; maxAssigned = true; } break; } case TSDB_DATA_TYPE_UBIGINT: { - pColAgg->sum += colVal.value.u64; - if (!minAssigned || pColAgg->min > colVal.value.u64) { - pColAgg->min = colVal.value.u64; + uint64_t u64 = *(uint64_t *)&colVal.value.val; + pColAgg->sum += u64; + if (!minAssigned || pColAgg->min > u64) { + pColAgg->min = u64; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.u64) { - pColAgg->max = colVal.value.u64; + if (!maxAssigned || pColAgg->max < u64) { + pColAgg->max = u64; maxAssigned = true; } break; From f870f7e4538b29166d318de806df720477d23768 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 16:08:51 +0800 Subject: [PATCH 13/59] fix gtest compile error --- source/common/test/dataformatTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 65f21bee40..5121724a08 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#if 0 #include #include @@ -476,4 +477,5 @@ TEST(testCase, NoneTest) { taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } +#endif #endif \ No newline at end of file From 0929326c1dbd9b0f458d75c9224d1803e8bcbbc6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 17:13:45 +0800 Subject: [PATCH 14/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 213 ++++++++++----------- 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index a514bddabb..25c1ea20d6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -17,6 +17,7 @@ typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SDiskColBuilder SDiskColBuilder; +typedef struct SDiskCol SDiskCol; typedef struct SDiskData SDiskData; struct SDiskColBuilder { @@ -31,6 +32,34 @@ struct SDiskColBuilder { SCompressor *pValC; }; +struct SDiskDataBuilder { + int64_t suid; + int64_t uid; + int32_t nRow; + uint8_t cmprAlg; + SCompressor *pUidC; + SCompressor *pVerC; + SCompressor *pKeyC; + int32_t nBuilder; + SArray *aBuilder; + uint8_t *aBuf[2]; +}; + +struct SDiskCol { + SBlockCol bCol; + const uint8_t *pBit; + const uint8_t *pOff; + const uint8_t *pVal; +}; + +struct SDiskData { + SDiskDataHdr hdr; + const uint8_t *pUid; + const uint8_t *pVer; + const uint8_t *pKey; + SArray *aDiskCol; // SArray +}; + // SDiskColBuilder ================================================ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; @@ -71,28 +100,40 @@ static int32_t tDiskColClear(SDiskColBuilder *pBuilder) { return code; } -static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SBlockCol *pBlockCol, const uint8_t **ppData) { +static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { int32_t code = 0; ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); - if (pBuilder->flag == HAS_NULL) { - return code; - } + pDiskCol->bCol = (SBlockCol){.cid = pBuilder->cid, + .type = pBuilder->type, + .smaOn = 1, /* todo */ + .flag = pBuilder->flag, + .szOrigin = 0, // todo + .szBitmap = 0, // todo + .szOffset = 0, + .szValue = 0, // todo + .offset = 0}; + pDiskCol->pBit = NULL; + pDiskCol->pOff = NULL; + pDiskCol->pVal = NULL; - // bitmap (todo) + if (pBuilder->flag == HAS_NULL) return code; + + // BITMAP if (pBuilder->flag != HAS_VALUE) { + // TODO } - // offset (todo) + // OFFSET if (IS_VAR_DATA_TYPE(pBuilder->type)) { - code = tCompGen(pBuilder->pOffC, NULL /* todo */, NULL /* todo */); + code = tCompGen(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); if (code) return code; } - // value (todo) + // VALUE if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { - code = tCompGen(pBuilder->pValC, NULL /* todo */, NULL /* todo */); + code = tCompGen(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); if (code) return code; } @@ -368,19 +409,6 @@ static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColV }; // SDiskDataBuilder ================================================ -struct SDiskDataBuilder { - int64_t suid; - int64_t uid; - int32_t nRow; - uint8_t cmprAlg; - SCompressor *pUidC; - SCompressor *pVerC; - SCompressor *pKeyC; - int32_t nDiskCol; - SArray *aDiskColBuilder; - uint8_t *aBuf[2]; -}; - int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { int32_t code = 0; @@ -410,32 +438,32 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; - if (pBuilder->aDiskColBuilder == NULL) { - pBuilder->aDiskColBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); - if (pBuilder->aDiskColBuilder == NULL) { + if (pBuilder->aBuilder == NULL) { + pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); + if (pBuilder->aBuilder == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - pBuilder->nDiskCol = 0; + pBuilder->nBuilder = 0; for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { STColumn *pTColumn = &pTSchema->columns[iCol]; - if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskColBuilder)) { + if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { SDiskColBuilder dc = (SDiskColBuilder){0}; - if (taosArrayPush(pBuilder->aDiskColBuilder, &dc) == NULL) { + if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, pBuilder->nDiskCol); + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); code = tDiskColInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; - pBuilder->nDiskCol++; + pBuilder->nBuilder++; } return code; @@ -448,13 +476,12 @@ int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - if (pBuilder->aDiskColBuilder) { - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aDiskColBuilder); - iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + if (pBuilder->aBuilder) { + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aBuilder); iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); tDiskColClear(pDiskColBuilder); } - taosArrayDestroy(pBuilder->aDiskColBuilder); + taosArrayDestroy(pBuilder->aBuilder); } for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { tFree(pBuilder->aBuf[iBuf]); @@ -495,8 +522,8 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nBuilder; iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); while (pColVal && pColVal->cid < pDiskColBuilder->cid) { pColVal = tRowIterNext(&iter); @@ -523,102 +550,74 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { ASSERT(pBuilder->nRow); - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, - .fmtVer = 0, - .suid = pBuilder->suid, - .uid = pBuilder->uid, - .szUid = 0, - .szVer = 0, - .szKey = 0, - .szBlkCol = 0, - .nRow = pBuilder->nRow, - .cmprAlg = pBuilder->cmprAlg}; + // reset SDiskData + pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pBuilder->suid, + .uid = pBuilder->uid, + .szUid = 0, + .szVer = 0, + .szKey = 0, + .szBlkCol = 0, + .nRow = pBuilder->nRow, + .cmprAlg = pBuilder->cmprAlg}; + pDiskData->pUid = NULL; + pDiskData->pVer = NULL; + pDiskData->pKey = NULL; + if (pDiskData->aDiskCol) { + taosArrayClear(pDiskData->aDiskCol); + } else { + pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol)); + if (pDiskData->aDiskCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } // UID - const uint8_t *pUid = NULL; if (pBuilder->uid == 0) { - code = tCompGen(pBuilder->pUidC, &pUid, &hdr.szUid); + code = tCompGen(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); if (code) return code; } // VERSION - const uint8_t *pVer = NULL; - code = tCompGen(pBuilder->pVerC, &pVer, &hdr.szVer); + code = tCompGen(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); if (code) return code; // TSKEY - const uint8_t *pKey = NULL; - code = tCompGen(pBuilder->pKeyC, &pKey, &hdr.szKey); + code = tCompGen(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); if (code) return code; int32_t offset = 0; - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - if (pDiskColBuilder->flag == HAS_NONE) continue; + if (pDCBuilder->flag == HAS_NONE) continue; - code = tGnrtDiskCol(pDiskColBuilder, NULL, NULL); + SDiskCol dCol; + + code = tGnrtDiskCol(pDCBuilder, &dCol); if (code) return code; - SBlockCol bCol = {.cid = pDiskColBuilder->cid, - .type = pDiskColBuilder->type, - // .smaOn = , - .flag = pDiskColBuilder->flag, - // .szOrigin = - // .szBitmap = - // .szOffset = - // .szValue = - .offset = offset}; + dCol.bCol.offset = offset; + offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue; - hdr.szBlkCol += tPutBlockCol(NULL, &bCol); - offset = offset + bCol.szBitmap + bCol.szOffset + bCol.szValue; - } + if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } -#if 0 - *nData = tPutDiskDataHdr(NULL, &hdr) + hdr.szUid + hdr.szVer + hdr.szKey + hdr.szBlkCol + offset; - code = tRealloc(&pBuilder->aBuf[0], *nData); - if (code) return code; - *ppData = pBuilder->aBuf[0]; - - int32_t n = 0; - n += tPutDiskDataHdr(pBuilder->aBuf[0] + n, &hdr); - if (hdr.szUid) { - memcpy(pBuilder->aBuf[0] + n, pUid, hdr.szUid); - n += hdr.szUid; + if (pDCBuilder->flag != HAS_NULL) { + pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); + } } - memcpy(pBuilder->aBuf[0] + n, pVer, hdr.szVer); - n += hdr.szVer; - memcpy(pBuilder->aBuf[0] + n, pKey, hdr.szKey); - n += hdr.szKey; - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - n += tPutBlockCol(pBuilder->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); - } - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - // memcpy(pDiskData->aBuf[0] + n, NULL, ); - // n += 0; - } -#endif return code; } // SDiskData ================================================ -struct SDiskData { - SDiskDataHdr hdr; - const uint8_t *pUid; - const uint8_t *pVer; - const uint8_t *pKey; - SArray *aBlockCol; - SArray *aColData; -}; - int32_t tDiskDataDestroy(SDiskData *pDiskData) { int32_t code = 0; - - if (pDiskData->aBlockCol) taosArrayDestroy(pDiskData->aBlockCol); - if (pDiskData->aColData) taosArrayDestroy(pDiskData->aColData); - + if (pDiskData->aDiskCol) taosArrayDestroy(pDiskData->aDiskCol); return code; } From a8b919918f79097b7aeb3a7a9990e7e3485a8385 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 17:58:44 +0800 Subject: [PATCH 15/59] more code --- source/dnode/vnode/src/inc/tsdb.h | 23 +++++ source/dnode/vnode/src/tsdb/tsdbDiskData.c | 25 ++---- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 87 +++++++++++++++++++ 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c2a31679d0..e67fa2ee2d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -32,6 +32,12 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ + if (CODE) { \ + LINO = __LINE__; \ + goto LABEL; \ + } + typedef struct TSDBROW TSDBROW; typedef struct TABLEID TABLEID; typedef struct TSDBKEY TSDBKEY; @@ -65,6 +71,8 @@ typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; typedef struct SVersionRange SVersionRange; typedef struct SLDataIter SLDataIter; +typedef struct SDiskCol SDiskCol; +typedef struct SDiskData SDiskData; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -652,6 +660,21 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; +struct SDiskCol { + SBlockCol bCol; + const uint8_t *pBit; + const uint8_t *pOff; + const uint8_t *pVal; +}; + +struct SDiskData { + SDiskDataHdr hdr; + const uint8_t *pUid; + const uint8_t *pVer; + const uint8_t *pKey; + SArray *aDiskCol; // SArray +}; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 25c1ea20d6..9b5934a04f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -17,8 +17,6 @@ typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SDiskColBuilder SDiskColBuilder; -typedef struct SDiskCol SDiskCol; -typedef struct SDiskData SDiskData; struct SDiskColBuilder { int16_t cid; @@ -45,21 +43,6 @@ struct SDiskDataBuilder { uint8_t *aBuf[2]; }; -struct SDiskCol { - SBlockCol bCol; - const uint8_t *pBit; - const uint8_t *pOff; - const uint8_t *pVal; -}; - -struct SDiskData { - SDiskDataHdr hdr; - const uint8_t *pUid; - const uint8_t *pVer; - const uint8_t *pKey; - SArray *aDiskCol; // SArray -}; - // SDiskColBuilder ================================================ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; @@ -147,9 +130,13 @@ static int32_t tDiskColAddValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); if (code) goto _exit; pBuilder->offset += pColVal->value.nData; + + code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + } else { + code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes); + if (code) goto _exit; } - code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); - if (code) goto _exit; _exit: return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 5fe0b408b1..8674ae571e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -607,6 +607,93 @@ _err: return code; } +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInfo *pBlkInfo) { + int32_t code = 0; + int32_t lino = 0; + + STsdbFD *pFD = pWriter->pDataFD; // todo + int64_t offset = pWriter->fData.size; + + // hdr + int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); + code = tRealloc(&pWriter->aBuf[0], n); + TSDB_CHECK_CODE(code, lino, _exit); + + tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); + + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], n); + TSDB_CHECK_CODE(code, lino, _exit); + offset += n; + + // uid + ver + key + if (pDiskData->hdr.szUid) { + code = tsdbWriteFile(pFD, offset, pDiskData->pUid, pDiskData->hdr.szUid); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskData->hdr.szUid; + } + + code = tsdbWriteFile(pFD, offset, pDiskData->pVer, pDiskData->hdr.szVer); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskData->hdr.szVer; + + code = tsdbWriteFile(pFD, offset, pDiskData->pKey, pDiskData->hdr.szKey); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskData->hdr.szKey; + + // SBlockCol + if (pDiskData->hdr.szBlkCol) { + code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + n = 0; + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + n += tPutBlockCol(pWriter->aBuf[0] + n, &pDiskCol->bCol); + } + + ASSERT(n == pDiskData->hdr.szBlkCol); + + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + offset += pDiskData->hdr.szBlkCol; + } + + // pData + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (pDiskCol->bCol.flag == HAS_NULL) continue; + + if (pDiskCol->bCol.szBitmap) { + code = tsdbWriteFile(pFD, offset, pDiskCol->pBit, pDiskCol->bCol.szBitmap); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskCol->bCol.szBitmap; + } + + if (pDiskCol->bCol.szOffset) { + code = tsdbWriteFile(pFD, offset, pDiskCol->pOff, pDiskCol->bCol.szOffset); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskCol->bCol.szOffset; + } + + if (pDiskCol->bCol.szValue) { + code = tsdbWriteFile(pFD, offset, pDiskCol->pVal, pDiskCol->bCol.szValue); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskCol->bCol.szValue; + } + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d %s", TD_VID(pWriter->pTsdb->pVnode), __func__); + } + return code; +} + int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t code = 0; int64_t n; From 9f7771cde4a892407a3d8802488addaea79a9f19 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 11:52:47 +0800 Subject: [PATCH 16/59] refact more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbDiskData.c | 97 ++++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/util/src/tcompression.c | 2 +- 5 files changed, 56 insertions(+), 48 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 8d7c607e6b..f22205d144 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -130,7 +130,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e67fa2ee2d..2121d779ab 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -665,6 +665,7 @@ struct SDiskCol { const uint8_t *pBit; const uint8_t *pOff; const uint8_t *pVal; + SColumnDataAgg agg; }; struct SDiskData { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9b5934a04f..9e2f8eec7c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -35,16 +35,17 @@ struct SDiskDataBuilder { int64_t uid; int32_t nRow; uint8_t cmprAlg; + uint8_t calcSma; SCompressor *pUidC; SCompressor *pVerC; SCompressor *pKeyC; int32_t nBuilder; - SArray *aBuilder; + SArray *aBuilder; // SArray uint8_t *aBuf[2]; }; // SDiskColBuilder ================================================ -static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { +static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; pBuilder->cid = cid; @@ -59,7 +60,7 @@ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, code = tCompressorCreate(&pBuilder->pOffC); if (code) return code; } - code = tCompressorReset(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + code = tCompressorInit(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } @@ -67,13 +68,13 @@ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, code = tCompressorCreate(&pBuilder->pValC); if (code) return code; } - code = tCompressorReset(pBuilder->pValC, type, cmprAlg); + code = tCompressorInit(pBuilder->pValC, type, cmprAlg); if (code) return code; return code; } -static int32_t tDiskColClear(SDiskColBuilder *pBuilder) { +static int32_t tDiskColBuilderClear(SDiskColBuilder *pBuilder) { int32_t code = 0; tFree(pBuilder->pBitMap); @@ -396,33 +397,60 @@ static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColV }; // SDiskDataBuilder ================================================ -int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { + int32_t code = 0; + + *ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder)); + if (*ppBuilder == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + return code; +} + +void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { + if (pBuilder == NULL) return NULL; + + if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); + if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); + if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); + + if (pBuilder->aBuilder) { + for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); + tDiskColBuilderClear(pDCBuilder); + } + taosArrayDestroy(pBuilder->aBuilder); + } + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); + } + taosMemoryFree(pBuilder); + + return NULL; +} + +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, + uint8_t calcSma) { int32_t code = 0; pBuilder->suid = pId->suid; pBuilder->uid = pId->uid; pBuilder->nRow = 0; pBuilder->cmprAlg = cmprAlg; + pBuilder->calcSma = calcSma; - if (pBuilder->pUidC == NULL) { - code = tCompressorCreate(&pBuilder->pUidC); - if (code) return code; - } - code = tCompressorReset(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; + code = tCompressorInit(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pBuilder->pVerC == NULL) { - code = tCompressorCreate(&pBuilder->pVerC); - if (code) return code; - } - code = tCompressorReset(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code; + code = tCompressorInit(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pBuilder->pKeyC == NULL) { - code = tCompressorCreate(&pBuilder->pKeyC); - if (code) return code; - } - code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code; + code = tCompressorInit(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; if (pBuilder->aBuilder == NULL) { @@ -438,7 +466,7 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB STColumn *pTColumn = &pTSchema->columns[iCol]; if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { - SDiskColBuilder dc = (SDiskColBuilder){0}; + SDiskColBuilder dc = {0}; if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; @@ -447,7 +475,7 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); - code = tDiskColInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); + code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; pBuilder->nBuilder++; @@ -456,27 +484,6 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB return code; } -int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { - int32_t code = 0; - - if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); - if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); - if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - - if (pBuilder->aBuilder) { - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aBuilder); iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); - tDiskColClear(pDiskColBuilder); - } - taosArrayDestroy(pBuilder->aBuilder); - } - for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { - tFree(pBuilder->aBuf[iBuf]); - } - - return code; -} - int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { int32_t code = 0; @@ -605,6 +612,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { // SDiskData ================================================ int32_t tDiskDataDestroy(SDiskData *pDiskData) { int32_t code = 0; - if (pDiskData->aDiskCol) taosArrayDestroy(pDiskData->aDiskCol); + pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8674ae571e..679f7c3457 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -128,7 +128,7 @@ _exit: return code; } -static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { +static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index cd9ff5e325..619ae8360d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1518,7 +1518,7 @@ int32_t tCompressorDestroy(SCompressor *pCmprsor) { return code; } -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; pCmprsor->type = type; From a3cedbf1bc60a864e436ae294c345ba54f6b86f1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 13:49:34 +0800 Subject: [PATCH 17/59] more code --- source/dnode/vnode/src/inc/tsdb.h | 82 ++++++++++++---------- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 62 ++++++++-------- 2 files changed, 77 insertions(+), 67 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2121d779ab..dc218e86e0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -38,41 +38,42 @@ extern "C" { goto LABEL; \ } -typedef struct TSDBROW TSDBROW; -typedef struct TABLEID TABLEID; -typedef struct TSDBKEY TSDBKEY; -typedef struct SDelData SDelData; -typedef struct SDelIdx SDelIdx; -typedef struct STbData STbData; -typedef struct SMemTable SMemTable; -typedef struct STbDataIter STbDataIter; -typedef struct SMapData SMapData; -typedef struct SBlockIdx SBlockIdx; -typedef struct SDataBlk SDataBlk; -typedef struct SSttBlk SSttBlk; -typedef struct SDiskDataHdr SDiskDataHdr; -typedef struct SBlockData SBlockData; -typedef struct SDelFile SDelFile; -typedef struct SHeadFile SHeadFile; -typedef struct SDataFile SDataFile; -typedef struct SSttFile SSttFile; -typedef struct SSmaFile SSmaFile; -typedef struct SDFileSet SDFileSet; -typedef struct SDataFWriter SDataFWriter; -typedef struct SDataFReader SDataFReader; -typedef struct SDelFWriter SDelFWriter; -typedef struct SDelFReader SDelFReader; -typedef struct SRowIter SRowIter; -typedef struct STsdbFS STsdbFS; -typedef struct SRowMerger SRowMerger; -typedef struct STsdbReadSnap STsdbReadSnap; -typedef struct SBlockInfo SBlockInfo; -typedef struct SSmaInfo SSmaInfo; -typedef struct SBlockCol SBlockCol; -typedef struct SVersionRange SVersionRange; -typedef struct SLDataIter SLDataIter; -typedef struct SDiskCol SDiskCol; -typedef struct SDiskData SDiskData; +typedef struct TSDBROW TSDBROW; +typedef struct TABLEID TABLEID; +typedef struct TSDBKEY TSDBKEY; +typedef struct SDelData SDelData; +typedef struct SDelIdx SDelIdx; +typedef struct STbData STbData; +typedef struct SMemTable SMemTable; +typedef struct STbDataIter STbDataIter; +typedef struct SMapData SMapData; +typedef struct SBlockIdx SBlockIdx; +typedef struct SDataBlk SDataBlk; +typedef struct SSttBlk SSttBlk; +typedef struct SDiskDataHdr SDiskDataHdr; +typedef struct SBlockData SBlockData; +typedef struct SDelFile SDelFile; +typedef struct SHeadFile SHeadFile; +typedef struct SDataFile SDataFile; +typedef struct SSttFile SSttFile; +typedef struct SSmaFile SSmaFile; +typedef struct SDFileSet SDFileSet; +typedef struct SDataFWriter SDataFWriter; +typedef struct SDataFReader SDataFReader; +typedef struct SDelFWriter SDelFWriter; +typedef struct SDelFReader SDelFReader; +typedef struct SRowIter SRowIter; +typedef struct STsdbFS STsdbFS; +typedef struct SRowMerger SRowMerger; +typedef struct STsdbReadSnap STsdbReadSnap; +typedef struct SBlockInfo SBlockInfo; +typedef struct SSmaInfo SSmaInfo; +typedef struct SBlockCol SBlockCol; +typedef struct SVersionRange SVersionRange; +typedef struct SLDataIter SLDataIter; +typedef struct SDiskCol SDiskCol; +typedef struct SDiskData SDiskData; +typedef struct SDiskDataBuilder SDiskDataBuilder; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -302,7 +303,7 @@ int32_t tsdbMerge(STsdb *pTsdb); #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) #define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) -// tsdbCache +// tsdbCache ============================================================================================== int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); @@ -320,6 +321,15 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); +// tsdbDiskData ============================================================================================== +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); +void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder); +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, + uint8_t calcSma); +int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData); +int32_t tDiskDataDestroy(SDiskData *pDiskData); + // structs ======================= struct STsdbFS { SDelFile *pDelFile; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9e2f8eec7c..b93b6a7007 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -15,8 +15,7 @@ #include "tsdb.h" -typedef struct SDiskDataBuilder SDiskDataBuilder; -typedef struct SDiskColBuilder SDiskColBuilder; +typedef struct SDiskColBuilder SDiskColBuilder; struct SDiskColBuilder { int16_t cid; @@ -45,6 +44,19 @@ struct SDiskDataBuilder { }; // SDiskColBuilder ================================================ +#define tDiskColBuilderCreate() \ + (SDiskColBuilder) { 0 } + +static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) { + int32_t code = 0; + + tFree(pBuilder->pBitMap); + if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); + if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); + + return code; +} + static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; @@ -74,16 +86,6 @@ static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_ return code; } -static int32_t tDiskColBuilderClear(SDiskColBuilder *pBuilder) { - int32_t code = 0; - - tFree(pBuilder->pBitMap); - if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); - if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); - - return code; -} - static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { int32_t code = 0; @@ -419,7 +421,7 @@ void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { if (pBuilder->aBuilder) { for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - tDiskColBuilderClear(pDCBuilder); + tDiskColBuilderDestroy(pDCBuilder); } taosArrayDestroy(pBuilder->aBuilder); } @@ -466,7 +468,7 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB STColumn *pTColumn = &pTSchema->columns[iCol]; if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { - SDiskColBuilder dc = {0}; + SDiskColBuilder dc = tDiskColBuilderCreate(); if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; @@ -491,51 +493,51 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { + ASSERT(!pBuilder->calcSma); for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; } pBuilder->uid = 0; } if (pBuilder->uid == 0) { code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; } // version int64_t version = TSDBROW_VERSION(pRow); code = tCompress(pBuilder->pVerC, &version, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; // TSKEY TSKEY ts = TSDBROW_TS(pRow); code = tCompress(pBuilder->pKeyC, &ts, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nBuilder; iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); + for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - while (pColVal && pColVal->cid < pDiskColBuilder->cid) { + while (pColVal && pColVal->cid < pDCBuilder->cid) { pColVal = tRowIterNext(&iter); } - if (pColVal == NULL || pColVal->cid > pDiskColBuilder->cid) { - SColVal cv = COL_VAL_NONE(pDiskColBuilder->cid, pDiskColBuilder->type); - code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, &cv); - if (code) goto _exit; + if (pColVal == NULL || pColVal->cid > pDCBuilder->cid) { + SColVal cv = COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type); + code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, &cv); + if (code) return code; } else { - code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, pColVal); - if (code) goto _exit; + code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, pColVal); + if (code) return code; pColVal = tRowIterNext(&iter); } } pBuilder->nRow++; -_exit: return code; } @@ -601,9 +603,7 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { return code; } - if (pDCBuilder->flag != HAS_NULL) { - pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); - } + pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); } return code; From 1bab3c1b4d73c173521a458e7aa7ff3ce5351e0b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 18:24:24 +0800 Subject: [PATCH 18/59] optimize more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 244 +++++++++++++++------ 1 file changed, 178 insertions(+), 66 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b93b6a7007..5a235c5c82 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -18,15 +18,20 @@ typedef struct SDiskColBuilder SDiskColBuilder; struct SDiskColBuilder { - int16_t cid; - int8_t type; - int8_t flag; - uint8_t cmprAlg; - int32_t nVal; - uint8_t *pBitMap; - int32_t offset; - SCompressor *pOffC; - SCompressor *pValC; + int16_t cid; + int8_t type; + uint8_t cmprAlg; + uint8_t calcSma; + int8_t flag; + int32_t nVal; + uint8_t *pBitMap; + int32_t offset; + SCompressor *pOffC; + SCompressor *pValC; + SColumnDataAgg sma; + uint8_t minSet; + uint8_t maxSet; + uint8_t *aBuf[1]; }; struct SDiskDataBuilder { @@ -53,36 +58,41 @@ static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) { tFree(pBuilder->pBitMap); if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); + } return code; } -static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { +static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg, + uint8_t calcSma) { int32_t code = 0; pBuilder->cid = cid; pBuilder->type = type; - pBuilder->flag = 0; pBuilder->cmprAlg = cmprAlg; + pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma; + pBuilder->flag = 0; pBuilder->nVal = 0; pBuilder->offset = 0; if (IS_VAR_DATA_TYPE(type)) { - if (pBuilder->pOffC == NULL) { - code = tCompressorCreate(&pBuilder->pOffC); - if (code) return code; - } + if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code; code = tCompressorInit(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } - if (pBuilder->pValC == NULL) { - code = tCompressorCreate(&pBuilder->pValC); - if (code) return code; - } + if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code; code = tCompressorInit(pBuilder->pValC, type, cmprAlg); if (code) return code; + if (pBuilder->calcSma) { + pBuilder->sma = (SColumnDataAgg){.colId = cid}; + pBuilder->minSet = 0; + pBuilder->maxSet = 0; + } + return code; } @@ -91,24 +101,31 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); - pDiskCol->bCol = (SBlockCol){.cid = pBuilder->cid, - .type = pBuilder->type, - .smaOn = 1, /* todo */ - .flag = pBuilder->flag, - .szOrigin = 0, // todo - .szBitmap = 0, // todo - .szOffset = 0, - .szValue = 0, // todo - .offset = 0}; - pDiskCol->pBit = NULL; - pDiskCol->pOff = NULL; - pDiskCol->pVal = NULL; + *pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid, + .type = pBuilder->type, + .smaOn = pBuilder->calcSma, + .flag = pBuilder->flag, + .szOrigin = 0, // todo + .szBitmap = 0, + .szOffset = 0, + .szValue = 0, + .offset = 0}, + .pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma}; if (pBuilder->flag == HAS_NULL) return code; // BITMAP if (pBuilder->flag != HAS_VALUE) { - // TODO + int32_t nBit; + if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + nBit = BIT2_SIZE(pBuilder->nVal); + } else { + nBit = BIT1_SIZE(pBuilder->nVal); + } + + pDiskCol->bCol.szBitmap = tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], 0, pBuilder->cmprAlg, + NULL, 0); // todo: alloc + pDiskCol->pBit = pBuilder->aBuf[0]; } // OFFSET @@ -126,22 +143,114 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { return code; } -static int32_t tDiskColAddValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { +static FORCE_INLINE void tDiskColUpdateSmaBool(SDiskColBuilder *pBuilder, SColVal *pColVal) { + // TODO +} +static FORCE_INLINE void tDiskColUpdateSmaTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int8_t val = *(int8_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateSmaSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int16_t val = *(int16_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateSmaInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t val = *(int32_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int64_t val = *(int64_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateFloat(SDiskColBuilder *pBuilder, SColVal *pColVal) { + float val = *(float *)&pColVal->value.val; + *(double *)&pBuilder->sma.sum += val; + if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; + if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateDouble(SDiskColBuilder *pBuilder, SColVal *pColVal) { + double val = *(double *)&pColVal->value.val; + *(double *)&pBuilder->sma.sum += val; + if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; + if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint8_t val = *(uint8_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint16_t val = *(uint16_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint32_t val = *(uint32_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint64_t val = *(uint64_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static void (*tDiskColUpdateSmaImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { + NULL, // TSDB_DATA_TYPE_NULL + tDiskColUpdateSmaBool, // TSDB_DATA_TYPE_BOOL + tDiskColUpdateSmaTinyint, // TSDB_DATA_TYPE_TINYINT + tDiskColUpdateSmaSmallint, // TSDB_DATA_TYPE_SMALLINT + tDiskColUpdateSmaInt, // TSDB_DATA_TYPE_INT + tDiskColUpdateBigint, // TSDB_DATA_TYPE_BIGINT + tDiskColUpdateFloat, // TSDB_DATA_TYPE_FLOAT + tDiskColUpdateDouble, // TSDB_DATA_TYPE_DOUBLE + NULL, // TSDB_DATA_TYPE_VARCHAR + tDiskColUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP + NULL, // TSDB_DATA_TYPE_NCHAR + tDiskColUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT + tDiskColUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT + tDiskColUpdateUInt, // TSDB_DATA_TYPE_UINT + tDiskColUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT + NULL, // TSDB_DATA_TYPE_JSON + NULL, // TSDB_DATA_TYPE_VARBINARY + NULL, // TSDB_DATA_TYPE_DECIMAL + NULL, // TSDB_DATA_TYPE_BLOB + NULL, // TSDB_DATA_TYPE_MEDIUMBLOB +}; +static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { + if (pColVal->isNone || pColVal->isNull) { + pBuilder->sma.numOfNull++; + } else { + tDiskColUpdateSmaImpl[pBuilder->type](pBuilder, pColVal); + } +} + +static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColVal->type)) { code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); - if (code) goto _exit; + if (code) return code; pBuilder->offset += pColVal->value.nData; code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData); - if (code) goto _exit; + if (code) return code; } else { code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes); - if (code) goto _exit; + if (code) return code; } -_exit: return code; } static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 @@ -153,12 +262,10 @@ static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { / pBuilder->flag = HAS_NULL; } else { pBuilder->flag = HAS_VALUE; - code = tDiskColAddValue(pBuilder, pColVal); - if (code) goto _exit; + code = tDiskColPutValue(pBuilder, pColVal); + if (code) return code; } - pBuilder->nVal++; -_exit: return code; } static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE @@ -169,7 +276,7 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) goto _exit; + if (code) return code; memset(pBuilder->pBitMap, 0, nBit); SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); @@ -182,17 +289,15 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColAddValue(pBuilder, &cv); - if (code) goto _exit; + code = tDiskColPutValue(pBuilder, &cv); + if (code) return code; } - code = tDiskColAddValue(pBuilder, pColVal); - if (code) goto _exit; + code = tDiskColPutValue(pBuilder, pColVal); + if (code) return code; } } - pBuilder->nVal++; -_exit: return code; } static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL @@ -216,15 +321,14 @@ static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColAddValue(pBuilder, &cv); + code = tDiskColPutValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } } - pBuilder->nVal++; _exit: return code; @@ -259,14 +363,13 @@ static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColAddValue(pBuilder, &cv); + code = tDiskColPutValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } - pBuilder->nVal++; _exit: return code; @@ -288,13 +391,12 @@ static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { / memset(pBuilder->pBitMap, 255, nBit); SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } else { - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } - pBuilder->nVal++; _exit: return code; @@ -326,9 +428,8 @@ static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; - pBuilder->nVal++; _exit: return code; @@ -360,9 +461,8 @@ static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; - pBuilder->nVal++; _exit: return code; @@ -380,9 +480,8 @@ static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { / } else { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; - pBuilder->nVal++; _exit: return code; @@ -397,6 +496,18 @@ static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColV tDiskColAddVal6, // HAS_VALUE|HAS_NULL tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE }; +static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + if (pBuilder->calcSma) tDiskColUpdateSma(pBuilder, pColVal); + + code = tDiskColAddValImpl[pBuilder->type](pBuilder, pColVal); + if (code) return code; + + pBuilder->nVal++; + + return code; +} // SDiskDataBuilder ================================================ int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { @@ -477,7 +588,8 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); - code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); + code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg, + (calcSma && (pTColumn->flags & COL_SMA_ON))); if (code) return code; pBuilder->nBuilder++; @@ -528,10 +640,10 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche if (pColVal == NULL || pColVal->cid > pDCBuilder->cid) { SColVal cv = COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type); - code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, &cv); + code = tDiskColAddVal(pDCBuilder, &cv); if (code) return code; } else { - code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, pColVal); + code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; pColVal = tRowIterNext(&iter); } From 1cac65d8c0092261de98e5371282371eacf83051 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 19:08:33 +0800 Subject: [PATCH 19/59] refact more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 88 +------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 270 ++++++++------------- 2 files changed, 98 insertions(+), 260 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 5a235c5c82..8d25873036 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -143,96 +143,12 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { return code; } -static FORCE_INLINE void tDiskColUpdateSmaBool(SDiskColBuilder *pBuilder, SColVal *pColVal) { - // TODO -} -static FORCE_INLINE void tDiskColUpdateSmaTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int8_t val = *(int8_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateSmaSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int16_t val = *(int16_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateSmaInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t val = *(int32_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int64_t val = *(int64_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateFloat(SDiskColBuilder *pBuilder, SColVal *pColVal) { - float val = *(float *)&pColVal->value.val; - *(double *)&pBuilder->sma.sum += val; - if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; - if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateDouble(SDiskColBuilder *pBuilder, SColVal *pColVal) { - double val = *(double *)&pColVal->value.val; - *(double *)&pBuilder->sma.sum += val; - if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; - if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint8_t val = *(uint8_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint16_t val = *(uint16_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint32_t val = *(uint32_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint64_t val = *(uint64_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static void (*tDiskColUpdateSmaImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { - NULL, // TSDB_DATA_TYPE_NULL - tDiskColUpdateSmaBool, // TSDB_DATA_TYPE_BOOL - tDiskColUpdateSmaTinyint, // TSDB_DATA_TYPE_TINYINT - tDiskColUpdateSmaSmallint, // TSDB_DATA_TYPE_SMALLINT - tDiskColUpdateSmaInt, // TSDB_DATA_TYPE_INT - tDiskColUpdateBigint, // TSDB_DATA_TYPE_BIGINT - tDiskColUpdateFloat, // TSDB_DATA_TYPE_FLOAT - tDiskColUpdateDouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR - tDiskColUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR - tDiskColUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT - tDiskColUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT - tDiskColUpdateUInt, // TSDB_DATA_TYPE_UINT - tDiskColUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON - NULL, // TSDB_DATA_TYPE_VARBINARY - NULL, // TSDB_DATA_TYPE_DECIMAL - NULL, // TSDB_DATA_TYPE_BLOB - NULL, // TSDB_DATA_TYPE_MEDIUMBLOB -}; +extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { if (pColVal->isNone || pColVal->isNull) { pBuilder->sma.numOfNull++; } else { - tDiskColUpdateSmaImpl[pBuilder->type](pBuilder, pColVal); + tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c77af951af..0575b30526 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1492,185 +1492,107 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { return n; } +#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \ + do { \ + (SUM_V) += (VAL); \ + if (!(MINSET)) { \ + (MIN_V) = (VAL); \ + (MINSET) = 1; \ + } else if ((MIN_V) > (VAL)) { \ + (MIN_V) = (VAL); \ + } \ + if (!(MAXSET)) { \ + (MAX_V) = (VAL); \ + (MAXSET) = 1; \ + } else if ((MAX_V) < (VAL)) { \ + (MAX_V) = (VAL); \ + } \ + } while (0) + +static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + int8_t val = *(int8_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + int16_t val = *(int16_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int32_t val = *(int32_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int64_t val = *(int64_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + float val = *(float *)&pColVal->value.val; + SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + double val = *(double *)&pColVal->value.val; + SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint8_t val = *(uint8_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint16_t val = *(uint16_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + uint32_t val = *(uint32_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint64_t val = *(uint64_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = { + NULL, + tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL + tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT + tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT + tSmaUpdateInt, // TSDB_DATA_TYPE_INT + tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT + tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT + tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE + NULL, // TSDB_DATA_TYPE_VARCHAR + tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP + NULL, // TSDB_DATA_TYPE_NCHAR + tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT + tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT + tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT + tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT + NULL, // TSDB_DATA_TYPE_JSON + NULL, // TSDB_DATA_TYPE_VARBINARY + NULL, // TSDB_DATA_TYPE_DECIMAL + NULL, // TSDB_DATA_TYPE_BLOB + NULL, // TSDB_DATA_TYPE_MEDIUMBLOB +}; void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { - SColVal colVal; - SColVal *pColVal = &colVal; - - memset(pColAgg, 0, sizeof(*pColAgg)); - bool minAssigned = false; - bool maxAssigned = false; - *pColAgg = (SColumnDataAgg){.colId = pColData->cid}; - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - tColDataGetValue(pColData, iVal, pColVal); + uint8_t minSet = 0; + uint8_t maxSet = 0; - if (pColVal->isNone || pColVal->isNull) { + SColVal cv; + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + tColDataGetValue(pColData, iVal, &cv); + + if (cv.isNone || cv.isNull) { pColAgg->numOfNull++; } else { - switch (pColData->type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - break; - case TSDB_DATA_TYPE_TINYINT: { - int8_t i8 = *(int8_t *)&colVal.value.val; - pColAgg->sum += i8; - if (!minAssigned || pColAgg->min > i8) { - pColAgg->min = i8; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i8) { - pColAgg->max = i8; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t i16 = *(int16_t *)&colVal.value.val; - pColAgg->sum += i16; - if (!minAssigned || pColAgg->min > i16) { - pColAgg->min = i16; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i16) { - pColAgg->max = i16; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t i32 = *(int32_t *)&colVal.value.val; - pColAgg->sum += i32; - if (!minAssigned || pColAgg->min > i32) { - pColAgg->min = i32; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i32) { - pColAgg->max = i32; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t i64 = *(int64_t *)&colVal.value.val; - pColAgg->sum += i64; - if (!minAssigned || pColAgg->min > i64) { - pColAgg->min = i64; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i64) { - pColAgg->max = i64; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float f = *(float *)&colVal.value.val; - *(double *)(&pColAgg->sum) += f; - if (!minAssigned || *(double *)(&pColAgg->min) > f) { - *(double *)(&pColAgg->min) = f; - minAssigned = true; - } - if (!maxAssigned || *(double *)(&pColAgg->max) < f) { - *(double *)(&pColAgg->max) = f; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double d = *(double *)&colVal.value.val; - *(double *)(&pColAgg->sum) += d; - if (!minAssigned || *(double *)(&pColAgg->min) > d) { - *(double *)(&pColAgg->min) = d; - minAssigned = true; - } - if (!maxAssigned || *(double *)(&pColAgg->max) < d) { - *(double *)(&pColAgg->max) = d; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_VARCHAR: - break; - case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t ts = *(int64_t *)&colVal.value.val; - if (!minAssigned || pColAgg->min > ts) { - pColAgg->min = ts; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < ts) { - pColAgg->max = ts; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_NCHAR: - break; - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t u8 = *(uint8_t *)&colVal.value.val; - pColAgg->sum += u8; - if (!minAssigned || pColAgg->min > u8) { - pColAgg->min = u8; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < u8) { - pColAgg->max = u8; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t u16 = *(uint16_t *)&colVal.value.val; - pColAgg->sum += u16; - if (!minAssigned || pColAgg->min > u16) { - pColAgg->min = u16; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < u16) { - pColAgg->max = u16; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t u32 = *(uint32_t *)&colVal.value.val; - pColAgg->sum += u32; - if (!minAssigned || pColAgg->min > u32) { - pColAgg->min = u32; - minAssigned = true; - } - if (!minAssigned || pColAgg->max < u32) { - pColAgg->max = u32; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t u64 = *(uint64_t *)&colVal.value.val; - pColAgg->sum += u64; - if (!minAssigned || pColAgg->min > u64) { - pColAgg->min = u64; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < u64) { - pColAgg->max = u64; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_JSON: - break; - case TSDB_DATA_TYPE_VARBINARY: - break; - case TSDB_DATA_TYPE_DECIMAL: - break; - case TSDB_DATA_TYPE_BLOB: - break; - case TSDB_DATA_TYPE_MEDIUMBLOB: - break; - default: - ASSERT(0); - } + tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); } } } From d789fe6ac84300611965803ee43c483ce10297a7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 10:40:54 +0800 Subject: [PATCH 20/59] refact commit code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 483 ++++++++++++----------- 1 file changed, 253 insertions(+), 230 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a619b9f2e4..3daab532f3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -115,33 +115,32 @@ int32_t tRowInfoCmprFn(const void *p1, const void *p2) { int32_t tsdbBegin(STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; if (!pTsdb) return code; SMemTable *pMemTable; code = tsdbMemTableCreate(pTsdb, &pMemTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // lock - code = taosThreadRwlockWrlock(&pTsdb->rwLock); - if (code) { + if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pTsdb->mem = pMemTable; // unlock - code = taosThreadRwlockUnlock(&pTsdb->rwLock); - if (code) { + if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - return code; - -_err: - tsdbError("vgId:%d, tsdb begin failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } @@ -149,6 +148,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { if (!pTsdb) return 0; int32_t code = 0; + int32_t lino = 0; SCommitter commith; SMemTable *pMemTable = pTsdb->mem; @@ -164,76 +164,74 @@ int32_t tsdbCommit(STsdb *pTsdb) { // start commit code = tsdbStartCommit(pTsdb, &commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // commit impl code = tsdbCommitData(&commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbCommitDel(&commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // end commit code = tsdbEndCommit(&commith, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - return code; - -_err: - tsdbEndCommit(&commith, code); - tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbEndCommit(&commith, code); + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); - if (pCommitter->aDelIdx == NULL) { + if ((pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData)); - if (pCommitter->aDelData == NULL) { + if ((pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx)); - if (pCommitter->aDelIdxN == NULL) { + if ((pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } SDelFile *pDelFileR = pCommitter->fs.pDelFile; if (pDelFileR) { code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // prepare new SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0}; code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); - return code; - -_err: - tsdbError("vgId:%d, commit del start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); + } return code; } static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) { int32_t code = 0; + int32_t lino = 0; SDelData *pDelData; tb_uid_t suid; tb_uid_t uid; @@ -252,7 +250,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel uid = pDelIdx->uid; code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); } @@ -266,62 +264,63 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel for (; pDelData; pDelData = pDelData->pNext) { if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } // write code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // put delIdx if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d, commit table del failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->pDelFReader) { code = tsdbDelFReaderClose(&pCommitter->pDelFReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayDestroy(pCommitter->aDelIdx); taosArrayDestroy(pCommitter->aDelData); taosArrayDestroy(pCommitter->aDelIdxN); - return code; - -_err: - tsdbError("vgId:%d, commit del end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { int32_t code = 0; + int32_t lino = 0; if (suid) { if (pSkmInfo->suid == suid) { @@ -336,7 +335,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->uid = uid; tTSchemaDestroy(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); _exit: return code; @@ -344,6 +343,7 @@ _exit: static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { int32_t code = 0; + int32_t lino = 0; if (pCommitter->skmRow.pTSchema) { if (pCommitter->skmRow.suid == suid) { @@ -359,9 +359,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.uid = uid; tTSchemaDestroy(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); - if (code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); _exit: return code; @@ -369,6 +367,7 @@ _exit: static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; ASSERT(pCommitter->dReader.pBlockIdx); @@ -378,7 +377,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); ASSERT(pCommitter->dReader.mBlock.nItem > 0); } else { @@ -391,6 +390,7 @@ _exit: static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; pCommitter->pIter = NULL; tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); @@ -431,14 +431,14 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pIter->iStt = iStt; code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; pIter->iSttBlk = 0; SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); code = tsdbReadSttBlock(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; @@ -460,16 +460,18 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { } code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SDFileSet *pRSet = NULL; @@ -484,17 +486,17 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // data code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pCommitter->dReader.iBlockIdx = 0; if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { pCommitter->dReader.pBlockIdx = NULL; } @@ -531,7 +533,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } wSet.aSttF[wSet.nSttF - 1] = &fStt; code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); taosArrayClear(pCommitter->dWriter.aBlockIdx); taosArrayClear(pCommitter->dWriter.aSttBlk); @@ -541,18 +543,18 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { // open iter code = tsdbOpenCommitIter(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - return code; - -_err: - tsdbError("vgId:%d, commit file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { int32_t code = 0; + int32_t lino = 0; if (pBlockData->nRow == 0) return code; @@ -586,24 +588,25 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa dataBlk.nSubBlock++; code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // put SDataBlk code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // clear tBlockDataClear(pBlockData); - return code; - -_err: - tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { int32_t code = 0; + int32_t lino = 0; SSttBlk sstBlk; if (pBlockData->nRow == 0) return code; @@ -626,114 +629,117 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray // write code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // push SSttBlk if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // clear tBlockDataClear(pBlockData); - return code; - -_err: - tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; // write aBlockIdx code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // write aSttBlk code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // update file header code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // upsert SDFileSet code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // close and sync code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->dReader.pReader) { code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { int32_t code = 0; + int32_t lino = 0; while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - return code; - -_err: - tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); static int32_t tsdbCommitFileData(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; // commit file data start code = tsdbCommitFileDataStart(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // impl code = tsdbCommitFileDataImpl(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // commit file data end code = tsdbCommitFileDataEnd(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - tsdbDataFReaderClose(&pCommitter->dReader.pReader); - tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbDataFReaderClose(&pCommitter->dReader.pReader); + tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); + } return code; } // ---------------------------------------------------------------------------- static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; memset(pCommitter, 0, sizeof(*pCommitter)); ASSERT(pTsdb->mem && pTsdb->imem == NULL); @@ -754,30 +760,31 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFSCopy(pTsdb, &pCommitter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; // reader pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pCommitter->dReader.bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); // merger for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { @@ -785,33 +792,36 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pIter->bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } // writer pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pCommitter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pCommitter->dWriter.bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataCreate(&pCommitter->dWriter.bDatal); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); _exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } @@ -839,7 +849,9 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { } static int32_t tsdbCommitData(SCommitter *pCommitter) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; @@ -848,30 +860,29 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { // start ==================== code = tsdbCommitDataStart(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // impl ==================== pCommitter->nextKey = pMemTable->minKey; while (pCommitter->nextKey < TSKEY_MAX) { code = tsdbCommitFileData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // end ==================== tsdbCommitDataEnd(pCommitter); _exit: - tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); - return code; - -_err: - tsdbCommitDataEnd(pCommitter); - tsdbError("vgId:%d, commit data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDel(SCommitter *pCommitter) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; @@ -882,7 +893,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { // start code = tsdbCommitDelStart(pCommitter); if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // impl @@ -918,7 +929,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_mem_del: code = tsdbCommitTableDel(pCommitter, pTbData, NULL); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iTbData++; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; @@ -926,7 +937,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_disk_del: code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iDelIdx++; pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; @@ -934,7 +945,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_mem_and_disk_del: code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iTbData++; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; @@ -945,28 +956,28 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { // end code = tsdbCommitDelEnd(pCommitter); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); _exit: - tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); - return code; - -_err: - tsdbError("vgId:%d, commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); + } return code; } static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; ASSERT(eno == 0); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // lock taosThreadRwlockWrlock(&pTsdb->rwLock); @@ -975,7 +986,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { code = tsdbFSCommit2(pTsdb, &pCommitter->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pTsdb->imem = NULL; @@ -987,16 +998,12 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); - // if (pCommitter->toMerge) { - // code = tsdbMerge(pTsdb); - // if (code) goto _err; - // } - - tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); - return code; - -_err: - tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); + } return code; } @@ -1008,6 +1015,7 @@ static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; if (pCommitter->pIter) { SDataIter *pIter = pCommitter->pIter; @@ -1050,7 +1058,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); code = tsdbReadSttBlock(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; @@ -1085,11 +1093,16 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { } _exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SBlockData *pBlockData = &pCommitter->dWriter.bData; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; @@ -1098,13 +1111,13 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) while (pRowInfo) { ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo) { @@ -1119,29 +1132,31 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBlockData->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; SBlockData *pBDataR = &pCommitter->dReader.bData; SBlockData *pBDataW = &pCommitter->dWriter.bData; code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); tBlockDataClear(pBDataW); int32_t iRow = 0; @@ -1152,7 +1167,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); if (c < 0) { code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iRow++; if (iRow < pBDataR->nRow) { @@ -1163,13 +1178,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) } else if (c > 0) { ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo) { @@ -1186,13 +1201,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBDataW->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } while (pRow) { code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iRow++; if (iRow < pBDataR->nRow) { @@ -1203,22 +1218,24 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBDataW->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); @@ -1237,7 +1254,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { if (c < 0) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1247,13 +1264,13 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { } } else if (c > 0) { code = tsdbCommitAheadBlock(pCommitter, pDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; } else { code = tsdbCommitMergeBlock(pCommitter, pDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1268,7 +1285,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { while (pDataBlk) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1279,25 +1296,25 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { } code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; + int32_t lino = 0; SBlockData *pBDatal = &pCommitter->dWriter.bDatal; if (pBDatal->suid || pBDatal->uid) { if ((pBDatal->suid != id.suid) || (id.suid == 0)) { code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); tBlockDataReset(pBDatal); } } @@ -1306,42 +1323,48 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; SBlockData *pBData = &pCommitter->dWriter.bData; SBlockData *pBDatal = &pCommitter->dWriter.bDatal; TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; code = tsdbInitLastBlockIfNeed(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pBDatal->nRow >= pCommitter->maxRow) { code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } - return code; - -_err: +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; + int32_t lino = 0; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { @@ -1354,7 +1377,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pCommitter->toLastOnly) { pBData = &pCommitter->dWriter.bDatal; code = tsdbInitLastBlockIfNeed(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { pBData = &pCommitter->dWriter.bData; ASSERT(pBData->nRow == 0); @@ -1364,15 +1387,15 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { STSchema *pTSchema = NULL; if (pRowInfo->row.type == 0) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; } code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { @@ -1382,11 +1405,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pBData->nRow >= pCommitter->maxRow) { if (pCommitter->toLastOnly) { code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } } @@ -1394,23 +1417,23 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (!pCommitter->toLastOnly && pBData->nRow) { if (pBData->nRow > pCommitter->minRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { code = tsdbAppendLastBlock(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } _exit: - return code; - -_err: - tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; SRowInfo *pRowInfo; TABLEID id = {0}; @@ -1420,36 +1443,36 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { id.uid = pRowInfo->uid; code = tsdbMoveCommitData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // start tMapDataReset(&pCommitter->dWriter.mBlock); // impl code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* merge with data in .data file */ code = tsdbMergeTableData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* handle remain table data */ code = tsdbCommitTableData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // end if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } } @@ -1457,15 +1480,15 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { id.suid = INT64_MAX; id.uid = INT64_MAX; code = tsdbMoveCommitData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } From 9624073d8447963b4dc9204ab5f5018c28d8a420 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 10:46:05 +0800 Subject: [PATCH 21/59] refact more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 64 +++++++++++++++--------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3daab532f3..1e2a3d5b52 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -139,7 +139,7 @@ int32_t tsdbBegin(STsdb *pTsdb) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -180,7 +180,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { _exit: if (code) { tsdbEndCommit(&commith, code); - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -222,7 +222,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); } @@ -280,7 +280,8 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel _exit: if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -313,7 +314,8 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -464,7 +466,8 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -547,7 +550,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -599,7 +602,7 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -642,7 +645,7 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -678,7 +681,8 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -703,7 +707,8 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -729,7 +734,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbDataFReaderClose(&pCommitter->dReader.pReader); tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); } @@ -767,7 +772,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -820,7 +825,8 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -874,7 +880,7 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -960,7 +966,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); } @@ -1000,7 +1006,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); } @@ -1094,7 +1100,8 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1141,7 +1148,8 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1227,7 +1235,8 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1301,7 +1310,8 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1328,7 +1338,8 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1357,7 +1368,8 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1426,7 +1438,8 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1488,7 +1501,8 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } From da2150a15b3a99c668e9ebb4bd40045ef51cac9d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 16:45:38 +0800 Subject: [PATCH 22/59] more code 1 --- source/dnode/vnode/src/inc/tsdb.h | 14 ++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 192 ++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 14 -- 3 files changed, 120 insertions(+), 100 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index dc218e86e0..36aa5c0804 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -686,6 +686,20 @@ struct SDiskData { SArray *aDiskCol; // SArray }; +struct SDiskDataBuilder { + int64_t suid; + int64_t uid; + int32_t nRow; + uint8_t cmprAlg; + uint8_t calcSma; + SCompressor *pUidC; + SCompressor *pVerC; + SCompressor *pKeyC; + int32_t nBuilder; + SArray *aBuilder; // SArray + uint8_t *aBuf[2]; +}; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1e2a3d5b52..0ff9ae5085 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -70,12 +70,12 @@ typedef struct { int8_t toLastOnly; }; struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; // SArray - SArray *aSttBlk; // SArray - SMapData mBlock; // SMapData - SBlockData bData; - SBlockData bDatal; + SDataFWriter *pWriter; + SArray *aBlockIdx; // SArray + SArray *aSttBlk; // SArray + SMapData mBlock; // SMapData + SBlockData bData; + SDiskDataBuilder *pBuilder; } dWriter; SSkmInfo skmTable; SSkmInfo skmRow; @@ -139,7 +139,7 @@ int32_t tsdbBegin(STsdb *pTsdb) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -180,7 +180,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { _exit: if (code) { tsdbEndCommit(&commith, code); - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -222,7 +222,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); } @@ -280,7 +280,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel _exit: if (code) { - tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -314,7 +314,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -466,7 +466,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -542,7 +542,6 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); - tBlockDataReset(&pCommitter->dWriter.bDatal); // open iter code = tsdbOpenCommitIter(pCommitter); @@ -550,7 +549,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -602,7 +601,7 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -645,7 +644,7 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -681,7 +680,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -707,7 +706,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -734,7 +733,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbDataFReaderClose(&pCommitter->dReader.pReader); tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); } @@ -772,7 +771,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -820,12 +819,12 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataCreate(&pCommitter->dWriter.bData); TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataCreate(&pCommitter->dWriter.bDatal); + code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -849,7 +848,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); - tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); + tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } @@ -880,7 +879,7 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -966,7 +965,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); } @@ -1006,7 +1005,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); } @@ -1100,7 +1099,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1148,7 +1147,7 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1235,7 +1234,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1310,7 +1309,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1320,25 +1319,26 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; int32_t lino = 0; - SBlockData *pBDatal = &pCommitter->dWriter.bDatal; - if (pBDatal->suid || pBDatal->uid) { - if ((pBDatal->suid != id.suid) || (id.suid == 0)) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; + if (pBuilder->suid || pBuilder->uid) { + if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { + // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, + // pCommitter->cmprAlg); // todo TSDB_CHECK_CODE(code, lino, _exit); - tBlockDataReset(pBDatal); + // tBlockDataReset(pBDatal); } } - if (!pBDatal->suid && !pBDatal->uid) { + if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); + // code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); todo TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1349,7 +1349,6 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { int32_t lino = 0; SBlockData *pBData = &pCommitter->dWriter.bData; - SBlockData *pBDatal = &pCommitter->dWriter.bDatal; TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; code = tsdbInitLastBlockIfNeed(pCommitter, id); @@ -1357,18 +1356,20 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); - code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); + + code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); TSDB_CHECK_CODE(code, lino, _exit); - if (pBDatal->nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, NULL /*TODO */, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); TSDB_CHECK_CODE(code, lino, _exit); } } _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1385,60 +1386,79 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pRowInfo == NULL) goto _exit; - SBlockData *pBData; if (pCommitter->toLastOnly) { - pBData = &pCommitter->dWriter.bDatal; - code = tsdbInitLastBlockIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - pBData = &pCommitter->dWriter.bData; - ASSERT(pBData->nRow == 0); - } + // init the data if need - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - pTSchema = pCommitter->skmRow.pTSchema; - } - - code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - - if (pBData->nRow >= pCommitter->maxRow) { - if (pCommitter->toLastOnly) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); - } else { + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbNextCommitRow(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { + // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, + // pCommitter->cmprAlg); (todo) + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } else { + SBlockData *pBData = &pCommitter->dWriter.bData; + ASSERT(pBData->nRow == 0); + + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + TSDB_CHECK_CODE(code, lino, _exit); + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbNextCommitRow(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pBData->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); TSDB_CHECK_CODE(code, lino, _exit); } } - } - if (!pCommitter->toLastOnly && pBData->nRow) { - if (pBData->nRow > pCommitter->minRow) { - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbAppendLastBlock(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); + if (pBData->nRow) { + if (pBData->nRow > pCommitter->minRow) { + code = + tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbAppendLastBlock(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); + } } } _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1495,13 +1515,13 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); + // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + // pCommitter->cmprAlg); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 8d25873036..8c7933057d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -34,20 +34,6 @@ struct SDiskColBuilder { uint8_t *aBuf[1]; }; -struct SDiskDataBuilder { - int64_t suid; - int64_t uid; - int32_t nRow; - uint8_t cmprAlg; - uint8_t calcSma; - SCompressor *pUidC; - SCompressor *pVerC; - SCompressor *pKeyC; - int32_t nBuilder; - SArray *aBuilder; // SArray - uint8_t *aBuf[2]; -}; - // SDiskColBuilder ================================================ #define tDiskColBuilderCreate() \ (SDiskColBuilder) { 0 } From 8ff1eb41d1a17614bdc4382444a82d1e83c0e4ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 17:13:14 +0800 Subject: [PATCH 23/59] more code --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 58 +++++++++++++++++++----- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 36aa5c0804..618364d20f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -698,6 +698,7 @@ struct SDiskDataBuilder { int32_t nBuilder; SArray *aBuilder; // SArray uint8_t *aBuf[2]; + SDiskData dd; }; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 0ff9ae5085..70251933d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -649,6 +649,45 @@ _exit: return code; } +static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilder, SArray *aSttBlk) { + int32_t code = 0; + int32_t lino = 0; + + if (pBuilder->nRow == 0) return code; + + SSttBlk sttBlk = {.suid = pBuilder->suid, + .minUid = 0, // todo + .maxUid = 0, // todo + .minKey = 0, // todo + .maxKey = 0, // todo + .minVer = 0, // todo + .maxVer = 0, // todo + .nRow = pBuilder->nRow}; + + // gnrt + code = tGnrtDiskData(pBuilder, &pBuilder->dd); + TSDB_CHECK_CODE(code, lino, _exit); + + // write + // code = tsdbWriteDiskData(pWriter, &pBuilder->dd); + TSDB_CHECK_CODE(code, lino, _exit); + + // push + if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // clear (todo) + // tDiskDataBuilderClear(pBuilder); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; int32_t lino = 0; @@ -1315,24 +1354,22 @@ _exit: return code; } -static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { +static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; int32_t lino = 0; SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; if (pBuilder->suid || pBuilder->uid) { if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { - // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, - // pCommitter->cmprAlg); // todo + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); - // tBlockDataReset(pBDatal); } } if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - // code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); todo + code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1351,7 +1388,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { SBlockData *pBData = &pCommitter->dWriter.bData; TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - code = tsdbInitLastBlockIfNeed(pCommitter, id); + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { @@ -1361,8 +1398,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, NULL /*TODO */, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -1409,8 +1445,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { } if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, - // pCommitter->cmprAlg); (todo) + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -1515,8 +1550,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); - // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - // pCommitter->cmprAlg); + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); _exit: From b682fdd261282e88d579b8f2a7bfda2fd20499b7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 17:57:38 +0800 Subject: [PATCH 24/59] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 70251933d5..2817e26078 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1550,7 +1550,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); _exit: From f24c8fbe560e644e7e95cc2b050acb5bc9df9517 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 19:03:29 +0800 Subject: [PATCH 25/59] make it compile --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 34 +++++++++++----------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 8c7933057d..2c1d6d2149 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -131,7 +131,7 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { - if (pColVal->isNone || pColVal->isNull) { + if (!COL_VAL_IS_VALUE(pColVal)) { pBuilder->sma.numOfNull++; } else { tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); @@ -158,9 +158,9 @@ static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag = HAS_NONE; - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { pBuilder->flag = HAS_NULL; } else { pBuilder->flag = HAS_VALUE; @@ -173,7 +173,7 @@ static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { / static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE int32_t code = 0; - if (!pColVal->isNone) { + if (!COL_VAL_IS_NONE(pColVal)) { // bit map int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); @@ -184,7 +184,7 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); // value - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { pBuilder->flag |= HAS_NULL; } else { pBuilder->flag |= HAS_VALUE; @@ -205,12 +205,12 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL int32_t code = 0; - if (!pColVal->isNull) { + if (!COL_VAL_IS_NULL(pColVal)) { int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag |= HAS_NONE; memset(pBuilder->pBitMap, 255, nBit); @@ -238,12 +238,12 @@ _exit: static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; @@ -279,8 +279,8 @@ _exit: static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE int32_t code = 0; - if (pColVal->isNone || pColVal->isNull) { - if (pColVal->isNone) { + if (!COL_VAL_IS_VALUE(pColVal)) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag |= HAS_NONE; } else { pBuilder->flag |= HAS_NULL; @@ -306,7 +306,7 @@ _exit: static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NONE int32_t code = 0; - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { pBuilder->flag |= HAS_NULL; uint8_t *pBitMap = NULL; @@ -324,7 +324,7 @@ static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { / code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); @@ -339,7 +339,7 @@ _exit: static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag |= HAS_NONE; uint8_t *pBitMap = NULL; @@ -357,7 +357,7 @@ static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { / code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); @@ -375,9 +375,9 @@ static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { / code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); } else { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index ad99b04e5f..1d7eff9a08 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1589,7 +1589,7 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { tColDataGetValue(pColData, iVal, &cv); - if (cv.isNone || cv.isNull) { + if (!COL_VAL_IS_VALUE(&cv)) { pColAgg->numOfNull++; } else { tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); From 64434678cafe142410172543947b0c61d07b32c0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 25 Sep 2022 18:09:54 +0800 Subject: [PATCH 26/59] refact more code --- source/util/src/tcompression.c | 87 ++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 20 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 619ae8360d..3a6dbd4acc 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1001,38 +1001,49 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co *************************************************************************/ #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static struct { int8_t type; int32_t bytes; int8_t isVarLen; + int32_t (*initFn)(SCompressor *, int8_t type, int8_t cmprAlg); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); } DATA_TYPE_INFO[] = { - {TSDB_DATA_TYPE_NULL, 0, 0, NULL}, // TSDB_DATA_TYPE_NULL - {TSDB_DATA_TYPE_BOOL, 1, 0, tCompBool}, // TSDB_DATA_TYPE_BOOL - {TSDB_DATA_TYPE_TINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_TINYINT - {TSDB_DATA_TYPE_SMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_SMALLINT - {TSDB_DATA_TYPE_INT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_INT - {TSDB_DATA_TYPE_BIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_BIGINT - {TSDB_DATA_TYPE_FLOAT, 4, 0, tCompFloat}, // TSDB_DATA_TYPE_FLOAT - {TSDB_DATA_TYPE_DOUBLE, 8, 0, tCompDouble}, // TSDB_DATA_TYPE_DOUBLE - {TSDB_DATA_TYPE_VARCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARCHAR - {TSDB_DATA_TYPE_TIMESTAMP, 8, 0, tCompTimestamp}, // pTSDB_DATA_TYPE_TIMESTAMP - {TSDB_DATA_TYPE_NCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_NCHAR - {TSDB_DATA_TYPE_UTINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_UTINYINT - {TSDB_DATA_TYPE_USMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_USMALLINT - {TSDB_DATA_TYPE_UINT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_UINT - {TSDB_DATA_TYPE_UBIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_UBIGINT - {TSDB_DATA_TYPE_JSON, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_JSON - {TSDB_DATA_TYPE_VARBINARY, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARBINARY - {TSDB_DATA_TYPE_DECIMAL, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_DECIMAL - {TSDB_DATA_TYPE_BLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_BLOB - {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_MEDIUMBLOB + {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .initFn = NULL, .cmprFn = NULL}, // TSDB_DATA_TYPE_NULL + {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .initFn = tCompBoolInit, .cmprFn = tCompBool}, + {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .initFn = tCompFloatInit, .cmprFn = tCompFloat}, + {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .initFn = tCompDoubleInit, .cmprFn = tCompDouble}, + {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_TIMESTAMP, + .bytes = 8, + .isVarLen = 0, + .initFn = tCompTimestampInit, + .cmprFn = tCompTimestamp}, + {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, }; struct SCompressor { @@ -1072,6 +1083,12 @@ struct SCompressor { }; // Timestamp ===================================================== +static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompSetCopyMode(SCompressor *pCmprsor) { int32_t code = 0; @@ -1207,6 +1224,12 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; +static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1314,6 +1337,12 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } // Float ===================================================== +static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1377,6 +1406,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat } // Double ===================================================== +static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1440,6 +1475,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa } // Binary ===================================================== +static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1460,6 +1501,12 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nDa // Bool ===================================================== static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; +static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; From efacafde33d065c9942ad58d02d100b4d03f1afa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 10:47:24 +0800 Subject: [PATCH 27/59] refact more code --- include/util/tcompression.h | 4 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 20 +- source/util/src/tcompression.c | 316 ++++++++++++++------- 3 files changed, 219 insertions(+), 121 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index f22205d144..8f3d9634ec 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -130,8 +130,8 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); -int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); +int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 2c1d6d2149..9e7e4cefe0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -65,12 +65,12 @@ static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_ if (IS_VAR_DATA_TYPE(type)) { if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code; - code = tCompressorInit(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code; - code = tCompressorInit(pBuilder->pValC, type, cmprAlg); + code = tCompressStart(pBuilder->pValC, type, cmprAlg); if (code) return code; if (pBuilder->calcSma) { @@ -116,13 +116,13 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { // OFFSET if (IS_VAR_DATA_TYPE(pBuilder->type)) { - code = tCompGen(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); + code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); if (code) return code; } // VALUE if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { - code = tCompGen(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); + code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); if (code) return code; } @@ -457,15 +457,15 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB pBuilder->calcSma = calcSma; if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; - code = tCompressorInit(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code; - code = tCompressorInit(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code; - code = tCompressorInit(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; if (pBuilder->aBuilder == NULL) { @@ -586,16 +586,16 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { // UID if (pBuilder->uid == 0) { - code = tCompGen(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); + code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); if (code) return code; } // VERSION - code = tCompGen(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); + code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); if (code) return code; // TSKEY - code = tCompGen(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); + code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); if (code) return code; int32_t offset = 0; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 3a6dbd4acc..f1d9b860b8 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1001,49 +1001,157 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co *************************************************************************/ #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) -static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); static struct { int8_t type; int32_t bytes; int8_t isVarLen; - int32_t (*initFn)(SCompressor *, int8_t type, int8_t cmprAlg); + int32_t (*startFn)(SCompressor *, int8_t type, int8_t cmprAlg); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); + int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); } DATA_TYPE_INFO[] = { - {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .initFn = NULL, .cmprFn = NULL}, // TSDB_DATA_TYPE_NULL - {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .initFn = tCompBoolInit, .cmprFn = tCompBool}, - {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .initFn = tCompFloatInit, .cmprFn = tCompFloat}, - {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .initFn = tCompDoubleInit, .cmprFn = tCompDouble}, - {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_NULL, + .bytes = 0, + .isVarLen = 0, + .startFn = NULL, + .cmprFn = NULL, + .endFn = NULL}, // TSDB_DATA_TYPE_NULL + {.type = TSDB_DATA_TYPE_BOOL, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompBoolStart, + .cmprFn = tCompBool, + .endFn = tCompBoolEnd}, + {.type = TSDB_DATA_TYPE_TINYINT, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_SMALLINT, + .bytes = 2, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_INT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_BIGINT, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_FLOAT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompFloatStart, + .cmprFn = tCompFloat, + .endFn = tCompFloatEnd}, + {.type = TSDB_DATA_TYPE_DOUBLE, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompDoubleStart, + .cmprFn = tCompDouble, + .endFn = tCompDoubleEnd}, + {.type = TSDB_DATA_TYPE_VARCHAR, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8, .isVarLen = 0, - .initFn = tCompTimestampInit, - .cmprFn = tCompTimestamp}, - {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + .startFn = tCompTimestampStart, + .cmprFn = tCompTimestamp, + .endFn = tCompTimestampEnd}, + {.type = TSDB_DATA_TYPE_NCHAR, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_UTINYINT, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_USMALLINT, + .bytes = 2, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_UINT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_UBIGINT, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_JSON, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_VARBINARY, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_BLOB, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_MEDIUMBLOB, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, }; struct SCompressor { @@ -1083,9 +1191,13 @@ struct SCompressor { }; // Timestamp ===================================================== -static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - // TODO + pCmprsor->ts_prev_val = 0; + pCmprsor->ts_prev_delta = 0; + pCmprsor->ts_flag_p = NULL; + pCmprsor->aBuf[0][0] = 1; + pCmprsor->nBuf[0] = 1; return code; } @@ -1215,6 +1327,12 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t return code; } +static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + // TODO + return code; +} + // Integer ===================================================== #define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; @@ -1224,9 +1342,14 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; -static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - // TODO + pCmprsor->i_prev = 0; + pCmprsor->i_selector = 0; + pCmprsor->i_start = 0; + pCmprsor->i_end = 0; + pCmprsor->aBuf[0][0] = 0; + pCmprsor->nBuf[0] = 1; return code; } @@ -1336,13 +1459,22 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) return code; } -// Float ===================================================== -static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; // TODO return code; } +// Float ===================================================== +static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + pCmprsor->f_prev = 0; + pCmprsor->f_flag_p = NULL; + pCmprsor->aBuf[0][0] = 0; + pCmprsor->nBuf[0] = 1; + return code; +} + static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1405,13 +1537,22 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat return code; } -// Double ===================================================== -static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; // TODO return code; } +// Double ===================================================== +static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + pCmprsor->d_prev = 0; + pCmprsor->d_flag_p = NULL; + pCmprsor->aBuf[0][0] = 0; + pCmprsor->nBuf[0] = 1; + return code; +} + static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1474,13 +1615,19 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa return code; } -// Binary ===================================================== -static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; // TODO return code; } +// Binary ===================================================== +static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + pCmprsor->nBuf[0] = 0; + return code; +} + static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1498,12 +1645,18 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nDa return code; } +static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + // TODO + return code; +} + // Bool ===================================================== static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; -static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - // TODO + pCmprsor->nBuf[0] = 0; return code; } @@ -1531,6 +1684,12 @@ static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData return code; } +static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + // TODO + return code; +} + // SCompressor ===================================================== int32_t tCompressorCreate(SCompressor **ppCmprsor) { int32_t code = 0; @@ -1565,7 +1724,7 @@ int32_t tCompressorDestroy(SCompressor *pCmprsor) { return code; } -int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; pCmprsor->type = type; @@ -1573,82 +1732,21 @@ int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { pCmprsor->autoAlloc = 1; pCmprsor->nVal = 0; - switch (type) { - case TSDB_DATA_TYPE_TIMESTAMP: - pCmprsor->ts_prev_val = 0; - pCmprsor->ts_prev_delta = 0; - pCmprsor->ts_flag_p = NULL; - pCmprsor->aBuf[0][0] = 1; // For timestamp, 1 means compressed, 0 otherwise - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_BOOL: - pCmprsor->nBuf[0] = 0; - break; - case TSDB_DATA_TYPE_BINARY: - pCmprsor->nBuf[0] = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - pCmprsor->f_prev = 0; - pCmprsor->f_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_DOUBLE: - pCmprsor->d_prev = 0; - pCmprsor->d_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - pCmprsor->i_prev = 0; - pCmprsor->i_selector = 0; - pCmprsor->i_start = 0; - pCmprsor->i_end = 0; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - default: - break; + if (DATA_TYPE_INFO[type].startFn) { + DATA_TYPE_INFO[type].startFn(pCmprsor, type, cmprAlg); } return code; } -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - if (pCmprsor->nVal == 0) { - *ppData = NULL; - *nData = 0; - return code; - } + *ppData = NULL; + *nData = 0; - if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) { - code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1); - if (code) return code; - - int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]); - if (ret) { - pCmprsor->aBuf[1][0] = 0; - pCmprsor->nBuf[1] = ret + 1; - } else { - pCmprsor->aBuf[1][0] = 1; - memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]); - pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1; - } - - *ppData = pCmprsor->aBuf[1]; - *nData = pCmprsor->nBuf[1]; - } else { - *ppData = pCmprsor->aBuf[0]; - *nData = pCmprsor->nBuf[0]; + if (DATA_TYPE_INFO[pCmprsor->type].endFn) { + return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppData, nData); } return code; From a34cf50a83c39bfde85e0dee60ae9c543cc68f5d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 13:43:23 +0800 Subject: [PATCH 28/59] refact more code --- source/common/src/tdataformat.c | 609 +++++++++++++++++++------------- 1 file changed, 359 insertions(+), 250 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a5e8d4be66..e461e06158 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1135,288 +1135,397 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa _exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 +static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { int32_t code = 0; - - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag = HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag = HAS_NULL; - } else { - pColData->flag = HAS_VALUE; - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE - int32_t code = 0; - - if (!COL_VAL_IS_NONE(pColVal)) { - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - - if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag |= HAS_NULL; - } else { - pColData->flag |= HAS_VALUE; - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL - int32_t code = 0; - - if (!COL_VAL_IS_NULL(pColVal)) { - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; - - memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else { - pColData->flag |= HAS_VALUE; - - memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE - int32_t code = 0; - - if (COL_VAL_IS_NONE(pColVal)) { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - } else { - pColData->flag |= HAS_VALUE; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); - } - SET_BIT2(pBitMap, pColData->nVal, 2); - - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE - int32_t code = 0; - - if (!COL_VAL_IS_VALUE(pColVal)) { - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; - } else { - pColData->flag |= HAS_NULL; - } - - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } else { - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE - int32_t code = 0; - - if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag |= HAS_NULL; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); - } - SET_BIT2(pBitMap, pColData->nVal, 1); - - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - } else { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else { - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - } - } + pColData->flag = HAS_VALUE; code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - + if (code) return code; pColData->nVal++; - -_exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL +static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->flag = HAS_NONE; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->flag = HAS_NULL; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); - } - SET_BIT2(pBitMap, pColData->nVal, 0); + pColData->flag |= HAS_VALUE; - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - } else { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); } else { - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); } } + code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + pColData->flag |= HAS_NULL; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + pColData->flag |= HAS_VALUE; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + pColData->flag |= HAS_NONE; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_VALUE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pColData->nVal, 2); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NONE; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; pColData->nVal++; -_exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, - SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NULL; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pColData->nVal, 1); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pColData->nVal, 0); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - } else { - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - } code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; + if (code) return code; pColData->nVal++; -_exit: return code; } -static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = { - tColDataAppendValue0, // 0 - tColDataAppendValue1, // HAS_NONE - tColDataAppendValue2, // HAS_NULL - tColDataAppendValue3, // HAS_NULL|HAS_NONE - tColDataAppendValue4, // HAS_VALUE - tColDataAppendValue5, // HAS_VALUE|HAS_NONE - tColDataAppendValue6, // HAS_VALUE|HAS_NULL - tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { + {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 + {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE + {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL + {tColDataAppendValue30, tColDataAppendValue31, tColDataAppendValue32}, // HAS_NULL|HAS_NONE + {tColDataAppendValue40, tColDataAppendValue41, tColDataAppendValue42}, // HAS_VALUE + {tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE + {tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL + {tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); - return tColDataAppendValueImpl[pColData->flag](pColData, pColVal); + return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE From 30937189aa9ec7d854a54058a93a8f0c87ed6d02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 14:10:58 +0800 Subject: [PATCH 29/59] refact more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 416 +++++++++++---------- 1 file changed, 221 insertions(+), 195 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9e7e4cefe0..11f5c398ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -138,7 +138,7 @@ static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *p } } -static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { +static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColVal->type)) { @@ -155,256 +155,282 @@ static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { return code; } -static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 +static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_VALUE; + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_NONE; + return 0; +} +static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_NULL; + return 0; +} +static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag = HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - pBuilder->flag = HAS_NULL; - } else { - pBuilder->flag = HAS_VALUE; - code = tDiskColPutValue(pBuilder, pColVal); + // bit map + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + // value + pBuilder->flag |= HAS_VALUE; + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); if (code) return code; } - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (!COL_VAL_IS_NONE(pColVal)) { - // bit map - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; - code = tRealloc(&pBuilder->pBitMap, nBit); + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + pBuilder->flag |= HAS_NULL; + + return code; +} +static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + pBuilder->flag |= HAS_VALUE; + + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); if (code) return code; - - memset(pBuilder->pBitMap, 0, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - // value - if (COL_VAL_IS_NULL(pColVal)) { - pBuilder->flag |= HAS_NULL; - } else { - pBuilder->flag |= HAS_VALUE; - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) return code; - } - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) return code; - } } - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL +static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (!COL_VAL_IS_NULL(pColVal)) { - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) goto _exit; + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; - memset(pBuilder->pBitMap, 255, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else { - pBuilder->flag |= HAS_VALUE; + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - memset(pBuilder->pBitMap, 0, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) goto _exit; - } - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - } - } - -_exit: return code; } -static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + pBuilder->flag |= HAS_VALUE; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - } else { - pBuilder->flag |= HAS_VALUE; + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pBuilder->nVal, 2); - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); - } - SET_BIT2(pBitMap, pBuilder->nVal, 2); - - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) goto _exit; - } - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); + if (code) return code; } -_exit: - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE +static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (!COL_VAL_IS_VALUE(pColVal)) { - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag |= HAS_NONE; - } else { - pBuilder->flag |= HAS_NULL; - } + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) goto _exit; - - memset(pBuilder->pBitMap, 255, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - } else { - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - } - -_exit: return code; } -static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NULL(pColVal)) { - pBuilder->flag |= HAS_NULL; + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); - } - SET_BIT2(pBitMap, pBuilder->nVal, 1); - - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - } else { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - } - } - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - -_exit: return code; } -static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL +static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) { + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); - } - SET_BIT2(pBitMap, pBuilder->nVal, 0); + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - } else { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - } - } - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - -_exit: - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NULL; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pBuilder->nVal, 1); + + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pBuilder->nVal, 0); + + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); - } else { - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); - } - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - -_exit: - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { - tDiskColAddVal0, // 0 - tDiskColAddVal1, // HAS_NONE - tDiskColAddVal2, // HAS_NULL - tDiskColAddVal3, // HAS_NULL|HAS_NONE - tDiskColAddVal4, // HAS_VALUE - tDiskColAddVal5, // HAS_VALUE|HAS_NONE - tDiskColAddVal6, // HAS_VALUE|HAS_NULL - tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { + {tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0 + {tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE + {tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL + {tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE + {tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE + {tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE + {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL + {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE }; static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (pBuilder->calcSma) tDiskColUpdateSma(pBuilder, pColVal); - code = tDiskColAddValImpl[pBuilder->type](pBuilder, pColVal); - if (code) return code; + if (tDiskColAddValImpl[pBuilder->type][pColVal->type]) { + code = tDiskColAddValImpl[pBuilder->type][pColVal->type](pBuilder, pColVal); + if (code) return code; + } pBuilder->nVal++; From 08feade5fe3a9da96268008b9ef9a98a4e857d04 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 14:24:35 +0800 Subject: [PATCH 30/59] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 18 ++++++++---------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 6 +++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 11f5c398ba..bbf4e3337c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -129,15 +129,6 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { return code; } -extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); -static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { - if (!COL_VAL_IS_VALUE(pColVal)) { - pBuilder->sma.numOfNull++; - } else { - tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); - } -} - static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; @@ -422,10 +413,17 @@ static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *p {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE }; +extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (pBuilder->calcSma) tDiskColUpdateSma(pBuilder, pColVal); + if (pBuilder->calcSma) { + if (COL_VAL_IS_VALUE(pColVal)) { + tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); + } else { + pBuilder->sma.numOfNull++; + } + } if (tDiskColAddValImpl[pBuilder->type][pColVal->type]) { code = tDiskColAddValImpl[pBuilder->type][pColVal->type](pBuilder, pColVal); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1d7eff9a08..cae113dc4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1589,10 +1589,10 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { tColDataGetValue(pColData, iVal, &cv); - if (!COL_VAL_IS_VALUE(&cv)) { - pColAgg->numOfNull++; - } else { + if (COL_VAL_IS_VALUE(&cv)) { tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); + } else { + pColAgg->numOfNull++; } } } From 787e166215551d657884a4e063f2fd30ee8e3361 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 17:45:37 +0800 Subject: [PATCH 31/59] refact more code --- source/util/src/tcompression.c | 556 ++++++++++++++++++++++++--------- 1 file changed, 409 insertions(+), 147 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index f1d9b860b8..65e8cabe32 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1159,8 +1159,9 @@ struct SCompressor { int8_t cmprAlg; int8_t autoAlloc; int32_t nVal; - uint8_t *aBuf[2]; - int64_t nBuf[2]; + uint8_t *pBuf; + int32_t nBuf; + uint8_t *aBuf[1]; union { // Timestamp ---- struct { @@ -1193,93 +1194,91 @@ struct SCompressor { // Timestamp ===================================================== static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - pCmprsor->ts_prev_val = 0; - pCmprsor->ts_prev_delta = 0; - pCmprsor->ts_flag_p = NULL; - pCmprsor->aBuf[0][0] = 1; - pCmprsor->nBuf[0] = 1; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 1; + return code; } -static int32_t tCompSetCopyMode(SCompressor *pCmprsor) { +static int32_t tCompSwitchToCopyMode(SCompressor *pCmprsor) { int32_t code = 0; - if (pCmprsor->nVal) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->nVal); - if (code) return code; - } - pCmprsor->nBuf[1] = 0; + if (pCmprsor->nVal == 0) goto _exit; - int64_t n = 1; - int64_t value; - int64_t delta; - uint64_t vZigzag; - while (n < pCmprsor->nBuf[0]) { - uint8_t aN[2]; - aN[0] = pCmprsor->aBuf[0][n] & 0xf; - aN[1] = pCmprsor->aBuf[0][n] >> 4; - - n++; - - for (int32_t i = 0; i < 2; i++) { - vZigzag = 0; - for (uint8_t j = 0; j < aN[i]; j++) { - vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (8 * j)); - n++; - } - - int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); - if (pCmprsor->nBuf[1] == 0) { - delta = 0; - value = delta_of_delta; - } else { - delta = delta_of_delta + delta; - value = delta + value; - } - - memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &value, sizeof(int64_t)); - pCmprsor->nBuf[1] += sizeof(int64_t); - - if (n >= pCmprsor->nBuf[0]) break; - } - } - - ASSERT(n == pCmprsor->nBuf[0]); - - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[1] + 1); - if (code) return code; - } - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]); - pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1]; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(int64_t) * pCmprsor->nVal + 1))) { + return code; } - pCmprsor->aBuf[0][0] = 0; + int32_t n = 1; + int32_t nBuf = 1; + int64_t value; + int64_t delta; + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t aN[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int32_t i = 0; i < 2; i++) { + uint64_t vZigzag = 0; + for (uint8_t j = 0; j < aN[i]; j++) { + vZigzag |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); + if (iVal) { + delta = delta_of_delta + delta; + value = delta + value; + } else { + delta = 0; + value = delta_of_delta; + } + + memcpy(pCmprsor->aBuf[0] + nBuf, &value, sizeof(value)); + nBuf += sizeof(int64_t); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + + ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nBuf + 1); + + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 0; return code; } static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; int64_t ts = *(int64_t *)pData; - ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(nData == 8); - if (pCmprsor->aBuf[0][0] == 1) { + if (pCmprsor->pBuf[0] == 1) { if (pCmprsor->nVal == 0) { pCmprsor->ts_prev_val = ts; pCmprsor->ts_prev_delta = -ts; } if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) { - code = tCompSetCopyMode(pCmprsor); + code = tCompSwitchToCopyMode(pCmprsor); if (code) return code; goto _copy_cmpr; } int64_t delta = ts - pCmprsor->ts_prev_val; if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) { - code = tCompSetCopyMode(pCmprsor); + code = tCompSwitchToCopyMode(pCmprsor); if (code) return code; goto _copy_cmpr; } @@ -1290,37 +1289,35 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t pCmprsor->ts_prev_delta = delta; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) { + return code; } - pCmprsor->ts_flag_p = pCmprsor->aBuf[0] + pCmprsor->nBuf[0]; - pCmprsor->nBuf[0]++; + pCmprsor->ts_flag_p = pCmprsor->pBuf + pCmprsor->nBuf; + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0] = 0; while (vZigzag) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff); + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0]++; vZigzag >>= 8; } } else { while (vZigzag) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff); + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0] += 0x10; vZigzag >>= 8; } } } else { _copy_cmpr: - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(ts)); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(ts)))) { + return code; } - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts)); - pCmprsor->nBuf[0] += sizeof(ts); + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, &ts, sizeof(ts)); + pCmprsor->nBuf += sizeof(ts); } pCmprsor->nVal++; @@ -1329,7 +1326,34 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= sizeof(int64_t) * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 1) { + code = tCompSwitchToCopyMode(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } @@ -1344,12 +1368,24 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; + pCmprsor->i_prev = 0; pCmprsor->i_selector = 0; pCmprsor->i_start = 0; pCmprsor->i_end = 0; - pCmprsor->aBuf[0][0] = 0; - pCmprsor->nBuf[0] = 1; + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + + return code; +} + +static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + // ASSERT(0); return code; } @@ -1363,26 +1399,18 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) switch (pCmprsor->type) { case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: val = *(int8_t *)pData; break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: val = *(int16_t *)pData; break; case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: val = *(int32_t *)pData; break; case TSDB_DATA_TYPE_BIGINT: - val = *(int64_t *)pData; - break; - case TSDB_DATA_TYPE_UTINYINT: - val = *(uint8_t *)pData; - break; - case TSDB_DATA_TYPE_USMALLINT: - val = *(uint16_t *)pData; - break; - case TSDB_DATA_TYPE_UINT: - val = *(uint32_t *)pData; - break; case TSDB_DATA_TYPE_UBIGINT: val = *(int64_t *)pData; break; @@ -1392,14 +1420,16 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { - // TODO + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; goto _copy_cmpr; } int64_t diff = val - pCmprsor->i_prev; uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff); if (vZigzag >= SIMPLE8B_MAX) { - // TODO + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; goto _copy_cmpr; } @@ -1424,12 +1454,12 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t)); + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)); if (code) return code; } - uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]); - pCmprsor->nBuf[0] += sizeof(uint64_t); + uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); + pCmprsor->nBuf += sizeof(uint64_t); bp[0] = pCmprsor->i_selector; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; for (int32_t iVal = 0; iVal < nEle; iVal++) { @@ -1448,11 +1478,11 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } } else { _copy_cmpr: - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData); if (code) return code; - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); - pCmprsor->nBuf[0] += nData; + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData); + pCmprsor->nBuf += nData; } pCmprsor->nVal++; @@ -1461,20 +1491,107 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 0) { + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } // Float ===================================================== static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; + pCmprsor->f_prev = 0; pCmprsor->f_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; - pCmprsor->nBuf[0] = 1; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + return code; } +static int32_t tCompFloatSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(float) * pCmprsor->nVal + 1))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + union { + float f; + uint32_t u; + } val = {.u = 0}; + + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int8_t i = 0; i < 2; i++) { + uint8_t flag = flags[i]; + + uint32_t diff = 0; + int8_t nBytes = (flag & 0x7) + 1; + for (int j = 0; j < nBytes; j++) { + diff |= (((uint32_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + if (flag & 0x8) { + diff <<= (32 - nBytes * 8); + } + + val.u ^= diff; + + memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val)); + nBuf += sizeof(val); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 1; + return code; +} static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1507,13 +1624,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat if (nBytes == 0) nBytes++; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 9))) { + return code; } - pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; - pCmprsor->nBuf[0]++; + pCmprsor->f_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf]; + pCmprsor->nBuf++; if (clz < ctz) { pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1)); @@ -1528,8 +1644,8 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat } } for (; nBytes; nBytes--) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff); + pCmprsor->nBuf++; diff >>= BITS_PER_BYTE; } pCmprsor->nVal++; @@ -1539,20 +1655,107 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= sizeof(float) * pCmprsor->nVal + 1) { + code = tCompFloatSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } // Double ===================================================== static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; + pCmprsor->d_prev = 0; pCmprsor->d_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; - pCmprsor->nBuf[0] = 1; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + return code; } +static int32_t tCompDoubleSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(double) * pCmprsor->nVal + 1))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + union { + double f; + uint64_t u; + } val = {.u = 0}; + + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int8_t i = 0; i < 2; i++) { + uint8_t flag = flags[i]; + + uint64_t diff = 0; + int8_t nBytes = (flag & 0x7) + 1; + for (int j = 0; j < nBytes; j++) { + diff |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + if (flag & 0x8) { + diff <<= (64 - nBytes * 8); + } + + val.u ^= diff; + + memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val)); + nBuf += sizeof(val); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 1; + return code; +} static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1585,13 +1788,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa if (nBytes == 0) nBytes++; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) { + return code; } - pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; - pCmprsor->nBuf[0]++; + pCmprsor->d_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf]; + pCmprsor->nBuf++; if (clz < ctz) { pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1)); @@ -1606,8 +1808,8 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa } } for (; nBytes; nBytes--) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff); + pCmprsor->nBuf++; diff >>= BITS_PER_BYTE; } pCmprsor->nVal++; @@ -1617,28 +1819,53 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= sizeof(double) * pCmprsor->nVal + 1) { + code = tCompDoubleSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } // Binary ===================================================== static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { - int32_t code = 0; - pCmprsor->nBuf[0] = 0; - return code; + pCmprsor->nBuf = 1; + return 0; } static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; if (nData) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData))) { + return code; } - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); - pCmprsor->nBuf[0] += nData; + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData); + pCmprsor->nBuf += nData; } pCmprsor->nVal++; @@ -1647,7 +1874,25 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nDa static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf == 1) return code; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf))) { + return code; + } + + int32_t szComp = + LZ4_compress_default(pCmprsor->pBuf + 1, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf - 1, pCmprsor->nBuf - 1); + if (szComp && szComp < pCmprsor->nBuf - 1) { + pCmprsor->aBuf[0][0] = 1; + *ppData = pCmprsor->aBuf[0]; + *nData = szComp + 1; + } else { + pCmprsor->pBuf[0] = 0; + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } + return code; } @@ -1655,9 +1900,8 @@ static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { - int32_t code = 0; - pCmprsor->nBuf[0] = 0; - return code; + pCmprsor->nBuf = 0; + return 0; } static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) { @@ -1665,19 +1909,18 @@ static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData bool vBool = *(int8_t *)pData; - int32_t mod4 = pCmprsor->nVal & 3; + int32_t mod4 = (pCmprsor->nVal & 3); if (mod4 == 0) { - pCmprsor->nBuf[0]++; + pCmprsor->nBuf++; - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf))) { + return code; } - pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0; + pCmprsor->pBuf[pCmprsor->nBuf - 1] = 0; } if (vBool) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4]; + pCmprsor->pBuf[pCmprsor->nBuf - 1] |= BOOL_CMPR_TABLE[mod4]; } pCmprsor->nVal++; @@ -1686,7 +1929,30 @@ static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } @@ -1697,23 +1963,17 @@ int32_t tCompressorCreate(SCompressor **ppCmprsor) { *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor)); if ((*ppCmprsor) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + return code; } - code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024); - if (code) { - taosMemoryFree(*ppCmprsor); - *ppCmprsor = NULL; - goto _exit; - } - -_exit: return code; } int32_t tCompressorDestroy(SCompressor *pCmprsor) { int32_t code = 0; + tFree(pCmprsor->pBuf); + int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { tFree(pCmprsor->aBuf[iBuf]); @@ -1745,6 +2005,8 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nDa *ppData = NULL; *nData = 0; + if (pCmprsor->nVal == 0) return code; + if (DATA_TYPE_INFO[pCmprsor->type].endFn) { return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppData, nData); } From 4fdb37e887d005cc7c605b51d0bc6f8909532425 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 17:56:47 +0800 Subject: [PATCH 32/59] more code --- source/util/src/tcompression.c | 104 +++++++++++---------------------- 1 file changed, 34 insertions(+), 70 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 65e8cabe32..ef0a45f3a5 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1191,6 +1191,26 @@ struct SCompressor { }; }; +static int32_t tTwoStageComp(SCompressor *pCmprsor, int32_t *szComp) { + int32_t code = 0; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (*szComp && *szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *szComp += 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *szComp = pCmprsor->nBuf + 1; + } + + return code; +} + // Timestamp ===================================================== static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; @@ -1205,7 +1225,7 @@ static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cm return code; } -static int32_t tCompSwitchToCopyMode(SCompressor *pCmprsor) { +static int32_t tCompTSSwitchToCopy(SCompressor *pCmprsor) { int32_t code = 0; if (pCmprsor->nVal == 0) goto _exit; @@ -1271,14 +1291,14 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t } if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) { - code = tCompSwitchToCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; goto _copy_cmpr; } int64_t delta = ts - pCmprsor->ts_prev_val; if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) { - code = tCompSwitchToCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; goto _copy_cmpr; } @@ -1328,25 +1348,14 @@ static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t code = 0; if (pCmprsor->nBuf >= sizeof(int64_t) * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 1) { - code = tCompSwitchToCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1498,20 +1507,9 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_ } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1662,20 +1660,9 @@ static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int3 } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1826,20 +1813,9 @@ static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1931,21 +1907,9 @@ static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32 int32_t code = 0; if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; From 24e899f88b0a43e9cd618ad9126e2ee04e6b7b8d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 15:40:06 +0800 Subject: [PATCH 33/59] more code --- source/util/src/tcompression.c | 49 ++++++++++------------------------ 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index ef0a45f3a5..7da55a7bdc 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1032,12 +1032,7 @@ static struct { int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); } DATA_TYPE_INFO[] = { - {.type = TSDB_DATA_TYPE_NULL, - .bytes = 0, - .isVarLen = 0, - .startFn = NULL, - .cmprFn = NULL, - .endFn = NULL}, // TSDB_DATA_TYPE_NULL + {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .startFn = NULL, .cmprFn = NULL, .endFn = NULL}, {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, @@ -1394,7 +1389,13 @@ static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { int32_t code = 0; - // ASSERT(0); + + if (pCmprsor->nVal == 0) goto _exit; + + // todo + +_exit: + pCmprsor->pBuf[0] = 1; return code; } @@ -1404,29 +1405,8 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); if (pCmprsor->aBuf[0][0] == 0) { - int64_t val; - - switch (pCmprsor->type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - val = *(int8_t *)pData; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - val = *(int16_t *)pData; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - val = *(int32_t *)pData; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - val = *(int64_t *)pData; - break; - default: - ASSERT(0); - break; - } + int64_t val = 0; + memcpy(&val, pData, nData); // little-endian only if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { code = tCompIntSwitchToCopy(pCmprsor); @@ -1445,16 +1425,16 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0; pCmprsor->i_prev = val; - while (1) { + for (;;) { int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) { pCmprsor->i_selector = BIT_TO_SELECTOR[nBit]; } - pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag; pCmprsor->i_aBitN[pCmprsor->i_end] = nBit; + pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; break; } else { while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { @@ -1462,9 +1442,8 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { + return code; } uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); From 3be4b1d4dfdf1061d0192216ba40a51fd506a3a4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 16:16:02 +0800 Subject: [PATCH 34/59] finish stream compression --- source/util/src/tcompression.c | 60 +++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 7da55a7bdc..133d1b29b7 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1392,7 +1392,40 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { if (pCmprsor->nVal == 0) goto _exit; - // todo + int32_t size = DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], size))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + int64_t vPrev = 0; + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint64_t b; + memcpy(&b, pCmprsor->pBuf + n, sizeof(b)); + n += sizeof(b); + + int32_t i_selector = (b & 0xf); + int32_t nEle = SELECTOR_TO_ELEMS[i_selector]; + uint8_t bits = BIT_PER_INTEGER[i_selector]; + for (int32_t iEle = 0; iEle < nEle; iEle++) { + uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); + int64_t diff = ZIGZAG_DECODE(int64_t, vZigzag); + vPrev = diff + vPrev; + + memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); + nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; + } + iVal += nEle; + ASSERT(iVal <= pCmprsor->nVal); + } + + ASSERT(n == pCmprsor->nBuf && nBuf == size); + + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = size; _exit: pCmprsor->pBuf[0] = 1; @@ -1480,6 +1513,31 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; + for (;;) { + int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; + if (nEle == 0) break; + + while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + pCmprsor->i_selector++; + } + nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { + return code; + } + + uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); + pCmprsor->nBuf += sizeof(uint64_t); + bp[0] = pCmprsor->i_selector; + uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; + for (int32_t iVal = 0; iVal < nEle; iVal++) { + bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); + pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + } + + pCmprsor->i_selector = 0; + } + if (pCmprsor->nBuf >= DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 0) { code = tCompIntSwitchToCopy(pCmprsor); if (code) return code; From dc0d687a06dca97045f9c69ffee258138a06a446 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 16:26:16 +0800 Subject: [PATCH 35/59] make it compile --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a76ce76b85..09b790ee81 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -440,7 +440,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pIter->iSttBlk = 0; SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; @@ -1370,8 +1370,9 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; - code = tBlockDataInit(pBDatal, &tid, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _exit; + code = + tDiskDataBuilderInit(pCommitter->dWriter.pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); + TSDB_CHECK_CODE(code, lino, _exit); } _exit: @@ -1519,11 +1520,11 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { // impl code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* merge with data in .data file */ code = tsdbMergeTableData(pCommitter, id); From a4d16f1c0029c245446fba6009b14baca9af1c0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 17:36:02 +0800 Subject: [PATCH 36/59] more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 6 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 7 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 101 +++++++++++++-------- source/util/src/tcompression.c | 15 ++- 5 files changed, 80 insertions(+), 51 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 8f3d9634ec..84c2922c9e 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -131,7 +131,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); -int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 264746e8f2..ca16b87e17 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -326,9 +326,9 @@ int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder); int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, uint8_t calcSma); -int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData); -int32_t tDiskDataDestroy(SDiskData *pDiskData); +int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); +int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData); // structs ======================= struct STsdbFS { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 09b790ee81..7bc335d034 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -665,7 +665,7 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde .nRow = pBuilder->nRow}; // gnrt - code = tGnrtDiskData(pBuilder, &pBuilder->dd); + // code = tGnrtDiskData(pBuilder, &pBuilder->dd); TSDB_CHECK_CODE(code, lino, _exit); // write @@ -1369,7 +1369,6 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; code = tDiskDataBuilderInit(pCommitter->dWriter.pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); @@ -1396,7 +1395,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); - code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); + code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { @@ -1435,7 +1434,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { pTSchema = pCommitter->skmRow.pTSchema; } - code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); + code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index bbf4e3337c..8cba73d353 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -31,9 +31,16 @@ struct SDiskColBuilder { SColumnDataAgg sma; uint8_t minSet; uint8_t maxSet; - uint8_t *aBuf[1]; + uint8_t *aBuf[2]; }; +// SDiskData ================================================ +static int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); + return code; +} + // SDiskColBuilder ================================================ #define tDiskColBuilderCreate() \ (SDiskColBuilder) { 0 } @@ -91,7 +98,7 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { .type = pBuilder->type, .smaOn = pBuilder->calcSma, .flag = pBuilder->flag, - .szOrigin = 0, // todo + .szOrigin = 0, .szBitmap = 0, .szOffset = 0, .szValue = 0, @@ -109,20 +116,27 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { nBit = BIT1_SIZE(pBuilder->nVal); } - pDiskCol->bCol.szBitmap = tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], 0, pBuilder->cmprAlg, - NULL, 0); // todo: alloc + code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES); + if (code) return code; + + code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); + if (code) return code; + + pDiskCol->bCol.szBitmap = + tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES, + pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); pDiskCol->pBit = pBuilder->aBuf[0]; } // OFFSET if (IS_VAR_DATA_TYPE(pBuilder->type)) { - code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); + code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL); if (code) return code; } // VALUE if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { - code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); + code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin); if (code) return code; } @@ -425,8 +439,8 @@ static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { } } - if (tDiskColAddValImpl[pBuilder->type][pColVal->type]) { - code = tDiskColAddValImpl[pBuilder->type][pColVal->type](pBuilder, pColVal); + if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) { + code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal); if (code) return code; } @@ -465,6 +479,7 @@ void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { tFree(pBuilder->aBuf[iBuf]); } + tDiskDataDestroy(&pBuilder->dd); taosMemoryFree(pBuilder); return NULL; @@ -474,6 +489,8 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB uint8_t calcSma) { int32_t code = 0; + ASSERT(pId->suid || pId->uid); + pBuilder->suid = pId->suid; pBuilder->uid = pId->uid; pBuilder->nRow = 0; @@ -512,9 +529,9 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB } } - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); - code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg, + code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg, (calcSma && (pTColumn->flags & COL_SMA_ON))); if (code) return code; @@ -524,14 +541,22 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB return code; } -int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { +int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) { + int32_t code = 0; + pBuilder->suid = 0; + pBuilder->uid = 0; + return code; +} + +int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { int32_t code = 0; + ASSERT(pBuilder->suid || pBuilder->uid); ASSERT(pId->suid == pBuilder->suid); // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { - ASSERT(!pBuilder->calcSma); + ASSERT(pBuilder->suid); for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); if (code) return code; @@ -564,14 +589,13 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche pColVal = tRowIterNext(&iter); } - if (pColVal == NULL || pColVal->cid > pDCBuilder->cid) { - SColVal cv = COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type); - code = tDiskColAddVal(pDCBuilder, &cv); - if (code) return code; - } else { + if (pColVal && pColVal->cid == pDCBuilder->cid) { code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; pColVal = tRowIterNext(&iter); + } else { + code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); + if (code) return code; } } pBuilder->nRow++; @@ -579,11 +603,14 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche return code; } -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData) { int32_t code = 0; ASSERT(pBuilder->nRow); + *ppDiskData = NULL; + + SDiskData *pDiskData = &pBuilder->dd; // reset SDiskData pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, .fmtVer = 0, @@ -598,6 +625,22 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { pDiskData->pUid = NULL; pDiskData->pVer = NULL; pDiskData->pKey = NULL; + + // UID + if (pBuilder->uid == 0) { + code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL); + if (code) return code; + } + + // VERSION + code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL); + if (code) return code; + + // TSKEY + code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL); + if (code) return code; + + // aDiskCol if (pDiskData->aDiskCol) { taosArrayClear(pDiskData->aDiskCol); } else { @@ -608,20 +651,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { } } - // UID - if (pBuilder->uid == 0) { - code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); - if (code) return code; - } - - // VERSION - code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); - if (code) return code; - - // TSKEY - code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); - if (code) return code; - int32_t offset = 0; for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); @@ -644,12 +673,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); } - return code; -} - -// SDiskData ================================================ -int32_t tDiskDataDestroy(SDiskData *pDiskData) { - int32_t code = 0; - pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); + *ppDiskData = pDiskData; return code; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 133d1b29b7..8c38b4f20d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -2000,16 +2000,23 @@ int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { return code; } -int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin) { int32_t code = 0; - *ppData = NULL; - *nData = 0; + *ppOut = NULL; + *nOut = 0; + if (nOrigin) { + if (DATA_TYPE_INFO[pCmprsor->type].isVarLen) { + *nOrigin = pCmprsor->nBuf - 1; + } else { + *nOrigin = pCmprsor->nVal * DATA_TYPE_INFO[pCmprsor->type].bytes; + } + } if (pCmprsor->nVal == 0) return code; if (DATA_TYPE_INFO[pCmprsor->type].endFn) { - return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppData, nData); + return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppOut, nOut); } return code; From a53575a1524a1f777a2bb180acd339312e76b917 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 17:57:57 +0800 Subject: [PATCH 37/59] more code --- source/dnode/vnode/src/inc/tsdb.h | 13 +++++++++++- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 24 +++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ca16b87e17..a928cc725d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -328,7 +328,8 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB uint8_t calcSma); int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo *pBlkInfo); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo); // structs ======================= struct STsdbFS { @@ -450,6 +451,15 @@ struct SSmaInfo { int32_t size; }; +typedef struct { + int64_t minUid; + int64_t maxUid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t minVer; + int64_t maxVer; +} SBlkInfo; + struct SDataBlk { TSDBKEY minKey; TSDBKEY maxKey; @@ -702,6 +712,7 @@ struct SDiskDataBuilder { SArray *aBuilder; // SArray uint8_t *aBuf[2]; SDiskData dd; + SBlkInfo bi; }; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 8cba73d353..d3ea4f1dda 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -496,6 +496,12 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB pBuilder->nRow = 0; pBuilder->cmprAlg = cmprAlg; pBuilder->calcSma = calcSma; + pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX, + .maxUid = INT64_MIN, + .minKey = TSDBKEY_MAX, + .maxKey = TSDBKEY_MIN, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN}; if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); @@ -554,6 +560,8 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS ASSERT(pBuilder->suid || pBuilder->uid); ASSERT(pId->suid == pBuilder->suid); + TSDBKEY key = TSDBROW_KEY(pRow); + // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { ASSERT(pBuilder->suid); @@ -567,16 +575,20 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); if (code) return code; } + if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid; + if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid; // version - int64_t version = TSDBROW_VERSION(pRow); - code = tCompress(pBuilder->pVerC, &version, sizeof(int64_t)); + code = tCompress(pBuilder->pVerC, &key.version, sizeof(int64_t)); if (code) return code; + if (pBuilder->bi.minVer > key.version) pBuilder->bi.minVer = key.version; + if (pBuilder->bi.maxVer < key.version) pBuilder->bi.maxVer = key.version; // TSKEY - TSKEY ts = TSDBROW_TS(pRow); - code = tCompress(pBuilder->pKeyC, &ts, sizeof(int64_t)); + code = tCompress(pBuilder->pKeyC, &key.ts, sizeof(int64_t)); if (code) return code; + if (tsdbKeyCmprFn(&pBuilder->bi.minKey, &key) > 0) pBuilder->bi.minKey = key; + if (tsdbKeyCmprFn(&pBuilder->bi.maxKey, &key) < 0) pBuilder->bi.maxKey = key; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); @@ -603,12 +615,13 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS return code; } -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData) { +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) { int32_t code = 0; ASSERT(pBuilder->nRow); *ppDiskData = NULL; + *ppBlkInfo = NULL; SDiskData *pDiskData = &pBuilder->dd; // reset SDiskData @@ -674,5 +687,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData) } *ppDiskData = pDiskData; + *ppBlkInfo = &pBuilder->bi; return code; } From 8bd8ff152390f1d7100c316342f2ca339076eb00 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 18:59:29 +0800 Subject: [PATCH 38/59] more code --- source/dnode/vnode/src/inc/tsdb.h | 15 ++- source/dnode/vnode/src/tsdb/tsdbCommit.c | 27 ++--- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 25 +++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 106 ++++++++++++------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 5 +- 5 files changed, 111 insertions(+), 67 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a928cc725d..2a627af324 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -74,6 +74,7 @@ typedef struct SLDataIter SLDataIter; typedef struct SDiskCol SDiskCol; typedef struct SDiskData SDiskData; typedef struct SDiskDataBuilder SDiskDataBuilder; +typedef struct SBlkInfo SBlkInfo; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -173,7 +174,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut int32_t aBufN[]); int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr -int32_t tPutDiskDataHdr(uint8_t *p, void *ph); +int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); @@ -270,6 +271,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader @@ -328,7 +330,6 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB uint8_t calcSma); int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo *pBlkInfo); int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo); // structs ======================= @@ -451,14 +452,16 @@ struct SSmaInfo { int32_t size; }; -typedef struct { +struct SBlkInfo { int64_t minUid; int64_t maxUid; - TSDBKEY minKey; - TSDBKEY maxKey; + TSKEY minKey; + TSKEY maxKey; int64_t minVer; int64_t maxVer; -} SBlkInfo; + TSDBKEY minTKey; + TSDBKEY maxTKey; +}; struct SDataBlk { TSDBKEY minKey; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7bc335d034..1a0534df5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -542,6 +542,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); + tDiskDataBuilderClear(pCommitter->dWriter.pBuilder); // open iter code = tsdbOpenCommitIter(pCommitter); @@ -655,21 +656,21 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde if (pBuilder->nRow == 0) return code; - SSttBlk sttBlk = {.suid = pBuilder->suid, - .minUid = 0, // todo - .maxUid = 0, // todo - .minKey = 0, // todo - .maxKey = 0, // todo - .minVer = 0, // todo - .maxVer = 0, // todo - .nRow = pBuilder->nRow}; - // gnrt - // code = tGnrtDiskData(pBuilder, &pBuilder->dd); + const SDiskData *pDiskData; + const SBlkInfo *pBlkInfo; + code = tGnrtDiskData(pBuilder, &pDiskData, &pBlkInfo); TSDB_CHECK_CODE(code, lino, _exit); + SSttBlk sttBlk = {.suid = pBuilder->suid, + .minUid = pBlkInfo->minUid, + .maxUid = pBlkInfo->maxUid, + .minKey = pBlkInfo->minKey, + .maxKey = pBlkInfo->maxKey, + .minVer = pBlkInfo->minVer, + .maxVer = pBlkInfo->maxVer}; // write - // code = tsdbWriteDiskData(pWriter, &pBuilder->dd); + code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL); TSDB_CHECK_CODE(code, lino, _exit); // push @@ -678,8 +679,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde TSDB_CHECK_CODE(code, lino, _exit); } - // clear (todo) - // tDiskDataBuilderClear(pBuilder); + // clear + tDiskDataBuilderClear(pBuilder); _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index d3ea4f1dda..43be51a694 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -498,10 +498,12 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB pBuilder->calcSma = calcSma; pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX, .maxUid = INT64_MIN, - .minKey = TSDBKEY_MAX, - .maxKey = TSDBKEY_MIN, + .minKey = TSKEY_MAX, + .maxKey = TSKEY_MIN, .minVer = VERSION_MAX, - .maxVer = VERSION_MIN}; + .maxVer = VERSION_MIN, + .minTKey = TSDBKEY_MAX, + .maxTKey = TSDBKEY_MIN}; if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); @@ -551,6 +553,7 @@ int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) { int32_t code = 0; pBuilder->suid = 0; pBuilder->uid = 0; + pBuilder->nRow = 0; return code; } @@ -560,7 +563,9 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS ASSERT(pBuilder->suid || pBuilder->uid); ASSERT(pId->suid == pBuilder->suid); - TSDBKEY key = TSDBROW_KEY(pRow); + TSDBKEY kRow = TSDBROW_KEY(pRow); + if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow; + if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow; // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { @@ -579,16 +584,16 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid; // version - code = tCompress(pBuilder->pVerC, &key.version, sizeof(int64_t)); + code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t)); if (code) return code; - if (pBuilder->bi.minVer > key.version) pBuilder->bi.minVer = key.version; - if (pBuilder->bi.maxVer < key.version) pBuilder->bi.maxVer = key.version; + if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version; + if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version; // TSKEY - code = tCompress(pBuilder->pKeyC, &key.ts, sizeof(int64_t)); + code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t)); if (code) return code; - if (tsdbKeyCmprFn(&pBuilder->bi.minKey, &key) > 0) pBuilder->bi.minKey = key; - if (tsdbKeyCmprFn(&pBuilder->bi.maxKey, &key) < 0) pBuilder->bi.maxKey = key; + if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; + if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4352595d88..00b6830fbc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -522,9 +522,6 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, // write if (pSmaInfo->size) { - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size); - if (code) goto _err; - code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); if (code) goto _err; @@ -607,12 +604,20 @@ _err: return code; } -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInfo *pBlkInfo) { +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) { int32_t code = 0; int32_t lino = 0; - STsdbFD *pFD = pWriter->pDataFD; // todo - int64_t offset = pWriter->fData.size; + STsdbFD *pFD = NULL; + if (pSmaInfo) { + pFD = pWriter->pDataFD; + pBlkInfo->offset = pWriter->fData.size; + } else { + pFD = pWriter->pSttFD; + pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; + } + pBlkInfo->szBlock = 0; + pBlkInfo->szKey = 0; // hdr int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); @@ -621,26 +626,30 @@ int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInf tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], n); + code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n); TSDB_CHECK_CODE(code, lino, _exit); - offset += n; + pBlkInfo->szKey += n; + pBlkInfo->szBlock += n; // uid + ver + key - if (pDiskData->hdr.szUid) { - code = tsdbWriteFile(pFD, offset, pDiskData->pUid, pDiskData->hdr.szUid); + if (pDiskData->pUid) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szUid; + pBlkInfo->szKey += pDiskData->hdr.szUid; + pBlkInfo->szBlock += pDiskData->hdr.szUid; } - code = tsdbWriteFile(pFD, offset, pDiskData->pVer, pDiskData->hdr.szVer); + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szVer; + pBlkInfo->szKey += pDiskData->hdr.szVer; + pBlkInfo->szBlock += pDiskData->hdr.szVer; - code = tsdbWriteFile(pFD, offset, pDiskData->pKey, pDiskData->hdr.szKey); + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szKey; + pBlkInfo->szKey += pDiskData->hdr.szKey; + pBlkInfo->szBlock += pDiskData->hdr.szKey; - // SBlockCol + // aBlockCol if (pDiskData->hdr.szBlkCol) { code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); TSDB_CHECK_CODE(code, lino, _exit); @@ -648,48 +657,75 @@ int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInf n = 0; for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - n += tPutBlockCol(pWriter->aBuf[0] + n, &pDiskCol->bCol); + n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol); } - ASSERT(n == pDiskData->hdr.szBlkCol); - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szBlkCol; + pBlkInfo->szBlock += pDiskData->hdr.szBlkCol; } - // pData + // aDiskCol for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - if (pDiskCol->bCol.flag == HAS_NULL) continue; - - if (pDiskCol->bCol.szBitmap) { - code = tsdbWriteFile(pFD, offset, pDiskCol->pBit, pDiskCol->bCol.szBitmap); + if (pDiskCol->pBit) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskCol->bCol.szBitmap; + + pBlkInfo->szBlock += pDiskCol->bCol.szBitmap; } - if (pDiskCol->bCol.szOffset) { - code = tsdbWriteFile(pFD, offset, pDiskCol->pOff, pDiskCol->bCol.szOffset); + if (pDiskCol->pOff) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskCol->bCol.szOffset; + + pBlkInfo->szBlock += pDiskCol->bCol.szOffset; } - if (pDiskCol->bCol.szValue) { - code = tsdbWriteFile(pFD, offset, pDiskCol->pVal, pDiskCol->bCol.szValue); + if (pDiskCol->pVal) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskCol->bCol.szValue; + + pBlkInfo->szBlock += pDiskCol->bCol.szValue; } } + if (pSmaInfo) { + pWriter->fData.size += pBlkInfo->szBlock; + } else { + pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; + goto _exit; + } + + pSmaInfo->offset = 0; + pSmaInfo->size = 0; + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue; + if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue; + if (!pDiskCol->bCol.smaOn) continue; + + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg)); + TSDB_CHECK_CODE(code, lino, _exit); + pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg); + } + + if (pSmaInfo->size) { + pSmaInfo->offset = pWriter->fSma.size; + + code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size); + TSDB_CHECK_CODE(code, lino, _exit); + + pWriter->fSma.size += pSmaInfo->size; + } + _exit: if (code) { tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s", TD_VID(pWriter->pTsdb->pVnode), __func__); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8b95d15407..47c836d9c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1454,9 +1454,8 @@ _exit: } // SDiskDataHdr ============================== -int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { - int32_t n = 0; - SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; +int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) { + int32_t n = 0; n += tPutU32(p ? p + n : p, pHdr->delimiter); n += tPutU32v(p ? p + n : p, pHdr->fmtVer); From 18f3bd53c577c8ce50d5daa2fd90872d602d1652 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 23:34:06 +0800 Subject: [PATCH 39/59] fix code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1a0534df5c..99603500ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1364,14 +1364,15 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); + + tDiskDataBuilderClear(pBuilder); } } if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - code = - tDiskDataBuilderInit(pCommitter->dWriter.pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); + code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1388,8 +1389,8 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { int32_t lino = 0; SBlockData *pBData = &pCommitter->dWriter.bData; + TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); @@ -1402,6 +1403,9 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); } } @@ -1425,7 +1429,8 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pRowInfo == NULL) goto _exit; if (pCommitter->toLastOnly) { - // init the data if need + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); while (pRowInfo) { STSchema *pTSchema = NULL; @@ -1449,6 +1454,9 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); } } } else { From 27b6290f6bb5e50294abc85a452d681f93b1f4b6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 09:56:28 +0800 Subject: [PATCH 40/59] fix compression bug --- source/util/src/tcompression.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 8c38b4f20d..003843c5bb 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1379,7 +1379,7 @@ static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) pCmprsor->i_end = 0; pCmprsor->nBuf = 1; - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf); + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); if (code) return code; pCmprsor->pBuf[0] = 0; @@ -1437,7 +1437,7 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); - if (pCmprsor->aBuf[0][0] == 0) { + if (pCmprsor->pBuf[0] == 0) { int64_t val = 0; memcpy(&val, pData, nData); // little-endian only From df84baad12b559810083fadc111385299ad1a1fe Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 10:12:10 +0800 Subject: [PATCH 41/59] fix another bug --- source/util/src/tcompression.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 003843c5bb..dc473922d7 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -999,7 +999,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co /************************************************************************* * STREAM COMPRESSION *************************************************************************/ -#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (a)) || ((a) < 0 && (b) >= INT64_MIN - (a))) static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); From a4d426aa13a9b3621e2357e2ca080f5e34d8ab79 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 13:39:48 +0800 Subject: [PATCH 42/59] more fix --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 00b6830fbc..167b4a104c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1069,8 +1069,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(pBlockData->suid == hdr.suid); - ASSERT(pBlockData->uid == hdr.uid); + pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; // uid From 5be361ee087dfee67ed8645a4017ddd569827357 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 14:01:34 +0800 Subject: [PATCH 43/59] more fix --- source/util/src/tcompression.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index dc473922d7..7d1c6b02ad 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1400,7 +1400,7 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { int32_t n = 1; int32_t nBuf = 1; int64_t vPrev = 0; - for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + while (n < pCmprsor->nBuf) { uint64_t b; memcpy(&b, pCmprsor->pBuf + n, sizeof(b)); n += sizeof(b); @@ -1410,14 +1410,18 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { uint8_t bits = BIT_PER_INTEGER[i_selector]; for (int32_t iEle = 0; iEle < nEle; iEle++) { uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); - int64_t diff = ZIGZAG_DECODE(int64_t, vZigzag); - vPrev = diff + vPrev; + vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; } - iVal += nEle; - ASSERT(iVal <= pCmprsor->nVal); + } + + for (; pCmprsor->i_start != pCmprsor->i_end; pCmprsor->i_start = (pCmprsor->i_start + 1) % 241) { + vPrev = ZIGZAG_DECODE(int64_t, pCmprsor->i_aZigzag[pCmprsor->i_start]) + vPrev; + + memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); + nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; } ASSERT(n == pCmprsor->nBuf && nBuf == size); From 0ca19fb42bd2a072eaed048060c203104b8f7245 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 15:46:56 +0800 Subject: [PATCH 44/59] fix more --- source/util/src/tcompression.c | 74 ++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 7d1c6b02ad..a6167a1a75 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1024,6 +1024,11 @@ static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static FORCE_INLINE int64_t tI8ToI64(const void *pData) { return *(int8_t *)pData; } +static FORCE_INLINE int64_t tI16ToI64(const void *pData) { return *(int16_t *)pData; } +static FORCE_INLINE int64_t tI32ToI64(const void *pData) { return *(int32_t *)pData; } +static FORCE_INLINE int64_t tI64ToI64(const void *pData) { return *(int64_t *)pData; } static struct { int8_t type; int32_t bytes; @@ -1031,122 +1036,148 @@ static struct { int32_t (*startFn)(SCompressor *, int8_t type, int8_t cmprAlg); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); + int64_t (*getI64)(const void *pData); } DATA_TYPE_INFO[] = { - {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .startFn = NULL, .cmprFn = NULL, .endFn = NULL}, + {.type = TSDB_DATA_TYPE_NULL, + .bytes = 0, + .isVarLen = 0, + .startFn = NULL, + .cmprFn = NULL, + .endFn = NULL, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .startFn = tCompBoolStart, .cmprFn = tCompBool, - .endFn = tCompBoolEnd}, + .endFn = tCompBoolEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI8ToI64}, {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI16ToI64}, {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI32ToI64}, {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI64ToI64}, {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .startFn = tCompFloatStart, .cmprFn = tCompFloat, - .endFn = tCompFloatEnd}, + .endFn = tCompFloatEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .startFn = tCompDoubleStart, .cmprFn = tCompDouble, - .endFn = tCompDoubleEnd}, + .endFn = tCompDoubleEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8, .isVarLen = 0, .startFn = tCompTimestampStart, .cmprFn = tCompTimestamp, - .endFn = tCompTimestampEnd}, + .endFn = tCompTimestampEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI8ToI64}, {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI16ToI64}, {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI32ToI64}, {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI64ToI64}, {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, }; struct SCompressor { @@ -1442,8 +1473,7 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); if (pCmprsor->pBuf[0] == 0) { - int64_t val = 0; - memcpy(&val, pData, nData); // little-endian only + int64_t val = DATA_TYPE_INFO[pCmprsor->type].getI64(pData); if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { code = tCompIntSwitchToCopy(pCmprsor); From 82a227b9c74ad0ec7bac2cb59fe1f68d5cfccbd3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 18:48:35 +0800 Subject: [PATCH 45/59] more fix --- include/util/tcoding.h | 6 +- source/util/src/tcompression.c | 117 ++++++++++++++++++++++++--------- 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 5962949a70..38eb0d8fc6 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -23,8 +23,8 @@ extern "C" { #endif #define ENCODE_LIMIT (((uint8_t)1) << 7) -#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode -#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode +#define ZIGZAGE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode +#define ZIGZAGD(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode /* ------------------------ LEGACY CODES ------------------------ */ #if 1 @@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) { } static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { - *value = ( (((int8_t *)buf)[0] == 0) ? false : true ); + *value = ((((int8_t *)buf)[0] == 0) ? false : true); return POINTER_SHIFT(buf, sizeof(int8_t)); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index a6167a1a75..9cac1fb957 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1; #define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) -#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode -#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode +#define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode +#define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode #ifdef TD_TSZ bool lossyFloat = false; @@ -1025,10 +1025,16 @@ static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprA static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); -static FORCE_INLINE int64_t tI8ToI64(const void *pData) { return *(int8_t *)pData; } -static FORCE_INLINE int64_t tI16ToI64(const void *pData) { return *(int16_t *)pData; } -static FORCE_INLINE int64_t tI32ToI64(const void *pData) { return *(int32_t *)pData; } -static FORCE_INLINE int64_t tI64ToI64(const void *pData) { return *(int64_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI8(const void *pData) { return *(int8_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI16(const void *pData) { return *(int16_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI32(const void *pData) { return *(int32_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI64(const void *pData) { return *(int64_t *)pData; } + +static FORCE_INLINE void tPutI64OfI8(int64_t v, void *pData) { *(int8_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI16(int64_t v, void *pData) { *(int16_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI32(int64_t v, void *pData) { *(int32_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI64(int64_t v, void *pData) { *(int64_t *)pData = v; } + static struct { int8_t type; int32_t bytes; @@ -1037,6 +1043,7 @@ static struct { int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); int64_t (*getI64)(const void *pData); + void (*putI64)(int64_t v, void *pData); } DATA_TYPE_INFO[] = { {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, @@ -1044,140 +1051,160 @@ static struct { .startFn = NULL, .cmprFn = NULL, .endFn = NULL, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .startFn = tCompBoolStart, .cmprFn = tCompBool, .endFn = tCompBoolEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI8ToI64}, + .getI64 = tGetI64OfI8, + .putI64 = tPutI64OfI8}, {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI16ToI64}, + .getI64 = tGetI64OfI16, + .putI64 = tPutI64OfI16}, {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI32ToI64}, + .getI64 = tGetI64OfI32, + .putI64 = tPutI64OfI32}, {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI64ToI64}, + .getI64 = tGetI64OfI64, + .putI64 = tPutI64OfI64}, {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .startFn = tCompFloatStart, .cmprFn = tCompFloat, .endFn = tCompFloatEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .startFn = tCompDoubleStart, .cmprFn = tCompDouble, .endFn = tCompDoubleEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8, .isVarLen = 0, .startFn = tCompTimestampStart, .cmprFn = tCompTimestamp, .endFn = tCompTimestampEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI8ToI64}, + .getI64 = tGetI64OfI8, + .putI64 = tPutI64OfI8}, {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI16ToI64}, + .getI64 = tGetI64OfI16, + .putI64 = tPutI64OfI16}, {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI32ToI64}, + .getI64 = tGetI64OfI32, + .putI64 = tPutI64OfI32}, {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI64ToI64}, + .getI64 = tGetI64OfI64, + .putI64 = tPutI64OfI64}, {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, }; struct SCompressor { @@ -1443,7 +1470,7 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; - memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); + DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf); nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; } } @@ -1504,8 +1531,22 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; break; } else { - while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { - pCmprsor->i_selector++; + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + int32_t lidx = pCmprsor->i_selector + 1; + int32_t ridx = 15; + while (lidx <= ridx) { + pCmprsor->i_selector = (lidx + ridx) >> 1; + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + lidx = pCmprsor->i_selector + 1; + } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + ridx = pCmprsor->i_selector - 1; + } else { + break; + } + } + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; @@ -1551,8 +1592,22 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_ int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; if (nEle == 0) break; - while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { - pCmprsor->i_selector++; + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + int32_t lidx = pCmprsor->i_selector + 1; + int32_t ridx = 15; + while (lidx <= ridx) { + pCmprsor->i_selector = (lidx + ridx) >> 1; + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + lidx = pCmprsor->i_selector + 1; + } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + ridx = pCmprsor->i_selector - 1; + } else { + break; + } + } + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; From cf0aecd3dd5701082f94eb73f94a0e9c692ecd44 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 00:02:09 +0800 Subject: [PATCH 46/59] more fix --- source/util/src/tcompression.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 9cac1fb957..b19bfbbd43 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1320,7 +1320,7 @@ static int32_t tCompTSSwitchToCopy(SCompressor *pCmprsor) { } } - ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nBuf + 1); + ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nVal + 1); uint8_t *pBuf = pCmprsor->pBuf; pCmprsor->pBuf = pCmprsor->aBuf[0]; From 768ec8942a8f7bdd6a49a0c3c1d46faa9a680a0d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 13:23:12 +0800 Subject: [PATCH 47/59] fix code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 99603500ae..8992ee47c4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -668,7 +668,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde .minKey = pBlkInfo->minKey, .maxKey = pBlkInfo->maxKey, .minVer = pBlkInfo->minVer, - .maxVer = pBlkInfo->maxVer}; + .maxVer = pBlkInfo->maxVer, + .nRow = pBuilder->nRow}; // write code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL); TSDB_CHECK_CODE(code, lino, _exit); From ce087d3dbc8f7b86fe87e1fb36e440800762ef29 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 14:54:58 +0800 Subject: [PATCH 48/59] optimize more code --- source/util/src/tcompression.c | 73 +++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b19bfbbd43..be220c2585 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1228,6 +1228,7 @@ struct SCompressor { int32_t i_selector; int32_t i_start; int32_t i_end; + int32_t i_nEle; uint64_t i_aZigzag[241]; int8_t i_aBitN[241]; }; @@ -1427,6 +1428,18 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; +static const int32_t NEXT_IDX[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 0}; static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; @@ -1435,6 +1448,7 @@ static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) pCmprsor->i_selector = 0; pCmprsor->i_start = 0; pCmprsor->i_end = 0; + pCmprsor->i_nEle = 0; pCmprsor->nBuf = 1; code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); @@ -1463,11 +1477,12 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { memcpy(&b, pCmprsor->pBuf + n, sizeof(b)); n += sizeof(b); - int32_t i_selector = (b & 0xf); - int32_t nEle = SELECTOR_TO_ELEMS[i_selector]; - uint8_t bits = BIT_PER_INTEGER[i_selector]; + int32_t i_selector = (b & 0xf); + int32_t nEle = SELECTOR_TO_ELEMS[i_selector]; + uint8_t bits = BIT_PER_INTEGER[i_selector]; + uint64_t mask = (((uint64_t)1) << bits) - 1; for (int32_t iEle = 0; iEle < nEle; iEle++) { - uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); + uint64_t vZigzag = (b >> (bits * iEle + 4)) & mask; vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf); @@ -1475,11 +1490,14 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { } } - for (; pCmprsor->i_start != pCmprsor->i_end; pCmprsor->i_start = (pCmprsor->i_start + 1) % 241) { + while (pCmprsor->i_nEle) { vPrev = ZIGZAG_DECODE(int64_t, pCmprsor->i_aZigzag[pCmprsor->i_start]) + vPrev; memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; + + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; } ASSERT(n == pCmprsor->nBuf && nBuf == size); @@ -1520,35 +1538,35 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) pCmprsor->i_prev = val; for (;;) { - int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; - - if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { + if (pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && + pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) { pCmprsor->i_selector = BIT_TO_SELECTOR[nBit]; } pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag; pCmprsor->i_aBitN[pCmprsor->i_end] = nBit; - pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; + pCmprsor->i_end = NEXT_IDX[pCmprsor->i_end]; + pCmprsor->i_nEle++; break; } else { - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { int32_t lidx = pCmprsor->i_selector + 1; int32_t ridx = 15; while (lidx <= ridx) { pCmprsor->i_selector = (lidx + ridx) >> 1; - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { lidx = pCmprsor->i_selector + 1; - } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { ridx = pCmprsor->i_selector - 1; } else { break; } } - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } - nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { return code; @@ -1559,13 +1577,14 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) bp[0] = pCmprsor->i_selector; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; for (int32_t iVal = 0; iVal < nEle; iVal++) { - bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); - pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4)); + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; } // reset and continue pCmprsor->i_selector = 0; - for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = (iVal + 1) % 241) { + for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = NEXT_IDX[iVal]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) { pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]; } @@ -1588,28 +1607,25 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - for (;;) { - int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; - if (nEle == 0) break; - - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + for (; pCmprsor->i_nEle;) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { int32_t lidx = pCmprsor->i_selector + 1; int32_t ridx = 15; while (lidx <= ridx) { pCmprsor->i_selector = (lidx + ridx) >> 1; - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { lidx = pCmprsor->i_selector + 1; - } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { ridx = pCmprsor->i_selector - 1; } else { break; } } - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } - nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { return code; @@ -1620,8 +1636,9 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_ bp[0] = pCmprsor->i_selector; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; for (int32_t iVal = 0; iVal < nEle; iVal++) { - bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); - pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4)); + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; } pCmprsor->i_selector = 0; From bdeb1ebcc5c18c376f78555a9dbf040fcbaddd33 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 17:06:18 +0800 Subject: [PATCH 49/59] compare --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 76 ++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8992ee47c4..05e08b6bf4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -17,6 +17,8 @@ typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; +#define USE_STREAM_COMPRESSION 1 + typedef struct { SRBTreeNode n; SRowInfo r; @@ -70,12 +72,16 @@ typedef struct { int8_t toLastOnly; }; struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; // SArray - SArray *aSttBlk; // SArray - SMapData mBlock; // SMapData - SBlockData bData; + SDataFWriter *pWriter; + SArray *aBlockIdx; // SArray + SArray *aSttBlk; // SArray + SMapData mBlock; // SMapData + SBlockData bData; +#if USE_STREAM_COMPRESSION SDiskDataBuilder *pBuilder; +#else + SBlockData bDatal; +#endif } dWriter; SSkmInfo skmTable; SSkmInfo skmRow; @@ -542,7 +548,11 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); +#if USE_STREAM_COMPRESSION tDiskDataBuilderClear(pCommitter->dWriter.pBuilder); +#else + tBlockDataReset(&pCommitter->dWriter.bDatal); +#endif // open iter code = tsdbOpenCommitIter(pCommitter); @@ -860,7 +870,11 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataCreate(&pCommitter->dWriter.bData); TSDB_CHECK_CODE(code, lino, _exit); +#if USE_STREAM_COMPRESSION code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder); +#else + code = tBlockDataCreate(&pCommitter->dWriter.bDatal); +#endif TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -889,7 +903,11 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); +#if USE_STREAM_COMPRESSION tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); +#else + tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); +#endif tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } @@ -1360,6 +1378,7 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) int32_t code = 0; int32_t lino = 0; +#if USE_STREAM_COMPRESSION SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; if (pBuilder->suid || pBuilder->uid) { if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { @@ -1376,6 +1395,25 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); } +#else + SBlockData *pBData = &pCommitter->dWriter.bDatal; + if (pBData->suid || pBData->uid) { + if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + + tBlockDataReset(pBData); + } + } + + if (!pBData->suid && !pBData->uid) { + ASSERT(pCommitter->skmTable.suid == id.suid); + ASSERT(pCommitter->skmTable.uid == id.uid); + TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; + code = tBlockDataInit(pBData, &tid, pCommitter->skmTable.pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif _exit: if (code) { @@ -1398,6 +1436,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); +#if USE_STREAM_COMPRESSION code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); TSDB_CHECK_CODE(code, lino, _exit); @@ -1408,6 +1447,16 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); } +#else + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &row, NULL, id.uid); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif } _exit: @@ -1441,7 +1490,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { pTSchema = pCommitter->skmRow.pTSchema; } +#if USE_STREAM_COMPRESSION code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); +#else + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pRowInfo->row, pTSchema, id.uid); +#endif TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); @@ -1452,6 +1505,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { pRowInfo = NULL; } +#if USE_STREAM_COMPRESSION if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); @@ -1459,6 +1513,13 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); } +#else + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif } } else { SBlockData *pBData = &pCommitter->dWriter.bData; @@ -1561,7 +1622,12 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); +#if USE_STREAM_COMPRESSION code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); +#else + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); +#endif TSDB_CHECK_CODE(code, lino, _exit); _exit: From 145abcb17fdb648eb69919411a6e7a73da1c1968 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 30 Sep 2022 10:52:17 +0800 Subject: [PATCH 50/59] more --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 05e08b6bf4..995d7a3155 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -17,7 +17,7 @@ typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; -#define USE_STREAM_COMPRESSION 1 +#define USE_STREAM_COMPRESSION 0 typedef struct { SRBTreeNode n; From dfb2b0678f2faae8a717aa3df183fdb68f77bc50 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 30 Sep 2022 14:03:46 +0800 Subject: [PATCH 51/59] fix: coverity issues --- source/dnode/mnode/impl/src/mndDef.c | 20 +++++++++++-- source/dnode/mnode/impl/src/mndStb.c | 22 +++++++-------- source/dnode/mnode/impl/src/mndStream.c | 37 ++++++++++++------------- source/dnode/vnode/src/vnd/vnodeSync.c | 12 ++++---- 4 files changed, 51 insertions(+), 40 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 50d26ce9a5..42e56cd488 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -42,9 +42,23 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + if (pObj->sql != NULL) { + if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } + + if (pObj->ast != NULL) { + if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } + + if (pObj->physicalPlan != NULL) { + if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } int32_t sz = taosArrayGetSize(pObj->tasks); if (tEncodeI32(pEncoder, sz) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 38c7850e6c..8cd8431c34 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -554,7 +554,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { SField *pField1 = taosArrayGet(pCreate->pColumns, i); - if (pField1->type < 0) { + if (pField1->type >= TSDB_DATA_TYPE_MAX) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -570,7 +570,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfTags; ++i) { SField *pField1 = taosArrayGet(pCreate->pTags, i); - if (pField1->type < 0) { + if (pField1->type >= TSDB_DATA_TYPE_MAX) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -982,8 +982,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } } else { - mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name, - createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); + mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid, origin tagVer:%d colVer:%d", + createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER; goto _OVER; } @@ -1603,9 +1603,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return -1; } - strcpy(pRsp->dbFName, pStb->db); - strcpy(pRsp->tbName, tbName); - strcpy(pRsp->stbName, tbName); + tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); + tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName)); + tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName)); pRsp->dbId = pDb->uid; pRsp->numOfTags = pStb->numOfTags; pRsp->numOfColumns = pStb->numOfColumns; @@ -1649,9 +1649,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, return -1; } - strcpy(pRsp->dbFName, pStb->db); - strcpy(pRsp->tbName, tbName); - strcpy(pRsp->stbName, tbName); + tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); + tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName)); + tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName)); pRsp->numOfTags = pStb->numOfTags; pRsp->numOfColumns = pStb->numOfColumns; pRsp->tableType = TSDB_SUPER_TABLE; @@ -2551,7 +2551,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); - char rollup[128 + VARSTR_HEADER_SIZE] = {0}; + char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); for (int32_t i = 0; i < rollupNum; ++i) { char *funcName = taosArrayGet(pStb->pFuncs, i); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index c6f46b28fc..03d7d8e825 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -425,8 +425,10 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream SStreamObj streamObj = {0}; memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); streamObj.status = STREAM_STATUS__RECOVER; + SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -771,12 +773,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); return -1; } // drop stream if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); return -1; } @@ -945,10 +949,8 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SName n; int32_t cols = 0; - char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(streamName)); - varDataSetLen(streamName, strlen(varDataVal(streamName))); + char streamName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)streamName, false); @@ -956,28 +958,24 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN); - varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); + STR_WITH_MAXSIZE_TO_VARSTR(streamName, pStream->sql, sizeof(sql)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); char status[20 + VARSTR_HEADER_SIZE] = {0}; - mndShowStreamStatus(&status[VARSTR_HEADER_SIZE], pStream); - varDataSetLen(status, strlen(varDataVal(status))); + char status2[20] = {0}; + mndShowStreamStatus(status2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&status, false); char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(sourceDB)); - varDataSetLen(sourceDB, strlen(varDataVal(sourceDB))); + STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(targetDB)); - varDataSetLen(targetDB, strlen(varDataVal(targetDB))); + STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); @@ -986,9 +984,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, NULL, true); } else { char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); - varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); + STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); } @@ -997,8 +993,9 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); char trigger[20 + VARSTR_HEADER_SIZE] = {0}; - mndShowStreamTrigger(&trigger[VARSTR_HEADER_SIZE], pStream); - varDataSetLen(trigger, strlen(varDataVal(trigger))); + char trigger2[20] = {0}; + mndShowStreamTrigger(trigger2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index aa3f72d281..f773de5280 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -328,14 +328,14 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { const STraceId *trace = &pMsg->info.traceId; if (!syncEnvIsStart()) { - vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId); + vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId, pMsg); terrno = TSDB_CODE_APP_ERROR; return -1; } SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); if (pSyncNode == NULL) { - vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId); + vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId, pMsg); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -394,7 +394,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); + vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); code = -1; } @@ -459,7 +459,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); + vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); code = -1; } } @@ -630,7 +630,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void vInfo("vgId:%d, start write vnode snapshot since apply queue is empty", pVnode->config.vgId); break; } else { - vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId); + vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId, itemSize); taosMsleep(10); } } while (true); @@ -683,7 +683,7 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); break; } else { - vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId); + vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId, itemSize); taosMsleep(10); } } while (true); From 3ed27805c37d5bbb2ca15d6b44700289444d0204 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 30 Sep 2022 14:43:51 +0800 Subject: [PATCH 52/59] fix: query information_schema.ins_tags error --- source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstParser.c | 21 +++++ source/libs/parser/src/parTranslater.c | 110 +++++++++++----------- source/libs/parser/src/parUtil.c | 67 ++++++++++++- source/libs/parser/test/parSelectTest.cpp | 6 +- source/libs/parser/test/parTestMain.cpp | 2 +- source/libs/planner/test/planTestMain.cpp | 2 +- 7 files changed, 149 insertions(+), 60 deletions(-) diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 308afd467f..75b631ec9d 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -91,6 +91,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); +int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index ce90bd3da0..27adb79e3c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -125,6 +125,23 @@ static bool needGetTableIndex(SNode* pStmt) { return false; } +static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pName) { + if (TSDB_DB_NAME_T == pName->type) { + return reserveDbVgInfoInCache(pName->acctId, pName->dbname, pCxt->pMetaCache); + } + return reserveTableVgroupInCacheExt(pName, pCxt->pMetaCache); +} + +static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) { + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt; + SName name = {0}; + int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name); + if (TSDB_CODE_SUCCESS == code) { + code = collectMetaKeyFromInsTagsImpl(pCxt, &name); + } + return code; +} + static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable, AUTH_TYPE authType) { int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); @@ -143,6 +160,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) && + QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { + code = collectMetaKeyFromInsTags(pCxt); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9d805b8e33..b58626e6fb 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2198,70 +2198,70 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge return code; } -static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) { - if (OP_TYPE_EQUAL != pOper->opType) { - return TSDB_CODE_SUCCESS; - } +// static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) { +// if (OP_TYPE_EQUAL != pOper->opType) { +// return TSDB_CODE_SUCCESS; +// } - SColumnNode* pCol = NULL; - SValueNode* pVal = NULL; - if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { - pCol = (SColumnNode*)pOper->pLeft; - } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { - pVal = (SValueNode*)pOper->pLeft; - } - if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { - pCol = (SColumnNode*)pOper->pRight; - } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { - pVal = (SValueNode*)pOper->pRight; - } - if (NULL == pCol || NULL == pVal) { - return TSDB_CODE_SUCCESS; - } +// SColumnNode* pCol = NULL; +// SValueNode* pVal = NULL; +// if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { +// pCol = (SColumnNode*)pOper->pLeft; +// } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { +// pVal = (SValueNode*)pOper->pLeft; +// } +// if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { +// pCol = (SColumnNode*)pOper->pRight; +// } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { +// pVal = (SValueNode*)pOper->pRight; +// } +// if (NULL == pCol || NULL == pVal) { +// return TSDB_CODE_SUCCESS; +// } - if (0 == strcmp(pCol->colName, "db_name")) { - return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal)); - } else if (0 == strcmp(pCol->colName, "table_name")) { - return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); - } +// if (0 == strcmp(pCol->colName, "db_name")) { +// return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal)); +// } else if (0 == strcmp(pCol->colName, "table_name")) { +// return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); +// } - return TSDB_CODE_SUCCESS; -} +// return TSDB_CODE_SUCCESS; +// } -static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) { - if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName); - } -} +// static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) { +// if (QUERY_NODE_OPERATOR == nodeType(pNode)) { +// getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName); +// } +// } -static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) { - if (LOGIC_COND_TYPE_AND != pCond->condType) { - return TSDB_CODE_SUCCESS; - } +// static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) { +// if (LOGIC_COND_TYPE_AND != pCond->condType) { +// return TSDB_CODE_SUCCESS; +// } - SNode* pNode = NULL; - FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } - if ('\0' == pName->dbname[0]) { - pName->type = 0; - } - return TSDB_CODE_SUCCESS; -} +// SNode* pNode = NULL; +// FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } +// if ('\0' == pName->dbname[0]) { +// pName->type = 0; +// } +// return TSDB_CODE_SUCCESS; +// } -static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) { - if (NULL == pWhere) { - return TSDB_CODE_SUCCESS; - } +// static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) { +// if (NULL == pWhere) { +// return TSDB_CODE_SUCCESS; +// } - if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { - return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName); - } +// if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { +// return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName); +// } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { - return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName); - } +// if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { +// return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName); +// } - return TSDB_CODE_SUCCESS; -} +// return TSDB_CODE_SUCCESS; +// } static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { if (!isSelectStmt(pCxt->pCurrStmt)) { @@ -2269,7 +2269,7 @@ static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SAr } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SName targetName = {0}; - int32_t code = getTagsTableTargetName(pCxt, pSelect->pWhere, &targetName); + int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName); if (TSDB_CODE_SUCCESS == code) { code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 3075be1a00..c6a51f22e8 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -420,6 +420,71 @@ end: return retCode; } +static int32_t getInsTagsTableTargetNameFromOp(int32_t acctId, SOperatorNode* pOper, SName* pName) { + if (OP_TYPE_EQUAL != pOper->opType) { + return TSDB_CODE_SUCCESS; + } + + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { + pCol = (SColumnNode*)pOper->pLeft; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { + pVal = (SValueNode*)pOper->pLeft; + } + if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { + pCol = (SColumnNode*)pOper->pRight; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { + pVal = (SValueNode*)pOper->pRight; + } + if (NULL == pCol || NULL == pVal) { + return TSDB_CODE_SUCCESS; + } + + if (0 == strcmp(pCol->colName, "db_name")) { + return tNameSetDbName(pName, acctId, pVal->literal, strlen(pVal->literal)); + } else if (0 == strcmp(pCol->colName, "table_name")) { + return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); + } + + return TSDB_CODE_SUCCESS; +} + +static void getInsTagsTableTargetObjName(int32_t acctId, SNode* pNode, SName* pName) { + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pNode, pName); + } +} + +static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicConditionNode* pCond, SName* pName) { + if (LOGIC_COND_TYPE_AND != pCond->condType) { + return TSDB_CODE_SUCCESS; + } + + SNode* pNode = NULL; + FOREACH(pNode, pCond->pParameterList) { getInsTagsTableTargetObjName(acctId, pNode, pName); } + if ('\0' == pName->dbname[0]) { + pName->type = 0; + } + return TSDB_CODE_SUCCESS; +} + +int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) { + if (NULL == pWhere) { + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { + return getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pWhere, pName); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { + return getInsTagsTableTargetNameFromCond(acctId, (SLogicConditionNode*)pWhere, pName); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) { return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type); } @@ -1173,7 +1238,7 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { taosArrayDestroy(p->pTableVgroupReq); p = taosHashIterate(pMetaCache->pInsertTables, p); - } + } taosHashCleanup(pMetaCache->pInsertTables); taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbCfg); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 79a7038401..0027ac9ca1 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -420,9 +420,11 @@ TEST_F(ParserSelectTest, setOperatorSemanticCheck) { } TEST_F(ParserSelectTest, informationSchema) { - useDb("root", "test"); + useDb("root", "information_schema"); - run("SELECT * FROM information_schema.ins_databases WHERE name = 'information_schema'"); + run("SELECT * FROM ins_databases WHERE name = 'information_schema'"); + + run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'"); } TEST_F(ParserSelectTest, withoutFrom) { diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 9e67249238..de2ce55459 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -53,7 +53,7 @@ class ParserEnv : public testing::Environment { private: void initLog(const char* path) { - int32_t logLevel = getLogLevel(); + int32_t logLevel = getLogLevel() | DEBUG_SCREEN; dDebugFlag = logLevel; vDebugFlag = logLevel; mDebugFlag = logLevel; diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index df6e72ce46..4e013c44b8 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -48,7 +48,7 @@ class PlannerEnv : public testing::Environment { private: void initLog(const char* path) { - int32_t logLevel = getLogLevel(); + int32_t logLevel = getLogLevel() | DEBUG_SCREEN; dDebugFlag = logLevel; vDebugFlag = logLevel; mDebugFlag = logLevel; From 6faf1af5f5f3c610b97827465bcaa1147380a480 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Sep 2022 14:44:42 +0800 Subject: [PATCH 53/59] fix(query): handle the block read case. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c3cb5f9eb8..de64b8a61e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -926,6 +926,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn } } + // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, i); colDataAppendNNULL(pColData, 0, remain); @@ -935,12 +936,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pResBlock->info.rows = remain; pDumpInfo->rowIndex += step * remain; + // check if current block are all handled if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { -// int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; -// setBlockAllDumped(pDumpInfo, ts, pReader->order); + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + if (outOfTimeWindow(ts, &pReader->window)) { // the remain data has out of query time window, ignore current block + setBlockAllDumped(pDumpInfo, ts, pReader->order); + } } else { - int64_t k = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; - setBlockAllDumped(pDumpInfo, k, pReader->order); + int64_t ts = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; + setBlockAllDumped(pDumpInfo, ts, pReader->order); } double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; From 31e424d5aba662b2922fe3a53d2b5dd4ba3c8f2a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 30 Sep 2022 14:59:31 +0800 Subject: [PATCH 54/59] fix: coverity issues --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 03d7d8e825..ea889e7001 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -958,7 +958,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(streamName, pStream->sql, sizeof(sql)); + STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); From 99befe4a8915f9bc01328bcb0c0babf1547ba2d6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 30 Sep 2022 15:08:21 +0800 Subject: [PATCH 55/59] fix: ins_tags table name optimization error --- source/libs/executor/src/scanoperator.c | 78 ++++++++++++++----------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 40fc8cb9a5..2251d09776 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -41,9 +41,9 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S static bool processBlockWithProbability(const SSampleExecInfo* pInfo); -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, - const char* tableName, int32_t* pNumOfRows, - const SSDataBlock* dataBlock); +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock); static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { @@ -2415,11 +2415,10 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && nodeType(node->pRight) == QUERY_NODE_VALUE) { SValueNode* pValue = (SValueNode*)node->pRight; - if (pValue->node.type == TSDB_DATA_TYPE_NCHAR || pValue->node.type == TSDB_DATA_TYPE_VARCHAR || - pValue->node.type == TSDB_DATA_TYPE_BINARY) { - char* value = nodesGetStrValueFromNode(pValue); - strncpy(condTable, value, TSDB_TABLE_NAME_LEN); - taosMemoryFree(value); + if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || + pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { + char* value = nodesGetValueFromNode(pValue); + strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); return true; } } @@ -2480,18 +2479,28 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tableName, condTableName); - SMetaReader smr = {0}; - metaReaderInit(&smr, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smr, condTableName); - sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smr); + SMetaReader smrChildTable = {0}; + metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); + metaGetTableEntryByName(&smrChildTable, condTableName); + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { + metaReaderClear(&smrChildTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); + metaReaderClear(&smrSuperTable); + metaReaderClear(&smrChildTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); numOfRows = 0; } blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + doSetOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -2508,23 +2517,22 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - SMetaReader smr = {0}; - metaReaderInit(&smr, pInfo->readHandle.meta, 0); - + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smr, suid); + int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&smr); + metaReaderClear(&smrSuperTable); metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } - sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smr); + metaReaderClear(&smrSuperTable); if (numOfRows >= pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); @@ -2562,15 +2570,15 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t blockDataCleanup(dataBlock); } -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, - const char* tableName, int32_t* pNumOfRows, - const SSDataBlock* dataBlock) { +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock) { char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(stableName, (*smr).me.name); + STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); int32_t numOfRows = *pNumOfRows; - int32_t numOfTags = (*smr).me.stbEntry.schemaTag.nCols; + int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; for (int32_t i = 0; i < numOfTags; ++i) { SColumnInfoData* pColInfoData = NULL; @@ -2588,35 +2596,35 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, // tag name char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tagName, (*smr).me.stbEntry.schemaTag.pSchema[i].name); + STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); colDataAppend(pColInfoData, numOfRows, tagName, false); // tag type - int8_t tagType = (*smr).me.stbEntry.schemaTag.pSchema[i].type; + int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); char tagTypeStr[VARSTR_HEADER_SIZE + 32]; int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); if (tagType == TSDB_DATA_TYPE_VARCHAR) { tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); + (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); } else if (tagType == TSDB_DATA_TYPE_NCHAR) { - tagTypeLen += - sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)(((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + tagTypeLen += sprintf( + varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(tagTypeStr, tagTypeLen); colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); STagVal tagVal = {0}; - tagVal.cid = (*smr).me.stbEntry.schemaTag.pSchema[i].colId; + tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; char* tagData = NULL; uint32_t tagLen = 0; if (tagType == TSDB_DATA_TYPE_JSON) { - tagData = (char*)pInfo->pCur->mr.me.ctbEntry.pTags; + tagData = (char*)smrChildTable->me.ctbEntry.pTags; } else { - bool exist = tTagGet((STag*)pInfo->pCur->mr.me.ctbEntry.pTags, &tagVal); + bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); if (exist) { if (IS_VAR_DATA_TYPE(tagType)) { tagData = (char*)tagVal.pData; From f3b2067675522118938d27f70b4bcf492a1e04df Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 30 Sep 2022 14:41:30 +0800 Subject: [PATCH 56/59] fix(stream): sliding error --- source/libs/executor/src/timewindowoperator.c | 20 +++++--- tests/script/tsim/stream/sliding.sim | 48 ++++++++++++++++++- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9b4b7f1ee8..dceb696d54 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1402,7 +1402,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); - while (win.skey <= endTsCols[i]) { + do { uint64_t winGpId = pGpDatas[i]; bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput); SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; @@ -1413,7 +1413,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - } + } while (win.ekey <= endTsCols[i]); } } @@ -3067,8 +3067,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + STimeWindow nextWin = {0}; + if (IS_FINAL_OP(pInfo)) { + nextWin = getFinalTimeWindow(ts, &pInfo->interval); + } else { + nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { @@ -3122,8 +3126,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); + if (IS_FINAL_OP(pInfo)) { + forwardRows = 1; + } else { + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, + NULL, TSDB_ORDER_ASC); + } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); } diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index bd8d3b0579..b7477fe36c 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -369,9 +369,10 @@ endi #$loop_all = 0 #=looptest: -sql drop database IF EXISTS test2; sql drop stream IF EXISTS streams21; sql drop stream IF EXISTS streams22; +sql drop stream IF EXISTS streams23; +sql drop database IF EXISTS test2; sql create database test2 vgroups 6; sql use test2; @@ -381,6 +382,7 @@ sql create table t2 using st tags(2,2,2); sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); +sql create stream streams23 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -508,6 +510,50 @@ if $data32 != 8 then goto loop3 endi + +$loop_count = 0 + +loop4: +sleep 100 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt3; + +# row 0 +if $rows != 5 then + print =====rows=$rows + goto loop4 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop4 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop4 +endi + +if $data21 != 4 then + print =====data21=$data21 + goto loop4 +endi + +if $data31 != 4 then + print =====data31=$data31 + goto loop4 +endi + +if $data41 != 2 then + print =====data41=$data41 + goto loop4 +endi + $loop_all = $loop_all + 1 print ============loop_all=$loop_all From 90c6116373b5166508ebd7caf87e6c9b01460a1f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 30 Sep 2022 15:42:58 +0800 Subject: [PATCH 57/59] fix mem leak --- source/dnode/vnode/src/meta/metaQuery.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index d499db4958..0309b8177e 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1145,6 +1145,7 @@ int32_t metaGetTableTagsOpt(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashO if (taosHashGet(tags, id, sizeof(tb_uid_t)) == NULL && 0 == tdbTbGet(pMeta->pCtbIdx, &ctbIdxKey, sizeof(SCtbIdxKey), &val, &len)) { taosHashPut(tags, id, sizeof(tb_uid_t), val, len); + tdbFree(val); } } return 0; From 4b1e8e0cec107981bfc187d38634b242e68a8f9b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Sep 2022 16:02:28 +0800 Subject: [PATCH 58/59] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3a921349e6..845b045360 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -447,7 +447,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - // tMergeTreeClose(&state->mergeTree); + tMergeTreeClose(&state->mergeTree); *ppRow = NULL; return code; @@ -463,7 +463,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; - // tMergeTreeClose(&state->mergeTree); + tMergeTreeClose(&state->mergeTree); goto _next_fileset; } state->state = SFSLASTNEXTROW_BLOCKROW; From 877c707ce6096a66460b7c1194d092231c4003bd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Sep 2022 17:10:55 +0800 Subject: [PATCH 59/59] fix(query): fix memory leak in last_row query --- source/dnode/vnode/src/tsdb/tsdbCache.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 845b045360..6e27d70096 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -590,7 +590,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { goto _next_fileset; } - tMapDataReset(&state->blockMap); + if (state->blockMap.pData != NULL) { + tMapDataClear(&state->blockMap); + } + code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap); if (code) goto _err; @@ -695,6 +698,10 @@ int32_t clearNextRowFromFS(void *iter) { state->pBlockData = NULL; } + if (state->blockMap.pData != NULL) { + tMapDataClear(&state->blockMap); + } + return code; }