From 678f9da4acf7ffad4fb803ebcde0e9b6b2dd5b6e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 10:11:55 +0800 Subject: [PATCH 01/64] refact compress code --- include/util/tcompression.h | 340 ++++----------------------------- source/util/src/tcompression.c | 275 +++++++++++++++++++++++++- 2 files changed, 313 insertions(+), 302 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index d7ba4dfa3f..5196c01fc6 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -51,287 +51,12 @@ extern "C" { #define HEAD_MODE(x) x % 2 #define HEAD_ALGO(x) x / 2 -extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type); -extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, - const char type); -extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize); -extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output, - int32_t outputSize); -extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output); -extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output); -// lossy -extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output); -extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, - char *const output); -extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output); -extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements, - char *const output); - #ifdef TD_TSZ extern bool lossyFloat; extern bool lossyDouble; int32_t tsCompressInit(); void tsCompressExit(); -#endif -static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressBoolImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressBoolImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -} - -static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - return tsCompressStringImp(input, inputSize, output, outputSize); -} - -static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - return tsDecompressStringImp(input, compressedSize, output, outputSize); -} - -static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { -#ifdef TD_TSZ - // lossy mode - if (lossyFloat) { - return tsCompressFloatLossyImp(input, nelements, output); - // lossless mode - } else { -#endif - if (algorithm == ONE_STAGE_COMP) { - return tsCompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressFloatImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { -#ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { - // decompress lossy - return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); - } else { -#endif - // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressFloatImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { -#ifdef TD_TSZ - if (lossyDouble) { - // lossy mode - return tsCompressDoubleLossyImp(input, nelements, output); - } else { -#endif - // lossless mode - if (algorithm == ONE_STAGE_COMP) { - return tsCompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressDoubleImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { -#ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { - // decompress lossy - return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); - } else { -#endif - // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressDoubleImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -#ifdef TD_TSZ - } -#endif -} - -#ifdef TD_TSZ -// -// lossy float double -// static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { @@ -358,33 +83,46 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int #endif -static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, - char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressTimestampImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); - } else { - assert(0); - return -1; - } -} +int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, - const int32_t nelements, char *const output, int32_t outputSize, - char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressTimestampImp(buffer, nelements, output); - } else { - assert(0); - return -1; - } -} +int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize); +int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize); +int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize); +int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); #ifdef __cplusplus } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 62b9d87628..1128c51f9b 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1612,4 +1612,277 @@ int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData); -} \ No newline at end of file +} + +/************************************************************************* + * REGULAR COMPRESSION + *************************************************************************/ +// Timestamp ===================================================== +int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressTimestampImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressTimestampImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressTimestampImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressTimestampImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +} + +// Float ===================================================== +int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + // lossy mode + if (lossyFloat) { + return tsCompressFloatLossyImp(input, nelements, output); + // lossless mode + } else { +#endif + if (algorithm == ONE_STAGE_COMP) { + return tsCompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressFloatImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { + // decompress lossy + return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); + } else { +#endif + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressFloatImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +// Double ===================================================== +int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (lossyDouble) { + // lossy mode + return tsCompressDoubleLossyImp(input, nelements, output); + } else { +#endif + // lossless mode + if (algorithm == ONE_STAGE_COMP) { + return tsCompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressDoubleImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +#ifdef TD_TSZ + if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { + // decompress lossy + return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); + } else { +#endif + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressDoubleImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +// Binary ===================================================== +int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + return tsCompressStringImp(input, inputSize, output, outputSize); +} + +int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + return tsDecompressStringImp(input, compressedSize, output, outputSize); +} + +// Bool ===================================================== +int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressBoolImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressBoolImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressBoolImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressBoolImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +} + +// Tinyint ===================================================== +int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else { + assert(0); + return -1; + } +} + +// Smallint ===================================================== +int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, + char *const output, int32_t outputSize, char algorithm, char *const buffer, + int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else { + assert(0); + return -1; + } +} + +// Int ===================================================== +int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); + } else { + assert(0); + return -1; + } +} + +// Bigint ===================================================== +int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else if (algorithm == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, + int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else { + assert(0); + return -1; + } +} From 215883ea643d43392991380a24fcd4d6fd7e1607 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 11:08:46 +0800 Subject: [PATCH 02/64] refact code --- include/common/ttypes.h | 9 +- include/util/tcompression.h | 76 +++++----- source/util/src/tcompression.c | 249 ++++++++++++++++----------------- 3 files changed, 163 insertions(+), 171 deletions(-) diff --git a/include/common/ttypes.h b/include/common/ttypes.h index a88f65f6ac..14e9a5af87 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -333,10 +333,10 @@ typedef struct tDataTypeDescriptor { char *name; int64_t minValue; int64_t maxValue; - int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); + int32_t (*compFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); + int32_t (*decompFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; @@ -356,7 +356,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); void *getDataMin(int32_t type); void *getDataMax(int32_t type); - #ifdef __cplusplus } #endif diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 5196c01fc6..b620615ab1 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -83,46 +83,42 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int #endif -int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - -int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize); -int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize); -int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize); -int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); -int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); +int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf); +int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); +int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf); #ifdef __cplusplus } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 1128c51f9b..30b72f6fbe 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1618,27 +1618,26 @@ int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) { * REGULAR COMPRESSION *************************************************************************/ // Timestamp ===================================================== -int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressTimestampImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressTimestampImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressTimestampImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressTimestampImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressTimestampImp(buffer, nelements, output); +int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressTimestampImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressTimestampImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1646,20 +1645,20 @@ int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize, c } // Float ===================================================== -int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ // lossy mode if (lossyFloat) { - return tsCompressFloatLossyImp(input, nelements, output); + return tsCompressFloatLossyImp(pIn, nEle, pOut); // lossless mode } else { #endif - if (algorithm == ONE_STAGE_COMP) { - return tsCompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressFloatImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressFloatImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressFloatImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; @@ -1669,20 +1668,20 @@ int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_ #endif } -int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ - if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { + if (HEAD_ALGO(pIn[0]) == ALGO_SZ_LOSSY) { // decompress lossy - return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); + return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut); } else { #endif // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressFloatImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressFloatImp(buffer, nelements, output); + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressFloatImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressFloatImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1693,20 +1692,20 @@ int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const } // Double ===================================================== -int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ if (lossyDouble) { // lossy mode - return tsCompressDoubleLossyImp(input, nelements, output); + return tsCompressDoubleLossyImp(pIn, nEle, pOut); } else { #endif // lossless mode - if (algorithm == ONE_STAGE_COMP) { - return tsCompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressDoubleImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressDoubleImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressDoubleImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; @@ -1716,20 +1715,20 @@ int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32 #endif } -int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { +int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { #ifdef TD_TSZ if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) { // decompress lossy - return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); + return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut); } else { #endif // decompress lossless - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressDoubleImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressDoubleImp(buffer, nelements, output); + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressDoubleImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressDoubleImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1740,37 +1739,37 @@ int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, cons } // Binary ===================================================== -int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - return tsCompressStringImp(input, inputSize, output, outputSize); +int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + return tsCompressStringImp(pIn, nIn, pOut, nOut); } -int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - return tsDecompressStringImp(input, compressedSize, output, outputSize); +int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + return tsDecompressStringImp(pIn, nIn, pOut, nOut); } // Bool ===================================================== -int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressBoolImp(input, nelements, buffer); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressBoolImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressBoolImp(pIn, nEle, pBuf); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressBoolImp(input, nelements, output); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressBoolImp(buffer, nelements, output); +int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressBoolImp(pIn, nEle, pOut); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressBoolImp(pBuf, nEle, pOut); } else { assert(0); return -1; @@ -1778,27 +1777,26 @@ int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const } // Tinyint ===================================================== -int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_TINYINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); +int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_TINYINT); } else { assert(0); return -1; @@ -1806,27 +1804,26 @@ int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize, con } // Smallint ===================================================== -int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_SMALLINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, const int32_t nelements, - char *const output, int32_t outputSize, char algorithm, char *const buffer, - int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); +int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, + void *pBuf, int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_SMALLINT); } else { assert(0); return -1; @@ -1834,26 +1831,26 @@ int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize, co } // Int ===================================================== -int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_INT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); +int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_INT); } else { assert(0); return -1; @@ -1861,26 +1858,26 @@ int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const i } // Bigint ===================================================== -int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); - return tsCompressStringImp(buffer, len, output, outputSize); +int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_BIGINT); + return tsCompressStringImp(pBuf, len, pOut, nOut); } else { assert(0); return -1; } } -int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, - int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize) { - if (algorithm == ONE_STAGE_COMP) { - return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); - } else if (algorithm == TWO_STAGE_COMP) { - if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; - return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); +int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, + int32_t nBuf) { + if (cmprAlg == ONE_STAGE_COMP) { + return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT); + } else if (cmprAlg == TWO_STAGE_COMP) { + if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1; + return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_BIGINT); } else { assert(0); return -1; From fb45b8d5773049d629506b0f0b9dce9c07b954a0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 13:39:32 +0800 Subject: [PATCH 03/64] refact code --- include/util/tcompression.h | 14 +++++ source/dnode/vnode/src/inc/tsdb.h | 4 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 63 ++-------------------- source/util/src/tcompression.c | 1 - 4 files changed, 20 insertions(+), 62 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index b620615ab1..a884c4753f 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -83,6 +83,9 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int #endif +/************************************************************************* + * REGULAR COMPRESSION + *************************************************************************/ int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf); int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, @@ -120,6 +123,17 @@ int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf); +/************************************************************************* + * STREAM COMPRESSION + *************************************************************************/ +typedef struct SCompressor SCompressor; + +int32_t tCompressorCreate(SCompressor **ppCmprsor); +int32_t tCompressorDestroy(SCompressor *pCmprsor); +int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc); +int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData); +int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 817089f237..c2a31679d0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -285,8 +285,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char* id); -void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char* id); +int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char *id); +void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char *id); // tsdbMerge.c ============================================================================================== int32_t tsdbMerge(STsdb *pTsdb); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 3bd71f0ea6..0a42edc94c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -15,70 +15,15 @@ #include "tsdb.h" -typedef struct SDiskColBuilder SDiskColBuilder; -struct SDiskColBuilder { - uint8_t flags; - uint8_t *pBitMap; - int32_t *aOffset; - int32_t nData; - uint8_t *pData; -}; +typedef struct SDiskData SDiskData; -int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - // TODO - return code; -} - -// ================================================================ -typedef struct SDiskDataBuilder SDiskDataBuilder; -struct SDiskDataBuilder { +struct SDiskData { SDiskDataHdr hdr; - SArray *aBlockCol; // SArray + /* data */ }; -int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { +int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow) { int32_t code = 0; // TODO return code; } - -void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { - // TODO -} - -void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) { - pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, // - .fmtVer = 0, - .suid = suid, - .uid = uid, - .cmprAlg = cmprAlg}; -} - -void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) { - // TODO -} - -int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { - int32_t code = 0; - - // uid (todo) - - // version (todo) - - // TSKEY (todo) - - SRowIter iter = {0}; - tRowIterInit(&iter, pRow, pTSchema); - - for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) { - } - - return code; -} - -int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) { - int32_t code = 0; - // TODO - return code; -} \ No newline at end of file diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 30b72f6fbe..3d4df28440 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1000,7 +1000,6 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co * STREAM COMPRESSION *************************************************************************/ #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) -typedef struct SCompressor SCompressor; static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); From 556a6fcd7fc27f9da69fb590add4b53b1ce55efb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 14:49:34 +0800 Subject: [PATCH 04/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 207 ++++++++++++++++++++- 1 file changed, 203 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 0a42edc94c..b7b03e8288 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -16,14 +16,213 @@ #include "tsdb.h" typedef struct SDiskData SDiskData; +typedef struct SDiskCol SDiskCol; -struct SDiskData { - SDiskDataHdr hdr; - /* data */ +struct SDiskCol { + int16_t cid; + int8_t type; + int8_t flag; + int32_t nVal; + SCompressor *pBitC; + SCompressor *pOffC; + SCompressor *pValC; }; -int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow) { +// SDiskCol ================================================ +static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { + int32_t code = 0; + + pDiskCol->cid = cid; + pDiskCol->type = type; + pDiskCol->flag = 0; + pDiskCol->nVal = 0; + + tCompressorReset(pDiskCol->pBitC, TSDB_DATA_TYPE_TINYINT, cmprAlg, 1); + tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg, 1); + tCompressorReset(pDiskCol->pValC, type, cmprAlg, 1); + + return code; +} + +static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + pDiskCol->flag = HAS_NONE; + } else if (pColVal->isNull) { + pDiskCol->flag = HAS_NULL; + } else { + pDiskCol->flag = HAS_VALUE; + code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + if (code) goto _exit; + } + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (!pColVal->isNone) { + } + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (!pColVal->isNull) { + } + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + } else if (pColVal->isNull) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone || pColVal->isNull) { + } else { + } + + pDiskCol->nVal++; + +_exit: + return code; +} +static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNull) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (pColVal->isNone) { + } else if (pColVal->isNull) { + } else { + } + pDiskCol->nVal++; + + return code; +} +static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { + tDiskColAddVal0, // 0 + tDiskColAddVal1, // HAS_NONE + tDiskColAddVal2, // HAS_NULL + tDiskColAddVal3, // HAS_NULL|HAS_NONE + tDiskColAddVal4, // HAS_VALUE + tDiskColAddVal5, // HAS_VALUE|HAS_NONE + tDiskColAddVal6, // HAS_VALUE|HAS_NULL + tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE +}; + +// SDiskData ================================================ +struct SDiskData { + int64_t suid; + int64_t uid; + uint8_t cmprAlg; + SCompressor *pUidC; + SCompressor *pVerC; + SCompressor *pKeyC; + int32_t nDiskCol; + SArray *aDiskCol; +}; + +int32_t tDiskDataCreate(SDiskData *pDiskData) { int32_t code = 0; // TODO return code; } + +int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + // TODO + return code; +} + +int32_t tDiskDataReset(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { + int32_t code = 0; + + pDiskData->suid = pId->suid; + pDiskData->uid = pId->uid; + pDiskData->cmprAlg = cmprAlg; + + for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { + } + + return code; +} + +int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { + int32_t code = 0; + + ASSERT(pId->suid == pDiskData->suid); + + // uid + code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); + if (code) goto _exit; + + // version + int64_t version = TSDBROW_VERSION(pRow); + code = tCompress(pDiskData->pVerC, &version, sizeof(int64_t)); + if (code) goto _exit; + + // TSKEY + TSKEY ts = TSDBROW_TS(pRow); + code = tCompress(pDiskData->pVerC, &ts, sizeof(int64_t)); + if (code) goto _exit; + + SRowIter iter = {0}; + tRowIterInit(&iter, pRow, pTSchema); + + SColVal *pColVal = tRowIterNext(&iter); + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + while (pColVal && pColVal->cid < pDiskCol->cid) { + pColVal = tRowIterNext(&iter); + } + + if (pColVal == NULL || pColVal->cid > pDiskCol->cid) { + code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &COL_VAL_NONE(pDiskCol->cid, pDiskCol->type)); + if (code) goto _exit; + } else { + code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, pColVal); + if (code) goto _exit; + pColVal = tRowIterNext(&iter); + } + } + +_exit: + return code; +} From d197f5019c6bba6c52d8ada06452fac431842f6a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 16:18:12 +0800 Subject: [PATCH 05/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 163 +++++++++++++++++++-- 1 file changed, 154 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b7b03e8288..c688a64580 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -23,7 +23,9 @@ struct SDiskCol { int8_t type; int8_t flag; int32_t nVal; + uint8_t bit; SCompressor *pBitC; + int32_t offset; SCompressor *pOffC; SCompressor *pValC; }; @@ -44,7 +46,21 @@ static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8 return code; } -static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddValue(SDiskCol *pDiskCol, SColVal *pColVal) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColVal->type)) { + code = tCompress(pDiskCol->pOffC, &pDiskCol->offset, sizeof(int32_t)); + if (code) goto _exit; + pDiskCol->offset += pColVal->value.nData; + } + code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + if (code) goto _exit; + +_exit: + return code; +} +static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { // 0 int32_t code = 0; if (pColVal->isNone) { @@ -53,7 +69,7 @@ static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { pDiskCol->flag = HAS_NULL; } else { pDiskCol->flag = HAS_VALUE; - code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; } pDiskCol->nVal++; @@ -61,42 +77,144 @@ static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { _exit: return code; } -static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NONE int32_t code = 0; if (!pColVal->isNone) { + // bit map + pDiskCol->bit = 0; + for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } + SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); + if ((pDiskCol->nVal & 0x3) == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + pDiskCol->bit = 0; + } + + // value + if (pColVal->isNull) { + pDiskCol->flag |= HAS_NULL; + } else { + pDiskCol->flag |= HAS_VALUE; + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + code = tDiskColAddValue(pDiskCol, &cv); + if (code) goto _exit; + } + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; + } } pDiskCol->nVal++; _exit: return code; } -static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL int32_t code = 0; if (!pColVal->isNull) { + if (pColVal->isNone) { + pDiskCol->flag |= HAS_NONE; + + pDiskCol->bit = 1; + for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } + SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 0); + if ((pDiskCol->nVal & 0x3) == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + pDiskCol->bit = 0; + } + } else { + pDiskCol->flag |= HAS_VALUE; + + pDiskCol->bit = 0; + for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } + SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); + if ((pDiskCol->nVal & 0x3) == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + pDiskCol->bit = 0; + } + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + code = tDiskColAddValue(pDiskCol, &cv); + if (code) goto _exit; + } + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; + } } pDiskCol->nVal++; _exit: return code; } -static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; + uint8_t mod8 = (pDiskCol->nVal & 0x3); if (pColVal->isNone) { + SET_BIT1(&pDiskCol->bit, mod8, 0); + if (mod8 == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } } else if (pColVal->isNull) { + SET_BIT1(&pDiskCol->bit, mod8, 1); + if (mod8 == 7) { + code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); + if (code) goto _exit; + } } else { + pDiskCol->flag |= HAS_VALUE; + + // convert from bit1 to bit2 and add a 2 (todo) + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + code = tDiskColAddValue(pDiskCol, &cv); + if (code) goto _exit; + } + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; } pDiskCol->nVal++; +_exit: return code; } -static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE int32_t code = 0; if (pColVal->isNone || pColVal->isNull) { + if (pColVal->isNone) { + pDiskCol->flag |= HAS_NONE; + } else { + pDiskCol->flag |= HAS_NULL; + } + + // set bitmap (todo) + + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; } else { + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; } pDiskCol->nVal++; @@ -104,35 +222,62 @@ static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { _exit: return code; } -static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NONE int32_t code = 0; if (pColVal->isNull) { + pDiskCol->flag |= HAS_NULL; + + // convert bit1 to bit2 } else { + if (pColVal->isNone) { + // SET_BIT1(0); + } else { + // SET_BIT1(1); + } } + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; pDiskCol->nVal++; +_exit: return code; } -static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL int32_t code = 0; if (pColVal->isNone) { + pDiskCol->flag |= HAS_NONE; + // bit1 to bit2 } else { + if (pColVal->isNull) { + // SET_BIT1(0) + } else { + // SET_BIT1(1) + } } + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; pDiskCol->nVal++; +_exit: return code; } -static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE int32_t code = 0; if (pColVal->isNone) { + // set_bit2(0); } else if (pColVal->isNull) { + // set_bit2(1); } else { + // set_bit2(2); } + code = tDiskColAddValue(pDiskCol, pColVal); + if (code) goto _exit; pDiskCol->nVal++; +_exit: return code; } static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { From 750a1a8d8679375e4fad3aca2091d61697da57d5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 20 Sep 2022 18:56:36 +0800 Subject: [PATCH 06/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 139 ++++++++++++--------- 1 file changed, 80 insertions(+), 59 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index c688a64580..1266efcf8e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -23,8 +23,7 @@ struct SDiskCol { int8_t type; int8_t flag; int32_t nVal; - uint8_t bit; - SCompressor *pBitC; + uint8_t *pBitMap; int32_t offset; SCompressor *pOffC; SCompressor *pValC; @@ -39,7 +38,6 @@ static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8 pDiskCol->flag = 0; pDiskCol->nVal = 0; - tCompressorReset(pDiskCol->pBitC, TSDB_DATA_TYPE_TINYINT, cmprAlg, 1); tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg, 1); tCompressorReset(pDiskCol->pValC, type, cmprAlg, 1); @@ -82,17 +80,13 @@ static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_N if (!pColVal->isNone) { // bit map - pDiskCol->bit = 0; - for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } - SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); - if ((pDiskCol->nVal & 0x3) == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - pDiskCol->bit = 0; - } + int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + + code = tRealloc(&pDiskCol->pBitMap, nBit); + if (code) goto _exit; + + memset(pDiskCol->pBitMap, 0, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); // value if (pColVal->isNull) { @@ -119,34 +113,20 @@ static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_N int32_t code = 0; if (!pColVal->isNull) { + int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + code = tRealloc(&pDiskCol->pBitMap, nBit); + if (code) goto _exit; + if (pColVal->isNone) { pDiskCol->flag |= HAS_NONE; - pDiskCol->bit = 1; - for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } - SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 0); - if ((pDiskCol->nVal & 0x3) == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - pDiskCol->bit = 0; - } + memset(pDiskCol->pBitMap, 255, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else { pDiskCol->flag |= HAS_VALUE; - pDiskCol->bit = 0; - for (int32_t i = 0; i < (pDiskCol->nVal >> 3); i++) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } - SET_BIT1(&pDiskCol->bit, (pDiskCol->nVal & 0x3), 1); - if ((pDiskCol->nVal & 0x3) == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - pDiskCol->bit = 0; - } + memset(pDiskCol->pBitMap, 0, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { @@ -166,23 +146,30 @@ _exit: static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; - uint8_t mod8 = (pDiskCol->nVal & 0x3); if (pColVal->isNone) { - SET_BIT1(&pDiskCol->bit, mod8, 0); - if (mod8 == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else if (pColVal->isNull) { - SET_BIT1(&pDiskCol->bit, mod8, 1); - if (mod8 == 7) { - code = tCompress(pDiskCol->pBitC, &pDiskCol->bit, 1); - if (code) goto _exit; - } + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); } else { pDiskCol->flag |= HAS_VALUE; - // convert from bit1 to bit2 and add a 2 (todo) + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pDiskCol->nVal, 2); + + tFree(pDiskCol->pBitMap); + pDiskCol->pBitMap = pBitMap; SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { @@ -208,7 +195,12 @@ static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V pDiskCol->flag |= HAS_NULL; } - // set bitmap (todo) + int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + code = tRealloc(&pDiskCol->pBitMap, nBit); + if (code) goto _exit; + + memset(pDiskCol->pBitMap, 255, nBit); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; @@ -216,7 +208,6 @@ static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; _exit: @@ -228,12 +219,25 @@ static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V if (pColVal->isNull) { pDiskCol->flag |= HAS_NULL; - // convert bit1 to bit2 + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pDiskCol->nVal, 1); + + tFree(pDiskCol->pBitMap); + pDiskCol->pBitMap = pBitMap; } else { + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + if (pColVal->isNone) { - // SET_BIT1(0); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else { - // SET_BIT1(1); + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); } } code = tDiskColAddValue(pDiskCol, pColVal); @@ -248,12 +252,26 @@ static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_V if (pColVal->isNone) { pDiskCol->flag |= HAS_NONE; - // bit1 to bit2 + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pDiskCol->nVal, 0); + + tFree(pDiskCol->pBitMap); + pDiskCol->pBitMap = pBitMap; } else { + code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + if (pColVal->isNull) { - // SET_BIT1(0) + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else { - // SET_BIT1(1) + SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); } } code = tDiskColAddValue(pDiskCol, pColVal); @@ -266,12 +284,15 @@ _exit: static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE int32_t code = 0; + code = tRealloc(&pDiskCol->pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + if (code) goto _exit; + if (pColVal->isNone) { - // set_bit2(0); + SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 0); } else if (pColVal->isNull) { - // set_bit2(1); + SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 1); } else { - // set_bit2(2); + SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 2); } code = tDiskColAddValue(pDiskCol, pColVal); if (code) goto _exit; From b4a43079fb04d2e9f71301a5ab7c1e184841345f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 11:03:08 +0800 Subject: [PATCH 07/64] more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 110 ++++++++++++++++++--- source/util/src/tcompression.c | 16 ++- 3 files changed, 102 insertions(+), 26 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index a884c4753f..fd01bcf1a3 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -130,7 +130,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc); +int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 1266efcf8e..c20a598b6c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -22,6 +22,7 @@ struct SDiskCol { int16_t cid; int8_t type; int8_t flag; + uint8_t cmprAlg; int32_t nVal; uint8_t *pBitMap; int32_t offset; @@ -30,16 +31,41 @@ struct SDiskCol { }; // SDiskCol ================================================ -static int32_t tDiskColReset(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { +static int32_t tDiskColInit(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; pDiskCol->cid = cid; pDiskCol->type = type; pDiskCol->flag = 0; + pDiskCol->cmprAlg = cmprAlg; pDiskCol->nVal = 0; + pDiskCol->offset = 0; - tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg, 1); - tCompressorReset(pDiskCol->pValC, type, cmprAlg, 1); + if (IS_VAR_DATA_TYPE(type)) { + if (pDiskCol->pOffC == NULL) { + code = tCompressorCreate(&pDiskCol->pOffC); + if (code) return code; + } + code = tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + if (code) return code; + } + + if (pDiskCol->pValC == NULL) { + code = tCompressorCreate(&pDiskCol->pValC); + if (code) return code; + } + code = tCompressorReset(pDiskCol->pValC, type, cmprAlg); + if (code) return code; + + return code; +} + +static int32_t tDiskColClear(SDiskCol *pDiskCol) { + int32_t code = 0; + + tFree(pDiskCol->pBitMap); + if (pDiskCol->pOffC) tCompressorDestroy(pDiskCol->pOffC); + if (pDiskCol->pValC) tCompressorDestroy(pDiskCol->pValC); return code; } @@ -324,26 +350,78 @@ struct SDiskData { SArray *aDiskCol; }; -int32_t tDiskDataCreate(SDiskData *pDiskData) { - int32_t code = 0; - // TODO - return code; -} - -int32_t tDiskDataDestroy(SDiskData *pDiskData) { - int32_t code = 0; - // TODO - return code; -} - -int32_t tDiskDataReset(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { +int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { int32_t code = 0; pDiskData->suid = pId->suid; pDiskData->uid = pId->uid; pDiskData->cmprAlg = cmprAlg; + if (pDiskData->pUidC == NULL) { + code = tCompressorCreate(&pDiskData->pUidC); + if (code) return code; + } + code = tCompressorReset(pDiskData->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (code) return code; + + if (pDiskData->pVerC == NULL) { + code = tCompressorCreate(&pDiskData->pVerC); + if (code) return code; + } + code = tCompressorReset(pDiskData->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (code) return code; + + if (pDiskData->pKeyC == NULL) { + code = tCompressorCreate(&pDiskData->pKeyC); + if (code) return code; + } + code = tCompressorReset(pDiskData->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + if (code) return code; + + if (pDiskData->aDiskCol == NULL) { + pDiskData->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); + if (pDiskData->aDiskCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + pDiskData->nDiskCol = 0; for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { + STColumn *pTColumn = &pTSchema->columns[iCol]; + + if (pDiskData->nDiskCol >= taosArrayGetSize(pDiskData->aDiskCol)) { + SDiskCol dc = (SDiskCol){0}; + if (taosArrayPush(pDiskData->aDiskCol, &dc) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } + + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, pDiskData->nDiskCol); + + code = tDiskColInit(pDiskCol, pTColumn->colId, pTColumn->type, cmprAlg); + if (code) return code; + + pDiskData->nDiskCol++; + } + + return code; +} + +int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + + if (pDiskData->pUidC) tCompressorDestroy(pDiskData->pUidC); + if (pDiskData->pVerC) tCompressorDestroy(pDiskData->pVerC); + if (pDiskData->pKeyC) tCompressorDestroy(pDiskData->pKeyC); + + if (pDiskData->aDiskCol) { + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + tDiskColClear(pDiskCol); + } + taosArrayDestroy(pDiskData->aDiskCol); } return code; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 3d4df28440..5d6e6d12ac 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1508,24 +1508,22 @@ _exit: int32_t tCompressorDestroy(SCompressor *pCmprsor) { int32_t code = 0; - if (pCmprsor) { - int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); - for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { - tFree(pCmprsor->aBuf[iBuf]); - } - - taosMemoryFree(pCmprsor); + int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); + for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { + tFree(pCmprsor->aBuf[iBuf]); } + taosMemoryFree(pCmprsor); + return code; } -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc) { +int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; pCmprsor->type = type; pCmprsor->cmprAlg = cmprAlg; - pCmprsor->autoAlloc = autoAlloc; + pCmprsor->autoAlloc = 1; pCmprsor->nVal = 0; switch (type) { From 8ffc7e9f3e42c3242f746d1d7a51e1f475718189 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 11:28:20 +0800 Subject: [PATCH 08/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 8 ++++++-- source/dnode/vnode/src/tsdb/tsdbUtil.c | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index c20a598b6c..3904012988 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -342,6 +342,7 @@ static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { struct SDiskData { int64_t suid; int64_t uid; + int32_t nRow; uint8_t cmprAlg; SCompressor *pUidC; SCompressor *pVerC; @@ -355,6 +356,7 @@ int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, ui pDiskData->suid = pId->suid; pDiskData->uid = pId->uid; + pDiskData->nRow = 0; pDiskData->cmprAlg = cmprAlg; if (pDiskData->pUidC == NULL) { @@ -443,7 +445,7 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, // TSKEY TSKEY ts = TSDBROW_TS(pRow); - code = tCompress(pDiskData->pVerC, &ts, sizeof(int64_t)); + code = tCompress(pDiskData->pKeyC, &ts, sizeof(int64_t)); if (code) goto _exit; SRowIter iter = {0}; @@ -458,7 +460,8 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, } if (pColVal == NULL || pColVal->cid > pDiskCol->cid) { - code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &COL_VAL_NONE(pDiskCol->cid, pDiskCol->type)); + SColVal cv = COL_VAL_NONE(pDiskCol->cid, pDiskCol->type); + code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &cv); if (code) goto _exit; } else { code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, pColVal); @@ -466,6 +469,7 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, pColVal = tRowIterNext(&iter); } } + pDiskData->nRow++; _exit: return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 64c150484b..4e6e1f2f7f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1082,9 +1082,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS pColVal = tRowIterNext(&rIter); } } - -_exit: pBlockData->nRow++; + return code; _err: From ce9afbae19c86ae3707cd95226c4bd583802edff Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 14:02:52 +0800 Subject: [PATCH 09/64] more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 101 ++++++++++++++++++++- source/util/src/tcompression.c | 2 +- 3 files changed, 101 insertions(+), 4 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index fd01bcf1a3..8d7c607e6b 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -131,7 +131,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData); +int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 3904012988..a5104dcfbd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -349,6 +349,7 @@ struct SDiskData { SCompressor *pKeyC; int32_t nDiskCol; SArray *aDiskCol; + uint8_t *aBuf[2]; }; int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { @@ -425,6 +426,9 @@ int32_t tDiskDataDestroy(SDiskData *pDiskData) { } taosArrayDestroy(pDiskData->aDiskCol); } + for (int32_t iBuf = 0; iBuf < sizeof(pDiskData->aBuf) / sizeof(pDiskData->aBuf[0]); iBuf++) { + tFree(pDiskData->aBuf[iBuf]); + } return code; } @@ -435,8 +439,17 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, ASSERT(pId->suid == pDiskData->suid); // uid - code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); - if (code) goto _exit; + if (pDiskData->uid && pDiskData->uid != pId->uid) { + for (int32_t iRow = 0; iRow < pDiskData->nRow; iRow++) { + code = tCompress(pDiskData->pUidC, &pDiskData->uid, sizeof(int64_t)); + if (code) goto _exit; + } + pDiskData->uid = 0; + } + if (pDiskData->uid == 0) { + code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); + if (code) goto _exit; + } // version int64_t version = TSDBROW_VERSION(pRow); @@ -474,3 +487,87 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, _exit: return code; } + +int32_t tDiskDataToBinary(SDiskData *pDiskData, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + ASSERT(pDiskData->nRow); + + SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pDiskData->suid, + .uid = pDiskData->uid, + .szUid = 0, + .szVer = 0, + .szKey = 0, + .szBlkCol = 0, + .nRow = pDiskData->nRow, + .cmprAlg = pDiskData->cmprAlg}; + + // UID + const uint8_t *pUid = NULL; + if (pDiskData->uid == 0) { + code = tCompGen(pDiskData->pUidC, &pUid, &hdr.szUid); + if (code) return code; + } + + // VERSION + const uint8_t *pVer = NULL; + code = tCompGen(pDiskData->pVerC, &pVer, &hdr.szVer); + if (code) return code; + + // TSKEY + const uint8_t *pKey = NULL; + code = tCompGen(pDiskData->pKeyC, &pKey, &hdr.szKey); + if (code) return code; + + int32_t offset = 0; + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (pDiskCol->flag == HAS_NONE) continue; + + // code = tDiskColToBinary(pDiskCol, ); + // if (code) return code; + + SBlockCol bCol = {.cid = pDiskCol->cid, + .type = pDiskCol->type, + // .smaOn = , + .flag = pDiskCol->flag, + // .szOrigin = + // .szBitmap = + // .szOffset = + // .szValue = + .offset = offset}; + + hdr.szBlkCol += tPutBlockCol(NULL, &bCol); + offset = offset + bCol.szBitmap + bCol.szOffset + bCol.szValue; + } + + *nData = tPutDiskDataHdr(NULL, &hdr) + hdr.szUid + hdr.szVer + hdr.szKey + hdr.szBlkCol + offset; + code = tRealloc(&pDiskData->aBuf[0], *nData); + if (code) return code; + *ppData = pDiskData->aBuf[0]; + + int32_t n = 0; + n += tPutDiskDataHdr(pDiskData->aBuf[0] + n, &hdr); + if (hdr.szUid) { + memcpy(pDiskData->aBuf[0] + n, pUid, hdr.szUid); + n += hdr.szUid; + } + memcpy(pDiskData->aBuf[0] + n, pVer, hdr.szVer); + n += hdr.szVer; + memcpy(pDiskData->aBuf[0] + n, pKey, hdr.szKey); + n += hdr.szKey; + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + n += tPutBlockCol(pDiskData->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); + } + for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + // memcpy(pDiskData->aBuf[0] + n, NULL, ); + // n += 0; + } + + return code; +} diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 5d6e6d12ac..cd9ff5e325 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1574,7 +1574,7 @@ int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { return code; } -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) { +int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; if (pCmprsor->nVal == 0) { From f51c86430a381a9bae6ec1c80726b3a0c09feb22 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 14:50:14 +0800 Subject: [PATCH 10/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 198 +++++++++++++-------- 1 file changed, 124 insertions(+), 74 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index a5104dcfbd..a0c1bb8934 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -15,8 +15,9 @@ #include "tsdb.h" -typedef struct SDiskData SDiskData; -typedef struct SDiskCol SDiskCol; +typedef struct SDiskDataBuilder SDiskDataBuilder; +typedef struct SDiskCol SDiskCol; +typedef struct SDiskData SDiskData; struct SDiskCol { int16_t cid; @@ -70,6 +71,34 @@ static int32_t tDiskColClear(SDiskCol *pDiskCol) { return code; } +static int32_t tDiskColToBinary(SDiskCol *pDiskCol, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + + ASSERT(pDiskCol->flag && pDiskCol->flag != HAS_NONE); + + if (pDiskCol->flag == HAS_NULL) { + return code; + } + + // bitmap (todo) + if (pDiskCol->flag != HAS_VALUE) { + } + + // offset (todo) + if (IS_VAR_DATA_TYPE(pDiskCol->type)) { + code = tCompGen(pDiskCol->pOffC, NULL /* todo */, NULL /* todo */); + if (code) return code; + } + + // value (todo) + if (pDiskCol->flag != (HAS_NULL | HAS_NONE)) { + code = tCompGen(pDiskCol->pValC, NULL /* todo */, NULL /* todo */); + if (code) return code; + } + + return code; +} + static int32_t tDiskColAddValue(SDiskCol *pDiskCol, SColVal *pColVal) { int32_t code = 0; @@ -338,8 +367,8 @@ static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE }; -// SDiskData ================================================ -struct SDiskData { +// SDiskDataBuilder ================================================ +struct SDiskDataBuilder { int64_t suid; int64_t uid; int32_t nRow; @@ -352,121 +381,121 @@ struct SDiskData { uint8_t *aBuf[2]; }; -int32_t tDiskDataInit(SDiskData *pDiskData, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { int32_t code = 0; - pDiskData->suid = pId->suid; - pDiskData->uid = pId->uid; - pDiskData->nRow = 0; - pDiskData->cmprAlg = cmprAlg; + pBuilder->suid = pId->suid; + pBuilder->uid = pId->uid; + pBuilder->nRow = 0; + pBuilder->cmprAlg = cmprAlg; - if (pDiskData->pUidC == NULL) { - code = tCompressorCreate(&pDiskData->pUidC); + if (pBuilder->pUidC == NULL) { + code = tCompressorCreate(&pBuilder->pUidC); if (code) return code; } - code = tCompressorReset(pDiskData->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressorReset(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pDiskData->pVerC == NULL) { - code = tCompressorCreate(&pDiskData->pVerC); + if (pBuilder->pVerC == NULL) { + code = tCompressorCreate(&pBuilder->pVerC); if (code) return code; } - code = tCompressorReset(pDiskData->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressorReset(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pDiskData->pKeyC == NULL) { - code = tCompressorCreate(&pDiskData->pKeyC); + if (pBuilder->pKeyC == NULL) { + code = tCompressorCreate(&pBuilder->pKeyC); if (code) return code; } - code = tCompressorReset(pDiskData->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; - if (pDiskData->aDiskCol == NULL) { - pDiskData->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); - if (pDiskData->aDiskCol == NULL) { + if (pBuilder->aDiskCol == NULL) { + pBuilder->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); + if (pBuilder->aDiskCol == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - pDiskData->nDiskCol = 0; + pBuilder->nDiskCol = 0; for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { STColumn *pTColumn = &pTSchema->columns[iCol]; - if (pDiskData->nDiskCol >= taosArrayGetSize(pDiskData->aDiskCol)) { + if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskCol)) { SDiskCol dc = (SDiskCol){0}; - if (taosArrayPush(pDiskData->aDiskCol, &dc) == NULL) { + if (taosArrayPush(pBuilder->aDiskCol, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, pDiskData->nDiskCol); + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, pBuilder->nDiskCol); code = tDiskColInit(pDiskCol, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; - pDiskData->nDiskCol++; + pBuilder->nDiskCol++; } return code; } -int32_t tDiskDataDestroy(SDiskData *pDiskData) { +int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { int32_t code = 0; - if (pDiskData->pUidC) tCompressorDestroy(pDiskData->pUidC); - if (pDiskData->pVerC) tCompressorDestroy(pDiskData->pVerC); - if (pDiskData->pKeyC) tCompressorDestroy(pDiskData->pKeyC); + if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); + if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); + if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - if (pDiskData->aDiskCol) { - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + if (pBuilder->aDiskCol) { + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pBuilder->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); tDiskColClear(pDiskCol); } - taosArrayDestroy(pDiskData->aDiskCol); + taosArrayDestroy(pBuilder->aDiskCol); } - for (int32_t iBuf = 0; iBuf < sizeof(pDiskData->aBuf) / sizeof(pDiskData->aBuf[0]); iBuf++) { - tFree(pDiskData->aBuf[iBuf]); + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); } return code; } -int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { +int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { int32_t code = 0; - ASSERT(pId->suid == pDiskData->suid); + ASSERT(pId->suid == pBuilder->suid); // uid - if (pDiskData->uid && pDiskData->uid != pId->uid) { - for (int32_t iRow = 0; iRow < pDiskData->nRow; iRow++) { - code = tCompress(pDiskData->pUidC, &pDiskData->uid, sizeof(int64_t)); + if (pBuilder->uid && pBuilder->uid != pId->uid) { + for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { + code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); if (code) goto _exit; } - pDiskData->uid = 0; + pBuilder->uid = 0; } - if (pDiskData->uid == 0) { - code = tCompress(pDiskData->pUidC, &pId->uid, sizeof(int64_t)); + if (pBuilder->uid == 0) { + code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); if (code) goto _exit; } // version int64_t version = TSDBROW_VERSION(pRow); - code = tCompress(pDiskData->pVerC, &version, sizeof(int64_t)); + code = tCompress(pBuilder->pVerC, &version, sizeof(int64_t)); if (code) goto _exit; // TSKEY TSKEY ts = TSDBROW_TS(pRow); - code = tCompress(pDiskData->pKeyC, &ts, sizeof(int64_t)); + code = tCompress(pBuilder->pKeyC, &ts, sizeof(int64_t)); if (code) goto _exit; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); while (pColVal && pColVal->cid < pDiskCol->cid) { pColVal = tRowIterNext(&iter); @@ -482,53 +511,53 @@ int32_t tDiskDataAddRow(SDiskData *pDiskData, TSDBROW *pRow, STSchema *pTSchema, pColVal = tRowIterNext(&iter); } } - pDiskData->nRow++; + pBuilder->nRow++; _exit: return code; } -int32_t tDiskDataToBinary(SDiskData *pDiskData, const uint8_t **ppData, int32_t *nData) { +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { int32_t code = 0; - ASSERT(pDiskData->nRow); + ASSERT(pBuilder->nRow); SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, .fmtVer = 0, - .suid = pDiskData->suid, - .uid = pDiskData->uid, + .suid = pBuilder->suid, + .uid = pBuilder->uid, .szUid = 0, .szVer = 0, .szKey = 0, .szBlkCol = 0, - .nRow = pDiskData->nRow, - .cmprAlg = pDiskData->cmprAlg}; + .nRow = pBuilder->nRow, + .cmprAlg = pBuilder->cmprAlg}; // UID const uint8_t *pUid = NULL; - if (pDiskData->uid == 0) { - code = tCompGen(pDiskData->pUidC, &pUid, &hdr.szUid); + if (pBuilder->uid == 0) { + code = tCompGen(pBuilder->pUidC, &pUid, &hdr.szUid); if (code) return code; } // VERSION const uint8_t *pVer = NULL; - code = tCompGen(pDiskData->pVerC, &pVer, &hdr.szVer); + code = tCompGen(pBuilder->pVerC, &pVer, &hdr.szVer); if (code) return code; // TSKEY const uint8_t *pKey = NULL; - code = tCompGen(pDiskData->pKeyC, &pKey, &hdr.szKey); + code = tCompGen(pBuilder->pKeyC, &pKey, &hdr.szKey); if (code) return code; int32_t offset = 0; - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); if (pDiskCol->flag == HAS_NONE) continue; - // code = tDiskColToBinary(pDiskCol, ); - // if (code) return code; + code = tDiskColToBinary(pDiskCol, NULL, NULL); + if (code) return code; SBlockCol bCol = {.cid = pDiskCol->cid, .type = pDiskCol->type, @@ -544,30 +573,51 @@ int32_t tDiskDataToBinary(SDiskData *pDiskData, const uint8_t **ppData, int32_t offset = offset + bCol.szBitmap + bCol.szOffset + bCol.szValue; } +#if 0 *nData = tPutDiskDataHdr(NULL, &hdr) + hdr.szUid + hdr.szVer + hdr.szKey + hdr.szBlkCol + offset; - code = tRealloc(&pDiskData->aBuf[0], *nData); + code = tRealloc(&pBuilder->aBuf[0], *nData); if (code) return code; - *ppData = pDiskData->aBuf[0]; + *ppData = pBuilder->aBuf[0]; int32_t n = 0; - n += tPutDiskDataHdr(pDiskData->aBuf[0] + n, &hdr); + n += tPutDiskDataHdr(pBuilder->aBuf[0] + n, &hdr); if (hdr.szUid) { - memcpy(pDiskData->aBuf[0] + n, pUid, hdr.szUid); + memcpy(pBuilder->aBuf[0] + n, pUid, hdr.szUid); n += hdr.szUid; } - memcpy(pDiskData->aBuf[0] + n, pVer, hdr.szVer); + memcpy(pBuilder->aBuf[0] + n, pVer, hdr.szVer); n += hdr.szVer; - memcpy(pDiskData->aBuf[0] + n, pKey, hdr.szKey); + memcpy(pBuilder->aBuf[0] + n, pKey, hdr.szKey); n += hdr.szKey; - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - n += tPutBlockCol(pDiskData->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + n += tPutBlockCol(pBuilder->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); } - for (int32_t iDiskCol = 0; iDiskCol < pDiskData->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); // memcpy(pDiskData->aBuf[0] + n, NULL, ); // n += 0; } +#endif + + return code; +} + +// SDiskData ================================================ +struct SDiskData { + SDiskDataHdr hdr; + const uint8_t *pUid; + const uint8_t *pVer; + const uint8_t *pKey; + SArray *aBlockCol; + SArray *aColData; +}; + +int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + + if (pDiskData->aBlockCol) taosArrayDestroy(pDiskData->aBlockCol); + if (pDiskData->aColData) taosArrayDestroy(pDiskData->aColData); return code; } From e12ac1dd6f9a0dd40bb7a8d1db3b649525803350 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 15:00:32 +0800 Subject: [PATCH 11/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 309 +++++++++++---------- 1 file changed, 155 insertions(+), 154 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index a0c1bb8934..a514bddabb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -16,10 +16,10 @@ #include "tsdb.h" typedef struct SDiskDataBuilder SDiskDataBuilder; -typedef struct SDiskCol SDiskCol; +typedef struct SDiskColBuilder SDiskColBuilder; typedef struct SDiskData SDiskData; -struct SDiskCol { +struct SDiskColBuilder { int16_t cid; int8_t type; int8_t flag; @@ -31,332 +31,332 @@ struct SDiskCol { SCompressor *pValC; }; -// SDiskCol ================================================ -static int32_t tDiskColInit(SDiskCol *pDiskCol, int16_t cid, int8_t type, uint8_t cmprAlg) { +// SDiskColBuilder ================================================ +static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; - pDiskCol->cid = cid; - pDiskCol->type = type; - pDiskCol->flag = 0; - pDiskCol->cmprAlg = cmprAlg; - pDiskCol->nVal = 0; - pDiskCol->offset = 0; + pBuilder->cid = cid; + pBuilder->type = type; + pBuilder->flag = 0; + pBuilder->cmprAlg = cmprAlg; + pBuilder->nVal = 0; + pBuilder->offset = 0; if (IS_VAR_DATA_TYPE(type)) { - if (pDiskCol->pOffC == NULL) { - code = tCompressorCreate(&pDiskCol->pOffC); + if (pBuilder->pOffC == NULL) { + code = tCompressorCreate(&pBuilder->pOffC); if (code) return code; } - code = tCompressorReset(pDiskCol->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + code = tCompressorReset(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } - if (pDiskCol->pValC == NULL) { - code = tCompressorCreate(&pDiskCol->pValC); + if (pBuilder->pValC == NULL) { + code = tCompressorCreate(&pBuilder->pValC); if (code) return code; } - code = tCompressorReset(pDiskCol->pValC, type, cmprAlg); + code = tCompressorReset(pBuilder->pValC, type, cmprAlg); if (code) return code; return code; } -static int32_t tDiskColClear(SDiskCol *pDiskCol) { +static int32_t tDiskColClear(SDiskColBuilder *pBuilder) { int32_t code = 0; - tFree(pDiskCol->pBitMap); - if (pDiskCol->pOffC) tCompressorDestroy(pDiskCol->pOffC); - if (pDiskCol->pValC) tCompressorDestroy(pDiskCol->pValC); + tFree(pBuilder->pBitMap); + if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); + if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); return code; } -static int32_t tDiskColToBinary(SDiskCol *pDiskCol, const uint8_t **ppData, int32_t *nData) { +static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SBlockCol *pBlockCol, const uint8_t **ppData) { int32_t code = 0; - ASSERT(pDiskCol->flag && pDiskCol->flag != HAS_NONE); + ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); - if (pDiskCol->flag == HAS_NULL) { + if (pBuilder->flag == HAS_NULL) { return code; } // bitmap (todo) - if (pDiskCol->flag != HAS_VALUE) { + if (pBuilder->flag != HAS_VALUE) { } // offset (todo) - if (IS_VAR_DATA_TYPE(pDiskCol->type)) { - code = tCompGen(pDiskCol->pOffC, NULL /* todo */, NULL /* todo */); + if (IS_VAR_DATA_TYPE(pBuilder->type)) { + code = tCompGen(pBuilder->pOffC, NULL /* todo */, NULL /* todo */); if (code) return code; } // value (todo) - if (pDiskCol->flag != (HAS_NULL | HAS_NONE)) { - code = tCompGen(pDiskCol->pValC, NULL /* todo */, NULL /* todo */); + if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { + code = tCompGen(pBuilder->pValC, NULL /* todo */, NULL /* todo */); if (code) return code; } return code; } -static int32_t tDiskColAddValue(SDiskCol *pDiskCol, SColVal *pColVal) { +static int32_t tDiskColAddValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColVal->type)) { - code = tCompress(pDiskCol->pOffC, &pDiskCol->offset, sizeof(int32_t)); + code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); if (code) goto _exit; - pDiskCol->offset += pColVal->value.nData; + pBuilder->offset += pColVal->value.nData; } - code = tCompress(pDiskCol->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); + code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); if (code) goto _exit; _exit: return code; } -static int32_t tDiskColAddVal0(SDiskCol *pDiskCol, SColVal *pColVal) { // 0 +static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 int32_t code = 0; if (pColVal->isNone) { - pDiskCol->flag = HAS_NONE; + pBuilder->flag = HAS_NONE; } else if (pColVal->isNull) { - pDiskCol->flag = HAS_NULL; + pBuilder->flag = HAS_NULL; } else { - pDiskCol->flag = HAS_VALUE; - code = tDiskColAddValue(pDiskCol, pColVal); + pBuilder->flag = HAS_VALUE; + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal1(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NONE +static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE int32_t code = 0; if (!pColVal->isNone) { // bit map - int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pDiskCol->pBitMap, nBit); + code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; - memset(pDiskCol->pBitMap, 0, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); // value if (pColVal->isNull) { - pDiskCol->flag |= HAS_NULL; + pBuilder->flag |= HAS_NULL; } else { - pDiskCol->flag |= HAS_VALUE; + pBuilder->flag |= HAS_VALUE; SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - code = tDiskColAddValue(pDiskCol, &cv); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColAddValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal2(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL +static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL int32_t code = 0; if (!pColVal->isNull) { - int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); - code = tRealloc(&pDiskCol->pBitMap, nBit); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; if (pColVal->isNone) { - pDiskCol->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; - memset(pDiskCol->pBitMap, 255, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { - pDiskCol->flag |= HAS_VALUE; + pBuilder->flag |= HAS_VALUE; - memset(pDiskCol->pBitMap, 0, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - code = tDiskColAddValue(pDiskCol, &cv); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColAddValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal3(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; if (pColVal->isNone) { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else if (pColVal->isNull) { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } else { - pDiskCol->flag |= HAS_VALUE; + pBuilder->flag |= HAS_VALUE; uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal)); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); } - SET_BIT2(pBitMap, pDiskCol->nVal, 2); + SET_BIT2(pBitMap, pBuilder->nVal, 2); - tFree(pDiskCol->pBitMap); - pDiskCol->pBitMap = pBitMap; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - code = tDiskColAddValue(pDiskCol, &cv); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColAddValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal4(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE +static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE int32_t code = 0; if (pColVal->isNone || pColVal->isNull) { if (pColVal->isNone) { - pDiskCol->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; } else { - pDiskCol->flag |= HAS_NULL; + pBuilder->flag |= HAS_NULL; } - int32_t nBit = BIT1_SIZE(pDiskCol->nVal + 1); - code = tRealloc(&pDiskCol->pBitMap, nBit); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; - memset(pDiskCol->pBitMap, 255, nBit); - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } else { - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; } - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal5(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NONE +static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NONE int32_t code = 0; if (pColVal->isNull) { - pDiskCol->flag |= HAS_NULL; + pBuilder->flag |= HAS_NULL; uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 0); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); } - SET_BIT2(pBitMap, pDiskCol->nVal, 1); + SET_BIT2(pBitMap, pBuilder->nVal, 1); - tFree(pDiskCol->pBitMap); - pDiskCol->pBitMap = pBitMap; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; } else { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; if (pColVal->isNone) { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal6(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL +static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL int32_t code = 0; if (pColVal->isNone) { - pDiskCol->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - for (int32_t iVal = 0; iVal < pDiskCol->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pDiskCol->pBitMap, iVal) ? 2 : 1); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); } - SET_BIT2(pBitMap, pDiskCol->nVal, 0); + SET_BIT2(pBitMap, pBuilder->nVal, 0); - tFree(pDiskCol->pBitMap); - pDiskCol->pBitMap = pBitMap; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; } else { - code = tRealloc(&pDiskCol->pBitMap, BIT1_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; if (pColVal->isNull) { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { - SET_BIT1(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t tDiskColAddVal7(SDiskCol *pDiskCol, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE +static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE int32_t code = 0; - code = tRealloc(&pDiskCol->pBitMap, BIT2_SIZE(pDiskCol->nVal + 1)); + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; if (pColVal->isNone) { - SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 0); + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); } else if (pColVal->isNull) { - SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 1); + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); } else { - SET_BIT2(pDiskCol->pBitMap, pDiskCol->nVal, 2); + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); } - code = tDiskColAddValue(pDiskCol, pColVal); + code = tDiskColAddValue(pBuilder, pColVal); if (code) goto _exit; - pDiskCol->nVal++; + pBuilder->nVal++; _exit: return code; } -static int32_t (*tDiskColAddValImpl[])(SDiskCol *pDiskCol, SColVal *pColVal) = { +static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { tDiskColAddVal0, // 0 tDiskColAddVal1, // HAS_NONE tDiskColAddVal2, // HAS_NULL @@ -377,7 +377,7 @@ struct SDiskDataBuilder { SCompressor *pVerC; SCompressor *pKeyC; int32_t nDiskCol; - SArray *aDiskCol; + SArray *aDiskColBuilder; uint8_t *aBuf[2]; }; @@ -410,9 +410,9 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; - if (pBuilder->aDiskCol == NULL) { - pBuilder->aDiskCol = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskCol)); - if (pBuilder->aDiskCol == NULL) { + if (pBuilder->aDiskColBuilder == NULL) { + pBuilder->aDiskColBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); + if (pBuilder->aDiskColBuilder == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } @@ -422,17 +422,17 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { STColumn *pTColumn = &pTSchema->columns[iCol]; - if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskCol)) { - SDiskCol dc = (SDiskCol){0}; - if (taosArrayPush(pBuilder->aDiskCol, &dc) == NULL) { + if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskColBuilder)) { + SDiskColBuilder dc = (SDiskColBuilder){0}; + if (taosArrayPush(pBuilder->aDiskColBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, pBuilder->nDiskCol); + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, pBuilder->nDiskCol); - code = tDiskColInit(pDiskCol, pTColumn->colId, pTColumn->type, cmprAlg); + code = tDiskColInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; pBuilder->nDiskCol++; @@ -448,12 +448,13 @@ int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - if (pBuilder->aDiskCol) { - for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pBuilder->aDiskCol); iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); - tDiskColClear(pDiskCol); + if (pBuilder->aDiskColBuilder) { + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aDiskColBuilder); + iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + tDiskColClear(pDiskColBuilder); } - taosArrayDestroy(pBuilder->aDiskCol); + taosArrayDestroy(pBuilder->aDiskColBuilder); } for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { tFree(pBuilder->aBuf[iBuf]); @@ -494,19 +495,19 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - while (pColVal && pColVal->cid < pDiskCol->cid) { + while (pColVal && pColVal->cid < pDiskColBuilder->cid) { pColVal = tRowIterNext(&iter); } - if (pColVal == NULL || pColVal->cid > pDiskCol->cid) { - SColVal cv = COL_VAL_NONE(pDiskCol->cid, pDiskCol->type); - code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, &cv); + if (pColVal == NULL || pColVal->cid > pDiskColBuilder->cid) { + SColVal cv = COL_VAL_NONE(pDiskColBuilder->cid, pDiskColBuilder->type); + code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, &cv); if (code) goto _exit; } else { - code = tDiskColAddValImpl[pDiskCol->flag](pDiskCol, pColVal); + code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, pColVal); if (code) goto _exit; pColVal = tRowIterNext(&iter); } @@ -551,18 +552,18 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { if (code) return code; int32_t offset = 0; - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - if (pDiskCol->flag == HAS_NONE) continue; + if (pDiskColBuilder->flag == HAS_NONE) continue; - code = tDiskColToBinary(pDiskCol, NULL, NULL); + code = tGnrtDiskCol(pDiskColBuilder, NULL, NULL); if (code) return code; - SBlockCol bCol = {.cid = pDiskCol->cid, - .type = pDiskCol->type, + SBlockCol bCol = {.cid = pDiskColBuilder->cid, + .type = pDiskColBuilder->type, // .smaOn = , - .flag = pDiskCol->flag, + .flag = pDiskColBuilder->flag, // .szOrigin = // .szBitmap = // .szOffset = @@ -589,12 +590,12 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { n += hdr.szVer; memcpy(pBuilder->aBuf[0] + n, pKey, hdr.szKey); n += hdr.szKey; - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); n += tPutBlockCol(pBuilder->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); } - for (int32_t iDiskCol = 0; iDiskCol < pBuilder->nDiskCol; iDiskCol++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskCol, iDiskCol); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); // memcpy(pDiskData->aBuf[0] + n, NULL, ); // n += 0; } From f8619dfbe8859dec9e5dae09abb60f5166ad92dc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 15:28:04 +0800 Subject: [PATCH 12/64] refact code for further refact --- include/common/tdataformat.h | 12 +-- source/common/src/tdataformat.c | 96 ++---------------- source/common/src/trow.c | 2 +- source/dnode/vnode/src/tsdb/tsdbCache.c | 6 +- source/dnode/vnode/src/tsdb/tsdbUtil.c | 127 +++++++++++++----------- 5 files changed, 80 insertions(+), 163 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 5b49560175..8e0fc2f680 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -154,17 +154,7 @@ struct STSRowBuilder { struct SValue { union { - int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT - uint8_t u8; // TSDB_DATA_TYPE_UTINYINT - int16_t i16; // TSDB_DATA_TYPE_SMALLINT - uint16_t u16; // TSDB_DATA_TYPE_USMALLINT - int32_t i32; // TSDB_DATA_TYPE_INT - uint32_t u32; // TSDB_DATA_TYPE_UINT - int64_t i64; // TSDB_DATA_TYPE_BIGINT - uint64_t u64; // TSDB_DATA_TYPE_UBIGINT - TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP - float f; // TSDB_DATA_TYPE_FLOAT - double d; // TSDB_DATA_TYPE_DOUBLE + int64_t val; struct { uint32_t nData; uint8_t *pData; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 15a369fe40..e37d587529 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -33,105 +33,21 @@ typedef struct { // SValue int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) { - int32_t n = 0; - if (IS_VAR_DATA_TYPE(type)) { - n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData); + return tPutBinary(p, pValue->pData, pValue->nData); } else { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tPutI8(p ? p + n : p, pValue->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tPutI16(p ? p + n : p, pValue->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tPutI32(p ? p + n : p, pValue->i32); - break; - case TSDB_DATA_TYPE_BIGINT: - n += tPutI64(p ? p + n : p, pValue->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tPutFloat(p ? p + n : p, pValue->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tPutDouble(p ? p + n : p, pValue->d); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - n += tPutI64(p ? p + n : p, pValue->ts); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tPutU8(p ? p + n : p, pValue->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tPutU16(p ? p + n : p, pValue->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tPutU32(p ? p + n : p, pValue->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tPutU64(p ? p + n : p, pValue->u64); - break; - default: - ASSERT(0); - } + if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes); + return tDataTypes[type].bytes; } - - return n; } int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) { - int32_t n = 0; - if (IS_VAR_DATA_TYPE(type)) { - n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); + return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL); } else { - switch (type) { - case TSDB_DATA_TYPE_BOOL: - n += tGetI8(p, &pValue->i8); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p, &pValue->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, &pValue->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tGetI32(p, &pValue->i32); - break; - case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, &pValue->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, &pValue->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, &pValue->d); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - n += tGetI64(p, &pValue->ts); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, &pValue->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, &pValue->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, &pValue->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, &pValue->u64); - break; - default: - ASSERT(0); - } + memcpy(&pValue->val, p, tDataTypes[type].bytes); + return tDataTypes[type].bytes; } - - return n; } int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 9880fe362e..f2e060ae86 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -689,7 +689,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) { memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData); val = varBuf; } else { - val = (const void *)&pColVal->value.i64; + val = (const void *)&pColVal->value.val; } } else { pColVal = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8452d14c86..185d836e26 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -260,7 +260,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol); if (keyTs > tTsVal->ts) { STColumn *pTColumn = &pTSchema->columns[0]; - SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs}); + SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs}); taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal}); } @@ -1048,7 +1048,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo lastRowTs = TSDBROW_TS(pRow); STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); if (taosArrayPush(pColArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1147,7 +1147,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) { lastRowTs = rowTs; STColumn *pTColumn = &pTSchema->columns[0]; - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs}); if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 4e6e1f2f7f..c77af951af 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -649,7 +649,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -690,7 +690,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { STColumn *pTColumn; int32_t iCol, jCol = 1; - ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { pTColumn = &pMerger->pTSchema->columns[iCol]; @@ -744,7 +744,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP); - *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts}); + *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts}); if (taosArrayPush(pMerger->pArray, pColVal) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -770,7 +770,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; - ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts); + ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal); @@ -1513,73 +1513,79 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { case TSDB_DATA_TYPE_BOOL: break; case TSDB_DATA_TYPE_TINYINT: { - pColAgg->sum += colVal.value.i8; - if (!minAssigned || pColAgg->min > colVal.value.i8) { - pColAgg->min = colVal.value.i8; + int8_t i8 = *(int8_t *)&colVal.value.val; + pColAgg->sum += i8; + if (!minAssigned || pColAgg->min > i8) { + pColAgg->min = i8; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i8) { - pColAgg->max = colVal.value.i8; + if (!maxAssigned || pColAgg->max < i8) { + pColAgg->max = i8; maxAssigned = true; } break; } case TSDB_DATA_TYPE_SMALLINT: { - pColAgg->sum += colVal.value.i16; - if (!minAssigned || pColAgg->min > colVal.value.i16) { - pColAgg->min = colVal.value.i16; + int16_t i16 = *(int16_t *)&colVal.value.val; + pColAgg->sum += i16; + if (!minAssigned || pColAgg->min > i16) { + pColAgg->min = i16; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i16) { - pColAgg->max = colVal.value.i16; + if (!maxAssigned || pColAgg->max < i16) { + pColAgg->max = i16; maxAssigned = true; } break; } case TSDB_DATA_TYPE_INT: { - pColAgg->sum += colVal.value.i32; - if (!minAssigned || pColAgg->min > colVal.value.i32) { - pColAgg->min = colVal.value.i32; + int32_t i32 = *(int32_t *)&colVal.value.val; + pColAgg->sum += i32; + if (!minAssigned || pColAgg->min > i32) { + pColAgg->min = i32; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i32) { - pColAgg->max = colVal.value.i32; + if (!maxAssigned || pColAgg->max < i32) { + pColAgg->max = i32; maxAssigned = true; } break; } case TSDB_DATA_TYPE_BIGINT: { - pColAgg->sum += colVal.value.i64; - if (!minAssigned || pColAgg->min > colVal.value.i64) { - pColAgg->min = colVal.value.i64; + int64_t i64 = *(int64_t *)&colVal.value.val; + pColAgg->sum += i64; + if (!minAssigned || pColAgg->min > i64) { + pColAgg->min = i64; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i64) { - pColAgg->max = colVal.value.i64; + if (!maxAssigned || pColAgg->max < i64) { + pColAgg->max = i64; maxAssigned = true; } break; } case TSDB_DATA_TYPE_FLOAT: { - *(double *)(&pColAgg->sum) += colVal.value.f; - if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) { - *(double *)(&pColAgg->min) = colVal.value.f; + float f = *(float *)&colVal.value.val; + *(double *)(&pColAgg->sum) += f; + if (!minAssigned || *(double *)(&pColAgg->min) > f) { + *(double *)(&pColAgg->min) = f; minAssigned = true; } - if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) { - *(double *)(&pColAgg->max) = colVal.value.f; + if (!maxAssigned || *(double *)(&pColAgg->max) < f) { + *(double *)(&pColAgg->max) = f; maxAssigned = true; } break; } case TSDB_DATA_TYPE_DOUBLE: { - *(double *)(&pColAgg->sum) += colVal.value.d; - if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) { - *(double *)(&pColAgg->min) = colVal.value.d; + double d = *(double *)&colVal.value.val; + *(double *)(&pColAgg->sum) += d; + if (!minAssigned || *(double *)(&pColAgg->min) > d) { + *(double *)(&pColAgg->min) = d; minAssigned = true; } - if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) { - *(double *)(&pColAgg->max) = colVal.value.d; + if (!maxAssigned || *(double *)(&pColAgg->max) < d) { + *(double *)(&pColAgg->max) = d; maxAssigned = true; } break; @@ -1587,12 +1593,13 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { case TSDB_DATA_TYPE_VARCHAR: break; case TSDB_DATA_TYPE_TIMESTAMP: { - if (!minAssigned || pColAgg->min > colVal.value.i64) { - pColAgg->min = colVal.value.i64; + int64_t ts = *(int64_t *)&colVal.value.val; + if (!minAssigned || pColAgg->min > ts) { + pColAgg->min = ts; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.i64) { - pColAgg->max = colVal.value.i64; + if (!maxAssigned || pColAgg->max < ts) { + pColAgg->max = ts; maxAssigned = true; } break; @@ -1600,49 +1607,53 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { case TSDB_DATA_TYPE_NCHAR: break; case TSDB_DATA_TYPE_UTINYINT: { - pColAgg->sum += colVal.value.u8; - if (!minAssigned || pColAgg->min > colVal.value.u8) { - pColAgg->min = colVal.value.u8; + uint8_t u8 = *(uint8_t *)&colVal.value.val; + pColAgg->sum += u8; + if (!minAssigned || pColAgg->min > u8) { + pColAgg->min = u8; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.u8) { - pColAgg->max = colVal.value.u8; + if (!maxAssigned || pColAgg->max < u8) { + pColAgg->max = u8; maxAssigned = true; } break; } case TSDB_DATA_TYPE_USMALLINT: { - pColAgg->sum += colVal.value.u16; - if (!minAssigned || pColAgg->min > colVal.value.u16) { - pColAgg->min = colVal.value.u16; + uint16_t u16 = *(uint16_t *)&colVal.value.val; + pColAgg->sum += u16; + if (!minAssigned || pColAgg->min > u16) { + pColAgg->min = u16; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.u16) { - pColAgg->max = colVal.value.u16; + if (!maxAssigned || pColAgg->max < u16) { + pColAgg->max = u16; maxAssigned = true; } break; } case TSDB_DATA_TYPE_UINT: { - pColAgg->sum += colVal.value.u32; - if (!minAssigned || pColAgg->min > colVal.value.u32) { - pColAgg->min = colVal.value.u32; + uint32_t u32 = *(uint32_t *)&colVal.value.val; + pColAgg->sum += u32; + if (!minAssigned || pColAgg->min > u32) { + pColAgg->min = u32; minAssigned = true; } - if (!minAssigned || pColAgg->max < colVal.value.u32) { - pColAgg->max = colVal.value.u32; + if (!minAssigned || pColAgg->max < u32) { + pColAgg->max = u32; maxAssigned = true; } break; } case TSDB_DATA_TYPE_UBIGINT: { - pColAgg->sum += colVal.value.u64; - if (!minAssigned || pColAgg->min > colVal.value.u64) { - pColAgg->min = colVal.value.u64; + uint64_t u64 = *(uint64_t *)&colVal.value.val; + pColAgg->sum += u64; + if (!minAssigned || pColAgg->min > u64) { + pColAgg->min = u64; minAssigned = true; } - if (!maxAssigned || pColAgg->max < colVal.value.u64) { - pColAgg->max = colVal.value.u64; + if (!maxAssigned || pColAgg->max < u64) { + pColAgg->max = u64; maxAssigned = true; } break; From f870f7e4538b29166d318de806df720477d23768 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 16:08:51 +0800 Subject: [PATCH 13/64] fix gtest compile error --- source/common/test/dataformatTest.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp index 65f21bee40..5121724a08 100644 --- a/source/common/test/dataformatTest.cpp +++ b/source/common/test/dataformatTest.cpp @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#if 0 #include #include @@ -476,4 +477,5 @@ TEST(testCase, NoneTest) { taosArrayDestroy(pArray); taosMemoryFree(pTSchema); } +#endif #endif \ No newline at end of file From 0929326c1dbd9b0f458d75c9224d1803e8bcbbc6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 17:13:45 +0800 Subject: [PATCH 14/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 213 ++++++++++----------- 1 file changed, 106 insertions(+), 107 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index a514bddabb..25c1ea20d6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -17,6 +17,7 @@ typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SDiskColBuilder SDiskColBuilder; +typedef struct SDiskCol SDiskCol; typedef struct SDiskData SDiskData; struct SDiskColBuilder { @@ -31,6 +32,34 @@ struct SDiskColBuilder { SCompressor *pValC; }; +struct SDiskDataBuilder { + int64_t suid; + int64_t uid; + int32_t nRow; + uint8_t cmprAlg; + SCompressor *pUidC; + SCompressor *pVerC; + SCompressor *pKeyC; + int32_t nBuilder; + SArray *aBuilder; + uint8_t *aBuf[2]; +}; + +struct SDiskCol { + SBlockCol bCol; + const uint8_t *pBit; + const uint8_t *pOff; + const uint8_t *pVal; +}; + +struct SDiskData { + SDiskDataHdr hdr; + const uint8_t *pUid; + const uint8_t *pVer; + const uint8_t *pKey; + SArray *aDiskCol; // SArray +}; + // SDiskColBuilder ================================================ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; @@ -71,28 +100,40 @@ static int32_t tDiskColClear(SDiskColBuilder *pBuilder) { return code; } -static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SBlockCol *pBlockCol, const uint8_t **ppData) { +static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { int32_t code = 0; ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); - if (pBuilder->flag == HAS_NULL) { - return code; - } + pDiskCol->bCol = (SBlockCol){.cid = pBuilder->cid, + .type = pBuilder->type, + .smaOn = 1, /* todo */ + .flag = pBuilder->flag, + .szOrigin = 0, // todo + .szBitmap = 0, // todo + .szOffset = 0, + .szValue = 0, // todo + .offset = 0}; + pDiskCol->pBit = NULL; + pDiskCol->pOff = NULL; + pDiskCol->pVal = NULL; - // bitmap (todo) + if (pBuilder->flag == HAS_NULL) return code; + + // BITMAP if (pBuilder->flag != HAS_VALUE) { + // TODO } - // offset (todo) + // OFFSET if (IS_VAR_DATA_TYPE(pBuilder->type)) { - code = tCompGen(pBuilder->pOffC, NULL /* todo */, NULL /* todo */); + code = tCompGen(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); if (code) return code; } - // value (todo) + // VALUE if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { - code = tCompGen(pBuilder->pValC, NULL /* todo */, NULL /* todo */); + code = tCompGen(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); if (code) return code; } @@ -368,19 +409,6 @@ static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColV }; // SDiskDataBuilder ================================================ -struct SDiskDataBuilder { - int64_t suid; - int64_t uid; - int32_t nRow; - uint8_t cmprAlg; - SCompressor *pUidC; - SCompressor *pVerC; - SCompressor *pKeyC; - int32_t nDiskCol; - SArray *aDiskColBuilder; - uint8_t *aBuf[2]; -}; - int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { int32_t code = 0; @@ -410,32 +438,32 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; - if (pBuilder->aDiskColBuilder == NULL) { - pBuilder->aDiskColBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); - if (pBuilder->aDiskColBuilder == NULL) { + if (pBuilder->aBuilder == NULL) { + pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); + if (pBuilder->aBuilder == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - pBuilder->nDiskCol = 0; + pBuilder->nBuilder = 0; for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { STColumn *pTColumn = &pTSchema->columns[iCol]; - if (pBuilder->nDiskCol >= taosArrayGetSize(pBuilder->aDiskColBuilder)) { + if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { SDiskColBuilder dc = (SDiskColBuilder){0}; - if (taosArrayPush(pBuilder->aDiskColBuilder, &dc) == NULL) { + if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; } } - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, pBuilder->nDiskCol); + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); code = tDiskColInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; - pBuilder->nDiskCol++; + pBuilder->nBuilder++; } return code; @@ -448,13 +476,12 @@ int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - if (pBuilder->aDiskColBuilder) { - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aDiskColBuilder); - iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + if (pBuilder->aBuilder) { + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aBuilder); iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); tDiskColClear(pDiskColBuilder); } - taosArrayDestroy(pBuilder->aDiskColBuilder); + taosArrayDestroy(pBuilder->aBuilder); } for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { tFree(pBuilder->aBuf[iBuf]); @@ -495,8 +522,8 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nBuilder; iDiskColBuilder++) { + SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); while (pColVal && pColVal->cid < pDiskColBuilder->cid) { pColVal = tRowIterNext(&iter); @@ -523,102 +550,74 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { ASSERT(pBuilder->nRow); - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, - .fmtVer = 0, - .suid = pBuilder->suid, - .uid = pBuilder->uid, - .szUid = 0, - .szVer = 0, - .szKey = 0, - .szBlkCol = 0, - .nRow = pBuilder->nRow, - .cmprAlg = pBuilder->cmprAlg}; + // reset SDiskData + pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pBuilder->suid, + .uid = pBuilder->uid, + .szUid = 0, + .szVer = 0, + .szKey = 0, + .szBlkCol = 0, + .nRow = pBuilder->nRow, + .cmprAlg = pBuilder->cmprAlg}; + pDiskData->pUid = NULL; + pDiskData->pVer = NULL; + pDiskData->pKey = NULL; + if (pDiskData->aDiskCol) { + taosArrayClear(pDiskData->aDiskCol); + } else { + pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol)); + if (pDiskData->aDiskCol == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + } // UID - const uint8_t *pUid = NULL; if (pBuilder->uid == 0) { - code = tCompGen(pBuilder->pUidC, &pUid, &hdr.szUid); + code = tCompGen(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); if (code) return code; } // VERSION - const uint8_t *pVer = NULL; - code = tCompGen(pBuilder->pVerC, &pVer, &hdr.szVer); + code = tCompGen(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); if (code) return code; // TSKEY - const uint8_t *pKey = NULL; - code = tCompGen(pBuilder->pKeyC, &pKey, &hdr.szKey); + code = tCompGen(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); if (code) return code; int32_t offset = 0; - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); + for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - if (pDiskColBuilder->flag == HAS_NONE) continue; + if (pDCBuilder->flag == HAS_NONE) continue; - code = tGnrtDiskCol(pDiskColBuilder, NULL, NULL); + SDiskCol dCol; + + code = tGnrtDiskCol(pDCBuilder, &dCol); if (code) return code; - SBlockCol bCol = {.cid = pDiskColBuilder->cid, - .type = pDiskColBuilder->type, - // .smaOn = , - .flag = pDiskColBuilder->flag, - // .szOrigin = - // .szBitmap = - // .szOffset = - // .szValue = - .offset = offset}; + dCol.bCol.offset = offset; + offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue; - hdr.szBlkCol += tPutBlockCol(NULL, &bCol); - offset = offset + bCol.szBitmap + bCol.szOffset + bCol.szValue; - } + if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } -#if 0 - *nData = tPutDiskDataHdr(NULL, &hdr) + hdr.szUid + hdr.szVer + hdr.szKey + hdr.szBlkCol + offset; - code = tRealloc(&pBuilder->aBuf[0], *nData); - if (code) return code; - *ppData = pBuilder->aBuf[0]; - - int32_t n = 0; - n += tPutDiskDataHdr(pBuilder->aBuf[0] + n, &hdr); - if (hdr.szUid) { - memcpy(pBuilder->aBuf[0] + n, pUid, hdr.szUid); - n += hdr.szUid; + if (pDCBuilder->flag != HAS_NULL) { + pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); + } } - memcpy(pBuilder->aBuf[0] + n, pVer, hdr.szVer); - n += hdr.szVer; - memcpy(pBuilder->aBuf[0] + n, pKey, hdr.szKey); - n += hdr.szKey; - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - n += tPutBlockCol(pBuilder->aBuf[0] + n, NULL /*pDiskCol->bCol (todo) */); - } - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nDiskCol; iDiskColBuilder++) { - SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pBuilder->aDiskColBuilder, iDiskColBuilder); - // memcpy(pDiskData->aBuf[0] + n, NULL, ); - // n += 0; - } -#endif return code; } // SDiskData ================================================ -struct SDiskData { - SDiskDataHdr hdr; - const uint8_t *pUid; - const uint8_t *pVer; - const uint8_t *pKey; - SArray *aBlockCol; - SArray *aColData; -}; - int32_t tDiskDataDestroy(SDiskData *pDiskData) { int32_t code = 0; - - if (pDiskData->aBlockCol) taosArrayDestroy(pDiskData->aBlockCol); - if (pDiskData->aColData) taosArrayDestroy(pDiskData->aColData); - + if (pDiskData->aDiskCol) taosArrayDestroy(pDiskData->aDiskCol); return code; } From a8b919918f79097b7aeb3a7a9990e7e3485a8385 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 21 Sep 2022 17:58:44 +0800 Subject: [PATCH 15/64] more code --- source/dnode/vnode/src/inc/tsdb.h | 23 +++++ source/dnode/vnode/src/tsdb/tsdbDiskData.c | 25 ++---- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 87 +++++++++++++++++++ 3 files changed, 116 insertions(+), 19 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index c2a31679d0..e67fa2ee2d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -32,6 +32,12 @@ extern "C" { #define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TSD ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on +#define TSDB_CHECK_CODE(CODE, LINO, LABEL) \ + if (CODE) { \ + LINO = __LINE__; \ + goto LABEL; \ + } + typedef struct TSDBROW TSDBROW; typedef struct TABLEID TABLEID; typedef struct TSDBKEY TSDBKEY; @@ -65,6 +71,8 @@ typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; typedef struct SVersionRange SVersionRange; typedef struct SLDataIter SLDataIter; +typedef struct SDiskCol SDiskCol; +typedef struct SDiskData SDiskData; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -652,6 +660,21 @@ typedef struct { STSchema *pTSchema; } SSkmInfo; +struct SDiskCol { + SBlockCol bCol; + const uint8_t *pBit; + const uint8_t *pOff; + const uint8_t *pVal; +}; + +struct SDiskData { + SDiskDataHdr hdr; + const uint8_t *pUid; + const uint8_t *pVer; + const uint8_t *pKey; + SArray *aDiskCol; // SArray +}; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 25c1ea20d6..9b5934a04f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -17,8 +17,6 @@ typedef struct SDiskDataBuilder SDiskDataBuilder; typedef struct SDiskColBuilder SDiskColBuilder; -typedef struct SDiskCol SDiskCol; -typedef struct SDiskData SDiskData; struct SDiskColBuilder { int16_t cid; @@ -45,21 +43,6 @@ struct SDiskDataBuilder { uint8_t *aBuf[2]; }; -struct SDiskCol { - SBlockCol bCol; - const uint8_t *pBit; - const uint8_t *pOff; - const uint8_t *pVal; -}; - -struct SDiskData { - SDiskDataHdr hdr; - const uint8_t *pUid; - const uint8_t *pVer; - const uint8_t *pKey; - SArray *aDiskCol; // SArray -}; - // SDiskColBuilder ================================================ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; @@ -147,9 +130,13 @@ static int32_t tDiskColAddValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); if (code) goto _exit; pBuilder->offset += pColVal->value.nData; + + code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData); + if (code) goto _exit; + } else { + code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes); + if (code) goto _exit; } - code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData /*TODO*/); - if (code) goto _exit; _exit: return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 5fe0b408b1..8674ae571e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -607,6 +607,93 @@ _err: return code; } +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInfo *pBlkInfo) { + int32_t code = 0; + int32_t lino = 0; + + STsdbFD *pFD = pWriter->pDataFD; // todo + int64_t offset = pWriter->fData.size; + + // hdr + int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); + code = tRealloc(&pWriter->aBuf[0], n); + TSDB_CHECK_CODE(code, lino, _exit); + + tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); + + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], n); + TSDB_CHECK_CODE(code, lino, _exit); + offset += n; + + // uid + ver + key + if (pDiskData->hdr.szUid) { + code = tsdbWriteFile(pFD, offset, pDiskData->pUid, pDiskData->hdr.szUid); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskData->hdr.szUid; + } + + code = tsdbWriteFile(pFD, offset, pDiskData->pVer, pDiskData->hdr.szVer); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskData->hdr.szVer; + + code = tsdbWriteFile(pFD, offset, pDiskData->pKey, pDiskData->hdr.szKey); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskData->hdr.szKey; + + // SBlockCol + if (pDiskData->hdr.szBlkCol) { + code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + n = 0; + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + n += tPutBlockCol(pWriter->aBuf[0] + n, &pDiskCol->bCol); + } + + ASSERT(n == pDiskData->hdr.szBlkCol); + + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + offset += pDiskData->hdr.szBlkCol; + } + + // pData + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (pDiskCol->bCol.flag == HAS_NULL) continue; + + if (pDiskCol->bCol.szBitmap) { + code = tsdbWriteFile(pFD, offset, pDiskCol->pBit, pDiskCol->bCol.szBitmap); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskCol->bCol.szBitmap; + } + + if (pDiskCol->bCol.szOffset) { + code = tsdbWriteFile(pFD, offset, pDiskCol->pOff, pDiskCol->bCol.szOffset); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskCol->bCol.szOffset; + } + + if (pDiskCol->bCol.szValue) { + code = tsdbWriteFile(pFD, offset, pDiskCol->pVal, pDiskCol->bCol.szValue); + TSDB_CHECK_CODE(code, lino, _exit); + offset += pDiskCol->bCol.szValue; + } + } + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d %s", TD_VID(pWriter->pTsdb->pVnode), __func__); + } + return code; +} + int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { int32_t code = 0; int64_t n; From 9f7771cde4a892407a3d8802488addaea79a9f19 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 11:52:47 +0800 Subject: [PATCH 16/64] refact more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbDiskData.c | 97 ++++++++++--------- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/util/src/tcompression.c | 2 +- 5 files changed, 56 insertions(+), 48 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 8d7c607e6b..f22205d144 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -130,7 +130,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index e67fa2ee2d..2121d779ab 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -665,6 +665,7 @@ struct SDiskCol { const uint8_t *pBit; const uint8_t *pOff; const uint8_t *pVal; + SColumnDataAgg agg; }; struct SDiskData { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9b5934a04f..9e2f8eec7c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -35,16 +35,17 @@ struct SDiskDataBuilder { int64_t uid; int32_t nRow; uint8_t cmprAlg; + uint8_t calcSma; SCompressor *pUidC; SCompressor *pVerC; SCompressor *pKeyC; int32_t nBuilder; - SArray *aBuilder; + SArray *aBuilder; // SArray uint8_t *aBuf[2]; }; // SDiskColBuilder ================================================ -static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { +static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; pBuilder->cid = cid; @@ -59,7 +60,7 @@ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, code = tCompressorCreate(&pBuilder->pOffC); if (code) return code; } - code = tCompressorReset(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + code = tCompressorInit(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } @@ -67,13 +68,13 @@ static int32_t tDiskColInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, code = tCompressorCreate(&pBuilder->pValC); if (code) return code; } - code = tCompressorReset(pBuilder->pValC, type, cmprAlg); + code = tCompressorInit(pBuilder->pValC, type, cmprAlg); if (code) return code; return code; } -static int32_t tDiskColClear(SDiskColBuilder *pBuilder) { +static int32_t tDiskColBuilderClear(SDiskColBuilder *pBuilder) { int32_t code = 0; tFree(pBuilder->pBitMap); @@ -396,33 +397,60 @@ static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColV }; // SDiskDataBuilder ================================================ -int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg) { +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { + int32_t code = 0; + + *ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder)); + if (*ppBuilder == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + return code; + } + + return code; +} + +void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { + if (pBuilder == NULL) return NULL; + + if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); + if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); + if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); + + if (pBuilder->aBuilder) { + for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); + tDiskColBuilderClear(pDCBuilder); + } + taosArrayDestroy(pBuilder->aBuilder); + } + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); + } + taosMemoryFree(pBuilder); + + return NULL; +} + +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, + uint8_t calcSma) { int32_t code = 0; pBuilder->suid = pId->suid; pBuilder->uid = pId->uid; pBuilder->nRow = 0; pBuilder->cmprAlg = cmprAlg; + pBuilder->calcSma = calcSma; - if (pBuilder->pUidC == NULL) { - code = tCompressorCreate(&pBuilder->pUidC); - if (code) return code; - } - code = tCompressorReset(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; + code = tCompressorInit(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pBuilder->pVerC == NULL) { - code = tCompressorCreate(&pBuilder->pVerC); - if (code) return code; - } - code = tCompressorReset(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code; + code = tCompressorInit(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; - if (pBuilder->pKeyC == NULL) { - code = tCompressorCreate(&pBuilder->pKeyC); - if (code) return code; - } - code = tCompressorReset(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code; + code = tCompressorInit(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; if (pBuilder->aBuilder == NULL) { @@ -438,7 +466,7 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB STColumn *pTColumn = &pTSchema->columns[iCol]; if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { - SDiskColBuilder dc = (SDiskColBuilder){0}; + SDiskColBuilder dc = {0}; if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; @@ -447,7 +475,7 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); - code = tDiskColInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); + code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); if (code) return code; pBuilder->nBuilder++; @@ -456,27 +484,6 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB return code; } -int32_t tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { - int32_t code = 0; - - if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); - if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); - if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - - if (pBuilder->aBuilder) { - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < taosArrayGetSize(pBuilder->aBuilder); iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); - tDiskColClear(pDiskColBuilder); - } - taosArrayDestroy(pBuilder->aBuilder); - } - for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { - tFree(pBuilder->aBuf[iBuf]); - } - - return code; -} - int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { int32_t code = 0; @@ -605,6 +612,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { // SDiskData ================================================ int32_t tDiskDataDestroy(SDiskData *pDiskData) { int32_t code = 0; - if (pDiskData->aDiskCol) taosArrayDestroy(pDiskData->aDiskCol); + pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 8674ae571e..679f7c3457 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -128,7 +128,7 @@ _exit: return code; } -static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { +static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index cd9ff5e325..619ae8360d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1518,7 +1518,7 @@ int32_t tCompressorDestroy(SCompressor *pCmprsor) { return code; } -int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; pCmprsor->type = type; From a3cedbf1bc60a864e436ae294c345ba54f6b86f1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 13:49:34 +0800 Subject: [PATCH 17/64] more code --- source/dnode/vnode/src/inc/tsdb.h | 82 ++++++++++++---------- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 62 ++++++++-------- 2 files changed, 77 insertions(+), 67 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 2121d779ab..dc218e86e0 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -38,41 +38,42 @@ extern "C" { goto LABEL; \ } -typedef struct TSDBROW TSDBROW; -typedef struct TABLEID TABLEID; -typedef struct TSDBKEY TSDBKEY; -typedef struct SDelData SDelData; -typedef struct SDelIdx SDelIdx; -typedef struct STbData STbData; -typedef struct SMemTable SMemTable; -typedef struct STbDataIter STbDataIter; -typedef struct SMapData SMapData; -typedef struct SBlockIdx SBlockIdx; -typedef struct SDataBlk SDataBlk; -typedef struct SSttBlk SSttBlk; -typedef struct SDiskDataHdr SDiskDataHdr; -typedef struct SBlockData SBlockData; -typedef struct SDelFile SDelFile; -typedef struct SHeadFile SHeadFile; -typedef struct SDataFile SDataFile; -typedef struct SSttFile SSttFile; -typedef struct SSmaFile SSmaFile; -typedef struct SDFileSet SDFileSet; -typedef struct SDataFWriter SDataFWriter; -typedef struct SDataFReader SDataFReader; -typedef struct SDelFWriter SDelFWriter; -typedef struct SDelFReader SDelFReader; -typedef struct SRowIter SRowIter; -typedef struct STsdbFS STsdbFS; -typedef struct SRowMerger SRowMerger; -typedef struct STsdbReadSnap STsdbReadSnap; -typedef struct SBlockInfo SBlockInfo; -typedef struct SSmaInfo SSmaInfo; -typedef struct SBlockCol SBlockCol; -typedef struct SVersionRange SVersionRange; -typedef struct SLDataIter SLDataIter; -typedef struct SDiskCol SDiskCol; -typedef struct SDiskData SDiskData; +typedef struct TSDBROW TSDBROW; +typedef struct TABLEID TABLEID; +typedef struct TSDBKEY TSDBKEY; +typedef struct SDelData SDelData; +typedef struct SDelIdx SDelIdx; +typedef struct STbData STbData; +typedef struct SMemTable SMemTable; +typedef struct STbDataIter STbDataIter; +typedef struct SMapData SMapData; +typedef struct SBlockIdx SBlockIdx; +typedef struct SDataBlk SDataBlk; +typedef struct SSttBlk SSttBlk; +typedef struct SDiskDataHdr SDiskDataHdr; +typedef struct SBlockData SBlockData; +typedef struct SDelFile SDelFile; +typedef struct SHeadFile SHeadFile; +typedef struct SDataFile SDataFile; +typedef struct SSttFile SSttFile; +typedef struct SSmaFile SSmaFile; +typedef struct SDFileSet SDFileSet; +typedef struct SDataFWriter SDataFWriter; +typedef struct SDataFReader SDataFReader; +typedef struct SDelFWriter SDelFWriter; +typedef struct SDelFReader SDelFReader; +typedef struct SRowIter SRowIter; +typedef struct STsdbFS STsdbFS; +typedef struct SRowMerger SRowMerger; +typedef struct STsdbReadSnap STsdbReadSnap; +typedef struct SBlockInfo SBlockInfo; +typedef struct SSmaInfo SSmaInfo; +typedef struct SBlockCol SBlockCol; +typedef struct SVersionRange SVersionRange; +typedef struct SLDataIter SLDataIter; +typedef struct SDiskCol SDiskCol; +typedef struct SDiskData SDiskData; +typedef struct SDiskDataBuilder SDiskDataBuilder; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -302,7 +303,7 @@ int32_t tsdbMerge(STsdb *pTsdb); #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) #define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0) -// tsdbCache +// tsdbCache ============================================================================================== int32_t tsdbOpenCache(STsdb *pTsdb); void tsdbCloseCache(STsdb *pTsdb); int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb); @@ -320,6 +321,15 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode); int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema); +// tsdbDiskData ============================================================================================== +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); +void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder); +int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, + uint8_t calcSma); +int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData); +int32_t tDiskDataDestroy(SDiskData *pDiskData); + // structs ======================= struct STsdbFS { SDelFile *pDelFile; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9e2f8eec7c..b93b6a7007 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -15,8 +15,7 @@ #include "tsdb.h" -typedef struct SDiskDataBuilder SDiskDataBuilder; -typedef struct SDiskColBuilder SDiskColBuilder; +typedef struct SDiskColBuilder SDiskColBuilder; struct SDiskColBuilder { int16_t cid; @@ -45,6 +44,19 @@ struct SDiskDataBuilder { }; // SDiskColBuilder ================================================ +#define tDiskColBuilderCreate() \ + (SDiskColBuilder) { 0 } + +static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) { + int32_t code = 0; + + tFree(pBuilder->pBitMap); + if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); + if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); + + return code; +} + static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { int32_t code = 0; @@ -74,16 +86,6 @@ static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_ return code; } -static int32_t tDiskColBuilderClear(SDiskColBuilder *pBuilder) { - int32_t code = 0; - - tFree(pBuilder->pBitMap); - if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); - if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); - - return code; -} - static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { int32_t code = 0; @@ -419,7 +421,7 @@ void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { if (pBuilder->aBuilder) { for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - tDiskColBuilderClear(pDCBuilder); + tDiskColBuilderDestroy(pDCBuilder); } taosArrayDestroy(pBuilder->aBuilder); } @@ -466,7 +468,7 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB STColumn *pTColumn = &pTSchema->columns[iCol]; if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { - SDiskColBuilder dc = {0}; + SDiskColBuilder dc = tDiskColBuilderCreate(); if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; return code; @@ -491,51 +493,51 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { + ASSERT(!pBuilder->calcSma); for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; } pBuilder->uid = 0; } if (pBuilder->uid == 0) { code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; } // version int64_t version = TSDBROW_VERSION(pRow); code = tCompress(pBuilder->pVerC, &version, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; // TSKEY TSKEY ts = TSDBROW_TS(pRow); code = tCompress(pBuilder->pKeyC, &ts, sizeof(int64_t)); - if (code) goto _exit; + if (code) return code; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); SColVal *pColVal = tRowIterNext(&iter); - for (int32_t iDiskColBuilder = 0; iDiskColBuilder < pBuilder->nBuilder; iDiskColBuilder++) { - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iDiskColBuilder); + for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - while (pColVal && pColVal->cid < pDiskColBuilder->cid) { + while (pColVal && pColVal->cid < pDCBuilder->cid) { pColVal = tRowIterNext(&iter); } - if (pColVal == NULL || pColVal->cid > pDiskColBuilder->cid) { - SColVal cv = COL_VAL_NONE(pDiskColBuilder->cid, pDiskColBuilder->type); - code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, &cv); - if (code) goto _exit; + if (pColVal == NULL || pColVal->cid > pDCBuilder->cid) { + SColVal cv = COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type); + code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, &cv); + if (code) return code; } else { - code = tDiskColAddValImpl[pDiskColBuilder->flag](pDiskColBuilder, pColVal); - if (code) goto _exit; + code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, pColVal); + if (code) return code; pColVal = tRowIterNext(&iter); } } pBuilder->nRow++; -_exit: return code; } @@ -601,9 +603,7 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { return code; } - if (pDCBuilder->flag != HAS_NULL) { - pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); - } + pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); } return code; From 1bab3c1b4d73c173521a458e7aa7ff3ce5351e0b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 18:24:24 +0800 Subject: [PATCH 18/64] optimize more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 244 +++++++++++++++------ 1 file changed, 178 insertions(+), 66 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index b93b6a7007..5a235c5c82 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -18,15 +18,20 @@ typedef struct SDiskColBuilder SDiskColBuilder; struct SDiskColBuilder { - int16_t cid; - int8_t type; - int8_t flag; - uint8_t cmprAlg; - int32_t nVal; - uint8_t *pBitMap; - int32_t offset; - SCompressor *pOffC; - SCompressor *pValC; + int16_t cid; + int8_t type; + uint8_t cmprAlg; + uint8_t calcSma; + int8_t flag; + int32_t nVal; + uint8_t *pBitMap; + int32_t offset; + SCompressor *pOffC; + SCompressor *pValC; + SColumnDataAgg sma; + uint8_t minSet; + uint8_t maxSet; + uint8_t *aBuf[1]; }; struct SDiskDataBuilder { @@ -53,36 +58,41 @@ static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) { tFree(pBuilder->pBitMap); if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); + for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { + tFree(pBuilder->aBuf[iBuf]); + } return code; } -static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg) { +static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg, + uint8_t calcSma) { int32_t code = 0; pBuilder->cid = cid; pBuilder->type = type; - pBuilder->flag = 0; pBuilder->cmprAlg = cmprAlg; + pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma; + pBuilder->flag = 0; pBuilder->nVal = 0; pBuilder->offset = 0; if (IS_VAR_DATA_TYPE(type)) { - if (pBuilder->pOffC == NULL) { - code = tCompressorCreate(&pBuilder->pOffC); - if (code) return code; - } + if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code; code = tCompressorInit(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } - if (pBuilder->pValC == NULL) { - code = tCompressorCreate(&pBuilder->pValC); - if (code) return code; - } + if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code; code = tCompressorInit(pBuilder->pValC, type, cmprAlg); if (code) return code; + if (pBuilder->calcSma) { + pBuilder->sma = (SColumnDataAgg){.colId = cid}; + pBuilder->minSet = 0; + pBuilder->maxSet = 0; + } + return code; } @@ -91,24 +101,31 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); - pDiskCol->bCol = (SBlockCol){.cid = pBuilder->cid, - .type = pBuilder->type, - .smaOn = 1, /* todo */ - .flag = pBuilder->flag, - .szOrigin = 0, // todo - .szBitmap = 0, // todo - .szOffset = 0, - .szValue = 0, // todo - .offset = 0}; - pDiskCol->pBit = NULL; - pDiskCol->pOff = NULL; - pDiskCol->pVal = NULL; + *pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid, + .type = pBuilder->type, + .smaOn = pBuilder->calcSma, + .flag = pBuilder->flag, + .szOrigin = 0, // todo + .szBitmap = 0, + .szOffset = 0, + .szValue = 0, + .offset = 0}, + .pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma}; if (pBuilder->flag == HAS_NULL) return code; // BITMAP if (pBuilder->flag != HAS_VALUE) { - // TODO + int32_t nBit; + if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + nBit = BIT2_SIZE(pBuilder->nVal); + } else { + nBit = BIT1_SIZE(pBuilder->nVal); + } + + pDiskCol->bCol.szBitmap = tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], 0, pBuilder->cmprAlg, + NULL, 0); // todo: alloc + pDiskCol->pBit = pBuilder->aBuf[0]; } // OFFSET @@ -126,22 +143,114 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { return code; } -static int32_t tDiskColAddValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { +static FORCE_INLINE void tDiskColUpdateSmaBool(SDiskColBuilder *pBuilder, SColVal *pColVal) { + // TODO +} +static FORCE_INLINE void tDiskColUpdateSmaTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int8_t val = *(int8_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateSmaSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int16_t val = *(int16_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateSmaInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t val = *(int32_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int64_t val = *(int64_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateFloat(SDiskColBuilder *pBuilder, SColVal *pColVal) { + float val = *(float *)&pColVal->value.val; + *(double *)&pBuilder->sma.sum += val; + if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; + if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateDouble(SDiskColBuilder *pBuilder, SColVal *pColVal) { + double val = *(double *)&pColVal->value.val; + *(double *)&pBuilder->sma.sum += val; + if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; + if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint8_t val = *(uint8_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint16_t val = *(uint16_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint32_t val = *(uint32_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static FORCE_INLINE void tDiskColUpdateUBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { + uint64_t val = *(uint64_t *)&pColVal->value.val; + pBuilder->sma.sum += val; + if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; + if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; +} +static void (*tDiskColUpdateSmaImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { + NULL, // TSDB_DATA_TYPE_NULL + tDiskColUpdateSmaBool, // TSDB_DATA_TYPE_BOOL + tDiskColUpdateSmaTinyint, // TSDB_DATA_TYPE_TINYINT + tDiskColUpdateSmaSmallint, // TSDB_DATA_TYPE_SMALLINT + tDiskColUpdateSmaInt, // TSDB_DATA_TYPE_INT + tDiskColUpdateBigint, // TSDB_DATA_TYPE_BIGINT + tDiskColUpdateFloat, // TSDB_DATA_TYPE_FLOAT + tDiskColUpdateDouble, // TSDB_DATA_TYPE_DOUBLE + NULL, // TSDB_DATA_TYPE_VARCHAR + tDiskColUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP + NULL, // TSDB_DATA_TYPE_NCHAR + tDiskColUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT + tDiskColUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT + tDiskColUpdateUInt, // TSDB_DATA_TYPE_UINT + tDiskColUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT + NULL, // TSDB_DATA_TYPE_JSON + NULL, // TSDB_DATA_TYPE_VARBINARY + NULL, // TSDB_DATA_TYPE_DECIMAL + NULL, // TSDB_DATA_TYPE_BLOB + NULL, // TSDB_DATA_TYPE_MEDIUMBLOB +}; +static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { + if (pColVal->isNone || pColVal->isNull) { + pBuilder->sma.numOfNull++; + } else { + tDiskColUpdateSmaImpl[pBuilder->type](pBuilder, pColVal); + } +} + +static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColVal->type)) { code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); - if (code) goto _exit; + if (code) return code; pBuilder->offset += pColVal->value.nData; code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData); - if (code) goto _exit; + if (code) return code; } else { code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes); - if (code) goto _exit; + if (code) return code; } -_exit: return code; } static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 @@ -153,12 +262,10 @@ static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { / pBuilder->flag = HAS_NULL; } else { pBuilder->flag = HAS_VALUE; - code = tDiskColAddValue(pBuilder, pColVal); - if (code) goto _exit; + code = tDiskColPutValue(pBuilder, pColVal); + if (code) return code; } - pBuilder->nVal++; -_exit: return code; } static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE @@ -169,7 +276,7 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) goto _exit; + if (code) return code; memset(pBuilder->pBitMap, 0, nBit); SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); @@ -182,17 +289,15 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColAddValue(pBuilder, &cv); - if (code) goto _exit; + code = tDiskColPutValue(pBuilder, &cv); + if (code) return code; } - code = tDiskColAddValue(pBuilder, pColVal); - if (code) goto _exit; + code = tDiskColPutValue(pBuilder, pColVal); + if (code) return code; } } - pBuilder->nVal++; -_exit: return code; } static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL @@ -216,15 +321,14 @@ static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColAddValue(pBuilder, &cv); + code = tDiskColPutValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } } - pBuilder->nVal++; _exit: return code; @@ -259,14 +363,13 @@ static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColAddValue(pBuilder, &cv); + code = tDiskColPutValue(pBuilder, &cv); if (code) goto _exit; } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } - pBuilder->nVal++; _exit: return code; @@ -288,13 +391,12 @@ static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { / memset(pBuilder->pBitMap, 255, nBit); SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } else { - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; } - pBuilder->nVal++; _exit: return code; @@ -326,9 +428,8 @@ static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; - pBuilder->nVal++; _exit: return code; @@ -360,9 +461,8 @@ static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); } } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; - pBuilder->nVal++; _exit: return code; @@ -380,9 +480,8 @@ static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { / } else { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); } - code = tDiskColAddValue(pBuilder, pColVal); + code = tDiskColPutValue(pBuilder, pColVal); if (code) goto _exit; - pBuilder->nVal++; _exit: return code; @@ -397,6 +496,18 @@ static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColV tDiskColAddVal6, // HAS_VALUE|HAS_NULL tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE }; +static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + if (pBuilder->calcSma) tDiskColUpdateSma(pBuilder, pColVal); + + code = tDiskColAddValImpl[pBuilder->type](pBuilder, pColVal); + if (code) return code; + + pBuilder->nVal++; + + return code; +} // SDiskDataBuilder ================================================ int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { @@ -477,7 +588,8 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); - code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg); + code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg, + (calcSma && (pTColumn->flags & COL_SMA_ON))); if (code) return code; pBuilder->nBuilder++; @@ -528,10 +640,10 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche if (pColVal == NULL || pColVal->cid > pDCBuilder->cid) { SColVal cv = COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type); - code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, &cv); + code = tDiskColAddVal(pDCBuilder, &cv); if (code) return code; } else { - code = tDiskColAddValImpl[pDCBuilder->flag](pDCBuilder, pColVal); + code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; pColVal = tRowIterNext(&iter); } From 1cac65d8c0092261de98e5371282371eacf83051 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 22 Sep 2022 19:08:33 +0800 Subject: [PATCH 19/64] refact more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 88 +------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 270 ++++++++------------- 2 files changed, 98 insertions(+), 260 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 5a235c5c82..8d25873036 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -143,96 +143,12 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { return code; } -static FORCE_INLINE void tDiskColUpdateSmaBool(SDiskColBuilder *pBuilder, SColVal *pColVal) { - // TODO -} -static FORCE_INLINE void tDiskColUpdateSmaTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int8_t val = *(int8_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateSmaSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int16_t val = *(int16_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateSmaInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t val = *(int32_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int64_t val = *(int64_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateFloat(SDiskColBuilder *pBuilder, SColVal *pColVal) { - float val = *(float *)&pColVal->value.val; - *(double *)&pBuilder->sma.sum += val; - if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; - if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateDouble(SDiskColBuilder *pBuilder, SColVal *pColVal) { - double val = *(double *)&pColVal->value.val; - *(double *)&pBuilder->sma.sum += val; - if (!pBuilder->minSet || *(double *)&pBuilder->sma.min > val) *(double *)&pBuilder->sma.min = val; - if (!pBuilder->maxSet || *(double *)&pBuilder->sma.max < val) *(double *)&pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUTinyint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint8_t val = *(uint8_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUSmallint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint16_t val = *(uint16_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUInt(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint32_t val = *(uint32_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static FORCE_INLINE void tDiskColUpdateUBigint(SDiskColBuilder *pBuilder, SColVal *pColVal) { - uint64_t val = *(uint64_t *)&pColVal->value.val; - pBuilder->sma.sum += val; - if (!pBuilder->minSet || pBuilder->sma.min > val) pBuilder->sma.min = val; - if (!pBuilder->maxSet || pBuilder->sma.max < val) pBuilder->sma.max = val; -} -static void (*tDiskColUpdateSmaImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { - NULL, // TSDB_DATA_TYPE_NULL - tDiskColUpdateSmaBool, // TSDB_DATA_TYPE_BOOL - tDiskColUpdateSmaTinyint, // TSDB_DATA_TYPE_TINYINT - tDiskColUpdateSmaSmallint, // TSDB_DATA_TYPE_SMALLINT - tDiskColUpdateSmaInt, // TSDB_DATA_TYPE_INT - tDiskColUpdateBigint, // TSDB_DATA_TYPE_BIGINT - tDiskColUpdateFloat, // TSDB_DATA_TYPE_FLOAT - tDiskColUpdateDouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR - tDiskColUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR - tDiskColUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT - tDiskColUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT - tDiskColUpdateUInt, // TSDB_DATA_TYPE_UINT - tDiskColUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON - NULL, // TSDB_DATA_TYPE_VARBINARY - NULL, // TSDB_DATA_TYPE_DECIMAL - NULL, // TSDB_DATA_TYPE_BLOB - NULL, // TSDB_DATA_TYPE_MEDIUMBLOB -}; +extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { if (pColVal->isNone || pColVal->isNull) { pBuilder->sma.numOfNull++; } else { - tDiskColUpdateSmaImpl[pBuilder->type](pBuilder, pColVal); + tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); } } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c77af951af..0575b30526 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1492,185 +1492,107 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { return n; } +#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \ + do { \ + (SUM_V) += (VAL); \ + if (!(MINSET)) { \ + (MIN_V) = (VAL); \ + (MINSET) = 1; \ + } else if ((MIN_V) > (VAL)) { \ + (MIN_V) = (VAL); \ + } \ + if (!(MAXSET)) { \ + (MAX_V) = (VAL); \ + (MAXSET) = 1; \ + } else if ((MAX_V) < (VAL)) { \ + (MAX_V) = (VAL); \ + } \ + } while (0) + +static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + int8_t val = *(int8_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + int16_t val = *(int16_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int32_t val = *(int32_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + int64_t val = *(int64_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + float val = *(float *)&pColVal->value.val; + SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + double val = *(double *)&pColVal->value.val; + SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint8_t val = *(uint8_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint16_t val = *(uint16_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) { + uint32_t val = *(uint32_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, + uint8_t *maxSet) { + uint64_t val = *(uint64_t *)&pColVal->value.val; + SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet); +} +void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = { + NULL, + tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL + tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT + tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT + tSmaUpdateInt, // TSDB_DATA_TYPE_INT + tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT + tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT + tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE + NULL, // TSDB_DATA_TYPE_VARCHAR + tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP + NULL, // TSDB_DATA_TYPE_NCHAR + tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT + tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT + tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT + tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT + NULL, // TSDB_DATA_TYPE_JSON + NULL, // TSDB_DATA_TYPE_VARBINARY + NULL, // TSDB_DATA_TYPE_DECIMAL + NULL, // TSDB_DATA_TYPE_BLOB + NULL, // TSDB_DATA_TYPE_MEDIUMBLOB +}; void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { - SColVal colVal; - SColVal *pColVal = &colVal; - - memset(pColAgg, 0, sizeof(*pColAgg)); - bool minAssigned = false; - bool maxAssigned = false; - *pColAgg = (SColumnDataAgg){.colId = pColData->cid}; - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - tColDataGetValue(pColData, iVal, pColVal); + uint8_t minSet = 0; + uint8_t maxSet = 0; - if (pColVal->isNone || pColVal->isNull) { + SColVal cv; + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + tColDataGetValue(pColData, iVal, &cv); + + if (cv.isNone || cv.isNull) { pColAgg->numOfNull++; } else { - switch (pColData->type) { - case TSDB_DATA_TYPE_NULL: - break; - case TSDB_DATA_TYPE_BOOL: - break; - case TSDB_DATA_TYPE_TINYINT: { - int8_t i8 = *(int8_t *)&colVal.value.val; - pColAgg->sum += i8; - if (!minAssigned || pColAgg->min > i8) { - pColAgg->min = i8; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i8) { - pColAgg->max = i8; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_SMALLINT: { - int16_t i16 = *(int16_t *)&colVal.value.val; - pColAgg->sum += i16; - if (!minAssigned || pColAgg->min > i16) { - pColAgg->min = i16; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i16) { - pColAgg->max = i16; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_INT: { - int32_t i32 = *(int32_t *)&colVal.value.val; - pColAgg->sum += i32; - if (!minAssigned || pColAgg->min > i32) { - pColAgg->min = i32; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i32) { - pColAgg->max = i32; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_BIGINT: { - int64_t i64 = *(int64_t *)&colVal.value.val; - pColAgg->sum += i64; - if (!minAssigned || pColAgg->min > i64) { - pColAgg->min = i64; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < i64) { - pColAgg->max = i64; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_FLOAT: { - float f = *(float *)&colVal.value.val; - *(double *)(&pColAgg->sum) += f; - if (!minAssigned || *(double *)(&pColAgg->min) > f) { - *(double *)(&pColAgg->min) = f; - minAssigned = true; - } - if (!maxAssigned || *(double *)(&pColAgg->max) < f) { - *(double *)(&pColAgg->max) = f; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_DOUBLE: { - double d = *(double *)&colVal.value.val; - *(double *)(&pColAgg->sum) += d; - if (!minAssigned || *(double *)(&pColAgg->min) > d) { - *(double *)(&pColAgg->min) = d; - minAssigned = true; - } - if (!maxAssigned || *(double *)(&pColAgg->max) < d) { - *(double *)(&pColAgg->max) = d; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_VARCHAR: - break; - case TSDB_DATA_TYPE_TIMESTAMP: { - int64_t ts = *(int64_t *)&colVal.value.val; - if (!minAssigned || pColAgg->min > ts) { - pColAgg->min = ts; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < ts) { - pColAgg->max = ts; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_NCHAR: - break; - case TSDB_DATA_TYPE_UTINYINT: { - uint8_t u8 = *(uint8_t *)&colVal.value.val; - pColAgg->sum += u8; - if (!minAssigned || pColAgg->min > u8) { - pColAgg->min = u8; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < u8) { - pColAgg->max = u8; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_USMALLINT: { - uint16_t u16 = *(uint16_t *)&colVal.value.val; - pColAgg->sum += u16; - if (!minAssigned || pColAgg->min > u16) { - pColAgg->min = u16; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < u16) { - pColAgg->max = u16; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_UINT: { - uint32_t u32 = *(uint32_t *)&colVal.value.val; - pColAgg->sum += u32; - if (!minAssigned || pColAgg->min > u32) { - pColAgg->min = u32; - minAssigned = true; - } - if (!minAssigned || pColAgg->max < u32) { - pColAgg->max = u32; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_UBIGINT: { - uint64_t u64 = *(uint64_t *)&colVal.value.val; - pColAgg->sum += u64; - if (!minAssigned || pColAgg->min > u64) { - pColAgg->min = u64; - minAssigned = true; - } - if (!maxAssigned || pColAgg->max < u64) { - pColAgg->max = u64; - maxAssigned = true; - } - break; - } - case TSDB_DATA_TYPE_JSON: - break; - case TSDB_DATA_TYPE_VARBINARY: - break; - case TSDB_DATA_TYPE_DECIMAL: - break; - case TSDB_DATA_TYPE_BLOB: - break; - case TSDB_DATA_TYPE_MEDIUMBLOB: - break; - default: - ASSERT(0); - } + tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); } } } From d789fe6ac84300611965803ee43c483ce10297a7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 10:40:54 +0800 Subject: [PATCH 20/64] refact commit code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 483 ++++++++++++----------- 1 file changed, 253 insertions(+), 230 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a619b9f2e4..3daab532f3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -115,33 +115,32 @@ int32_t tRowInfoCmprFn(const void *p1, const void *p2) { int32_t tsdbBegin(STsdb *pTsdb) { int32_t code = 0; + int32_t lino = 0; if (!pTsdb) return code; SMemTable *pMemTable; code = tsdbMemTableCreate(pTsdb, &pMemTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // lock - code = taosThreadRwlockWrlock(&pTsdb->rwLock); - if (code) { + if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pTsdb->mem = pMemTable; // unlock - code = taosThreadRwlockUnlock(&pTsdb->rwLock); - if (code) { + if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) { code = TAOS_SYSTEM_ERROR(code); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - return code; - -_err: - tsdbError("vgId:%d, tsdb begin failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } @@ -149,6 +148,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { if (!pTsdb) return 0; int32_t code = 0; + int32_t lino = 0; SCommitter commith; SMemTable *pMemTable = pTsdb->mem; @@ -164,76 +164,74 @@ int32_t tsdbCommit(STsdb *pTsdb) { // start commit code = tsdbStartCommit(pTsdb, &commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // commit impl code = tsdbCommitData(&commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbCommitDel(&commith); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // end commit code = tsdbEndCommit(&commith, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - return code; - -_err: - tsdbEndCommit(&commith, code); - tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbEndCommit(&commith, code); + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; - pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); - if (pCommitter->aDelIdx == NULL) { + if ((pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData)); - if (pCommitter->aDelData == NULL) { + if ((pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx)); - if (pCommitter->aDelIdxN == NULL) { + if ((pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx))) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } SDelFile *pDelFileR = pCommitter->fs.pDelFile; if (pDelFileR) { code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // prepare new SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0}; code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); - return code; - -_err: - tsdbError("vgId:%d, commit del start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); + } return code; } static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) { int32_t code = 0; + int32_t lino = 0; SDelData *pDelData; tb_uid_t suid; tb_uid_t uid; @@ -252,7 +250,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel uid = pDelIdx->uid; code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { taosArrayClear(pCommitter->aDelData); } @@ -266,62 +264,63 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel for (; pDelData; pDelData = pDelData->pNext) { if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } // write code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // put delIdx if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d, commit table del failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->pDelFReader) { code = tsdbDelFReaderClose(&pCommitter->pDelFReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } taosArrayDestroy(pCommitter->aDelIdx); taosArrayDestroy(pCommitter->aDelData); taosArrayDestroy(pCommitter->aDelIdxN); - return code; - -_err: - tsdbError("vgId:%d, commit del end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { int32_t code = 0; + int32_t lino = 0; if (suid) { if (pSkmInfo->suid == suid) { @@ -336,7 +335,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo pSkmInfo->uid = uid; tTSchemaDestroy(pSkmInfo->pTSchema); code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); _exit: return code; @@ -344,6 +343,7 @@ _exit: static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) { int32_t code = 0; + int32_t lino = 0; if (pCommitter->skmRow.pTSchema) { if (pCommitter->skmRow.suid == suid) { @@ -359,9 +359,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid pCommitter->skmRow.uid = uid; tTSchemaDestroy(pCommitter->skmRow.pTSchema); code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema); - if (code) { - goto _exit; - } + TSDB_CHECK_CODE(code, lino, _exit); _exit: return code; @@ -369,6 +367,7 @@ _exit: static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; ASSERT(pCommitter->dReader.pBlockIdx); @@ -378,7 +377,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); ASSERT(pCommitter->dReader.mBlock.nItem > 0); } else { @@ -391,6 +390,7 @@ _exit: static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; pCommitter->pIter = NULL; tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); @@ -431,14 +431,14 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pIter->iStt = iStt; code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; pIter->iSttBlk = 0; SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); code = tsdbReadSttBlock(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; @@ -460,16 +460,18 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { } code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SDFileSet *pRSet = NULL; @@ -484,17 +486,17 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // data code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pCommitter->dReader.iBlockIdx = 0; if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { pCommitter->dReader.pBlockIdx = NULL; } @@ -531,7 +533,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { } wSet.aSttF[wSet.nSttF - 1] = &fStt; code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); taosArrayClear(pCommitter->dWriter.aBlockIdx); taosArrayClear(pCommitter->dWriter.aSttBlk); @@ -541,18 +543,18 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { // open iter code = tsdbOpenCommitIter(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); _exit: - return code; - -_err: - tsdbError("vgId:%d, commit file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { int32_t code = 0; + int32_t lino = 0; if (pBlockData->nRow == 0) return code; @@ -586,24 +588,25 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa dataBlk.nSubBlock++; code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // put SDataBlk code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // clear tBlockDataClear(pBlockData); - return code; - -_err: - tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { int32_t code = 0; + int32_t lino = 0; SSttBlk sstBlk; if (pBlockData->nRow == 0) return code; @@ -626,114 +629,117 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray // write code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // push SSttBlk if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // clear tBlockDataClear(pBlockData); - return code; - -_err: - tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; // write aBlockIdx code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // write aSttBlk code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // update file header code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // upsert SDFileSet code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // close and sync code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->dReader.pReader) { code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { int32_t code = 0; + int32_t lino = 0; while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } - return code; - -_err: - tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); static int32_t tsdbCommitFileData(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; // commit file data start code = tsdbCommitFileDataStart(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // impl code = tsdbCommitFileDataImpl(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // commit file data end code = tsdbCommitFileDataEnd(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - tsdbDataFReaderClose(&pCommitter->dReader.pReader); - tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbDataFReaderClose(&pCommitter->dReader.pReader); + tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); + } return code; } // ---------------------------------------------------------------------------- static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; memset(pCommitter, 0, sizeof(*pCommitter)); ASSERT(pTsdb->mem && pTsdb->imem == NULL); @@ -754,30 +760,31 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); if (pCommitter->aTbDataP == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } code = tsdbFSCopy(pTsdb, &pCommitter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; // reader pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pCommitter->dReader.bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); // merger for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { @@ -785,33 +792,36 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pIter->aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pIter->bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } // writer pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); if (pCommitter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); if (pCommitter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } code = tBlockDataCreate(&pCommitter->dWriter.bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataCreate(&pCommitter->dWriter.bDatal); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); _exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } @@ -839,7 +849,9 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { } static int32_t tsdbCommitData(SCommitter *pCommitter) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; @@ -848,30 +860,29 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { // start ==================== code = tsdbCommitDataStart(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // impl ==================== pCommitter->nextKey = pMemTable->minKey; while (pCommitter->nextKey < TSKEY_MAX) { code = tsdbCommitFileData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // end ==================== tsdbCommitDataEnd(pCommitter); _exit: - tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); - return code; - -_err: - tsdbCommitDataEnd(pCommitter); - tsdbError("vgId:%d, commit data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitDel(SCommitter *pCommitter) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; @@ -882,7 +893,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { // start code = tsdbCommitDelStart(pCommitter); if (code) { - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } // impl @@ -918,7 +929,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_mem_del: code = tsdbCommitTableDel(pCommitter, pTbData, NULL); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iTbData++; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; @@ -926,7 +937,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_disk_del: code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iDelIdx++; pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL; @@ -934,7 +945,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _commit_mem_and_disk_del: code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iTbData++; pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL; @@ -945,28 +956,28 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { // end code = tsdbCommitDelEnd(pCommitter); - if (code) { - goto _err; - } + TSDB_CHECK_CODE(code, lino, _exit); _exit: - tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); - return code; - -_err: - tsdbError("vgId:%d, commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); + } return code; } static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + STsdb *pTsdb = pCommitter->pTsdb; SMemTable *pMemTable = pTsdb->imem; ASSERT(eno == 0); code = tsdbFSCommit1(pTsdb, &pCommitter->fs); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // lock taosThreadRwlockWrlock(&pTsdb->rwLock); @@ -975,7 +986,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { code = tsdbFSCommit2(pTsdb, &pCommitter->fs); if (code) { taosThreadRwlockUnlock(&pTsdb->rwLock); - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } pTsdb->imem = NULL; @@ -987,16 +998,12 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); - // if (pCommitter->toMerge) { - // code = tsdbMerge(pTsdb); - // if (code) goto _err; - // } - - tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); - return code; - -_err: - tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + } else { + tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); + } return code; } @@ -1008,6 +1015,7 @@ static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; if (pCommitter->pIter) { SDataIter *pIter = pCommitter->pIter; @@ -1050,7 +1058,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); code = tsdbReadSttBlock(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; @@ -1085,11 +1093,16 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { } _exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SBlockData *pBlockData = &pCommitter->dWriter.bData; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; @@ -1098,13 +1111,13 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) while (pRowInfo) { ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo) { @@ -1119,29 +1132,31 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBlockData->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; SBlockData *pBDataR = &pCommitter->dReader.bData; SBlockData *pBDataW = &pCommitter->dWriter.bData; code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); tBlockDataClear(pBDataW); int32_t iRow = 0; @@ -1152,7 +1167,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); if (c < 0) { code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iRow++; if (iRow < pBDataR->nRow) { @@ -1163,13 +1178,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) } else if (c > 0) { ASSERT(pRowInfo->row.type == 0); code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo) { @@ -1186,13 +1201,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBDataW->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } while (pRow) { code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iRow++; if (iRow < pBDataR->nRow) { @@ -1203,22 +1218,24 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) if (pBDataW->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { - int32_t code = 0; + int32_t code = 0; + int32_t lino = 0; + SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); @@ -1237,7 +1254,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { if (c < 0) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1247,13 +1264,13 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { } } else if (c > 0) { code = tsdbCommitAheadBlock(pCommitter, pDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; } else { code = tsdbCommitMergeBlock(pCommitter, pDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1268,7 +1285,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { while (pDataBlk) { code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); iBlock++; if (iBlock < pCommitter->dReader.mBlock.nItem) { @@ -1279,25 +1296,25 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { } code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: - return code; - -_err: - tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; + int32_t lino = 0; SBlockData *pBDatal = &pCommitter->dWriter.bDatal; if (pBDatal->suid || pBDatal->uid) { if ((pBDatal->suid != id.suid) || (id.suid == 0)) { code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); tBlockDataReset(pBDatal); } } @@ -1306,42 +1323,48 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); - if (code) goto _exit; + TSDB_CHECK_CODE(code, lino, _exit); } _exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; SBlockData *pBData = &pCommitter->dWriter.bData; SBlockData *pBDatal = &pCommitter->dWriter.bDatal; TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; code = tsdbInitLastBlockIfNeed(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (pBDatal->nRow >= pCommitter->maxRow) { code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } - return code; - -_err: +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; + int32_t lino = 0; SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { @@ -1354,7 +1377,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pCommitter->toLastOnly) { pBData = &pCommitter->dWriter.bDatal; code = tsdbInitLastBlockIfNeed(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { pBData = &pCommitter->dWriter.bData; ASSERT(pBData->nRow == 0); @@ -1364,15 +1387,15 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { STSchema *pTSchema = NULL; if (pRowInfo->row.type == 0) { code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pTSchema = pCommitter->skmRow.pTSchema; } code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pRowInfo = tsdbGetCommitRow(pCommitter); if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { @@ -1382,11 +1405,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pBData->nRow >= pCommitter->maxRow) { if (pCommitter->toLastOnly) { code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } } @@ -1394,23 +1417,23 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (!pCommitter->toLastOnly && pBData->nRow) { if (pBData->nRow > pCommitter->minRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } else { code = tsdbAppendLastBlock(pCommitter); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } _exit: - return code; - -_err: - tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { int32_t code = 0; + int32_t lino = 0; SRowInfo *pRowInfo; TABLEID id = {0}; @@ -1420,36 +1443,36 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { id.uid = pRowInfo->uid; code = tsdbMoveCommitData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // start tMapDataReset(&pCommitter->dWriter.mBlock); // impl code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* merge with data in .data file */ code = tsdbMergeTableData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* handle remain table data */ code = tsdbCommitTableData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); // end if (pCommitter->dWriter.mBlock.nItem > 0) { SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + TSDB_CHECK_CODE(code, lino, _exit); } } } @@ -1457,15 +1480,15 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { id.suid = INT64_MAX; id.uid = INT64_MAX; code = tsdbMoveCommitData(pCommitter, id); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); - return code; - -_err: - tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); +_exit: + if (code) { + tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } return code; } From 9624073d8447963b4dc9204ab5f5018c28d8a420 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 10:46:05 +0800 Subject: [PATCH 21/64] refact more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 64 +++++++++++++++--------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 3daab532f3..1e2a3d5b52 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -139,7 +139,7 @@ int32_t tsdbBegin(STsdb *pTsdb) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -180,7 +180,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { _exit: if (code) { tsdbEndCommit(&commith, code); - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -222,7 +222,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); } @@ -280,7 +280,8 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel _exit: if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -313,7 +314,8 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -464,7 +466,8 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -547,7 +550,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -599,7 +602,7 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -642,7 +645,7 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -678,7 +681,8 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -703,7 +707,8 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -729,7 +734,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbDataFReaderClose(&pCommitter->dReader.pReader); tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); } @@ -767,7 +772,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -820,7 +825,8 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -874,7 +880,7 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -960,7 +966,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); } @@ -1000,7 +1006,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); } @@ -1094,7 +1100,8 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1141,7 +1148,8 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1227,7 +1235,8 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1301,7 +1310,8 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1328,7 +1338,8 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1357,7 +1368,8 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1426,7 +1438,8 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } @@ -1488,7 +1501,8 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tstrerror(code)); } return code; } From da2150a15b3a99c668e9ebb4bd40045ef51cac9d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 16:45:38 +0800 Subject: [PATCH 22/64] more code 1 --- source/dnode/vnode/src/inc/tsdb.h | 14 ++ source/dnode/vnode/src/tsdb/tsdbCommit.c | 192 ++++++++++++--------- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 14 -- 3 files changed, 120 insertions(+), 100 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index dc218e86e0..36aa5c0804 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -686,6 +686,20 @@ struct SDiskData { SArray *aDiskCol; // SArray }; +struct SDiskDataBuilder { + int64_t suid; + int64_t uid; + int32_t nRow; + uint8_t cmprAlg; + uint8_t calcSma; + SCompressor *pUidC; + SCompressor *pVerC; + SCompressor *pKeyC; + int32_t nBuilder; + SArray *aBuilder; // SArray + uint8_t *aBuf[2]; +}; + int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1e2a3d5b52..0ff9ae5085 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -70,12 +70,12 @@ typedef struct { int8_t toLastOnly; }; struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; // SArray - SArray *aSttBlk; // SArray - SMapData mBlock; // SMapData - SBlockData bData; - SBlockData bDatal; + SDataFWriter *pWriter; + SArray *aBlockIdx; // SArray + SArray *aSttBlk; // SArray + SMapData mBlock; // SMapData + SBlockData bData; + SDiskDataBuilder *pBuilder; } dWriter; SSkmInfo skmTable; SSkmInfo skmRow; @@ -139,7 +139,7 @@ int32_t tsdbBegin(STsdb *pTsdb) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -180,7 +180,7 @@ int32_t tsdbCommit(STsdb *pTsdb) { _exit: if (code) { tsdbEndCommit(&commith, code); - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -222,7 +222,7 @@ static int32_t tsdbCommitDelStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode)); } @@ -280,7 +280,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel _exit: if (code) { - tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -314,7 +314,7 @@ static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d, %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -466,7 +466,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -542,7 +542,6 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); - tBlockDataReset(&pCommitter->dWriter.bDatal); // open iter code = tsdbOpenCommitIter(pCommitter); @@ -550,7 +549,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -602,7 +601,7 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -645,7 +644,7 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -681,7 +680,7 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -707,7 +706,7 @@ static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -734,7 +733,7 @@ static int32_t tsdbCommitFileData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); tsdbDataFReaderClose(&pCommitter->dReader.pReader); tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); } @@ -772,7 +771,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -820,12 +819,12 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataCreate(&pCommitter->dWriter.bData); TSDB_CHECK_CODE(code, lino, _exit); - code = tBlockDataCreate(&pCommitter->dWriter.bDatal); + code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -849,7 +848,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); - tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); + tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } @@ -880,7 +879,7 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; } @@ -966,7 +965,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel); } @@ -1006,7 +1005,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code)); } else { tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode)); } @@ -1100,7 +1099,7 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1148,7 +1147,7 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1235,7 +1234,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1310,7 +1309,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1320,25 +1319,26 @@ static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; int32_t lino = 0; - SBlockData *pBDatal = &pCommitter->dWriter.bDatal; - if (pBDatal->suid || pBDatal->uid) { - if ((pBDatal->suid != id.suid) || (id.suid == 0)) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; + if (pBuilder->suid || pBuilder->uid) { + if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { + // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, + // pCommitter->cmprAlg); // todo TSDB_CHECK_CODE(code, lino, _exit); - tBlockDataReset(pBDatal); + // tBlockDataReset(pBDatal); } } - if (!pBDatal->suid && !pBDatal->uid) { + if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); + // code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); todo TSDB_CHECK_CODE(code, lino, _exit); } _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1349,7 +1349,6 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { int32_t lino = 0; SBlockData *pBData = &pCommitter->dWriter.bData; - SBlockData *pBDatal = &pCommitter->dWriter.bDatal; TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; code = tsdbInitLastBlockIfNeed(pCommitter, id); @@ -1357,18 +1356,20 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); - code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); + + code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); TSDB_CHECK_CODE(code, lino, _exit); - if (pBDatal->nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, NULL /*TODO */, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); TSDB_CHECK_CODE(code, lino, _exit); } } _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1385,60 +1386,79 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pRowInfo == NULL) goto _exit; - SBlockData *pBData; if (pCommitter->toLastOnly) { - pBData = &pCommitter->dWriter.bDatal; - code = tsdbInitLastBlockIfNeed(pCommitter, id); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - pBData = &pCommitter->dWriter.bData; - ASSERT(pBData->nRow == 0); - } + // init the data if need - while (pRowInfo) { - STSchema *pTSchema = NULL; - if (pRowInfo->row.type == 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); - TSDB_CHECK_CODE(code, lino, _exit); - pTSchema = pCommitter->skmRow.pTSchema; - } - - code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); - TSDB_CHECK_CODE(code, lino, _exit); - - code = tsdbNextCommitRow(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); - - pRowInfo = tsdbGetCommitRow(pCommitter); - if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { - pRowInfo = NULL; - } - - if (pBData->nRow >= pCommitter->maxRow) { - if (pCommitter->toLastOnly) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); TSDB_CHECK_CODE(code, lino, _exit); - } else { + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbNextCommitRow(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { + // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, + // pCommitter->cmprAlg); (todo) + TSDB_CHECK_CODE(code, lino, _exit); + } + } + } else { + SBlockData *pBData = &pCommitter->dWriter.bData; + ASSERT(pBData->nRow == 0); + + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + TSDB_CHECK_CODE(code, lino, _exit); + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbNextCommitRow(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pBData->nRow >= pCommitter->maxRow) { code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); TSDB_CHECK_CODE(code, lino, _exit); } } - } - if (!pCommitter->toLastOnly && pBData->nRow) { - if (pBData->nRow > pCommitter->minRow) { - code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbAppendLastBlock(pCommitter); - TSDB_CHECK_CODE(code, lino, _exit); + if (pBData->nRow) { + if (pBData->nRow > pCommitter->minRow) { + code = + tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + code = tsdbAppendLastBlock(pCommitter); + TSDB_CHECK_CODE(code, lino, _exit); + } } } _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; @@ -1495,13 +1515,13 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); + // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + // pCommitter->cmprAlg); TSDB_CHECK_CODE(code, lino, _exit); _exit: if (code) { - tsdbError("vgId:%d %s fail at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino, tstrerror(code)); } return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 8d25873036..8c7933057d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -34,20 +34,6 @@ struct SDiskColBuilder { uint8_t *aBuf[1]; }; -struct SDiskDataBuilder { - int64_t suid; - int64_t uid; - int32_t nRow; - uint8_t cmprAlg; - uint8_t calcSma; - SCompressor *pUidC; - SCompressor *pVerC; - SCompressor *pKeyC; - int32_t nBuilder; - SArray *aBuilder; // SArray - uint8_t *aBuf[2]; -}; - // SDiskColBuilder ================================================ #define tDiskColBuilderCreate() \ (SDiskColBuilder) { 0 } From 8ff1eb41d1a17614bdc4382444a82d1e83c0e4ae Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 17:13:14 +0800 Subject: [PATCH 23/64] more code --- source/dnode/vnode/src/inc/tsdb.h | 1 + source/dnode/vnode/src/tsdb/tsdbCommit.c | 58 +++++++++++++++++++----- 2 files changed, 47 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 36aa5c0804..618364d20f 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -698,6 +698,7 @@ struct SDiskDataBuilder { int32_t nBuilder; SArray *aBuilder; // SArray uint8_t *aBuf[2]; + SDiskData dd; }; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 0ff9ae5085..70251933d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -649,6 +649,45 @@ _exit: return code; } +static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilder, SArray *aSttBlk) { + int32_t code = 0; + int32_t lino = 0; + + if (pBuilder->nRow == 0) return code; + + SSttBlk sttBlk = {.suid = pBuilder->suid, + .minUid = 0, // todo + .maxUid = 0, // todo + .minKey = 0, // todo + .maxKey = 0, // todo + .minVer = 0, // todo + .maxVer = 0, // todo + .nRow = pBuilder->nRow}; + + // gnrt + code = tGnrtDiskData(pBuilder, &pBuilder->dd); + TSDB_CHECK_CODE(code, lino, _exit); + + // write + // code = tsdbWriteDiskData(pWriter, &pBuilder->dd); + TSDB_CHECK_CODE(code, lino, _exit); + + // push + if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + + // clear (todo) + // tDiskDataBuilderClear(pBuilder); + +_exit: + if (code) { + tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); + } + return code; +} + static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { int32_t code = 0; int32_t lino = 0; @@ -1315,24 +1354,22 @@ _exit: return code; } -static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { +static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) { int32_t code = 0; int32_t lino = 0; SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; if (pBuilder->suid || pBuilder->uid) { if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { - // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, - // pCommitter->cmprAlg); // todo + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); - // tBlockDataReset(pBDatal); } } if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - // code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); todo + code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1351,7 +1388,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { SBlockData *pBData = &pCommitter->dWriter.bData; TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - code = tsdbInitLastBlockIfNeed(pCommitter, id); + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { @@ -1361,8 +1398,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, NULL /*TODO */, pCommitter->dWriter.aSttBlk, - pCommitter->cmprAlg); + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -1409,8 +1445,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { } if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { - // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, - // pCommitter->cmprAlg); (todo) + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); } } @@ -1515,8 +1550,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); - // code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, - // pCommitter->cmprAlg); + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); _exit: From b682fdd261282e88d579b8f2a7bfda2fd20499b7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 17:57:38 +0800 Subject: [PATCH 24/64] more code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 70251933d5..2817e26078 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1550,7 +1550,7 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); - code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); + code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); _exit: From f24c8fbe560e644e7e95cc2b050acb5bc9df9517 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 23 Sep 2022 19:03:29 +0800 Subject: [PATCH 25/64] make it compile --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 34 +++++++++++----------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 8c7933057d..2c1d6d2149 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -131,7 +131,7 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { - if (pColVal->isNone || pColVal->isNull) { + if (!COL_VAL_IS_VALUE(pColVal)) { pBuilder->sma.numOfNull++; } else { tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); @@ -158,9 +158,9 @@ static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag = HAS_NONE; - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { pBuilder->flag = HAS_NULL; } else { pBuilder->flag = HAS_VALUE; @@ -173,7 +173,7 @@ static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { / static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE int32_t code = 0; - if (!pColVal->isNone) { + if (!COL_VAL_IS_NONE(pColVal)) { // bit map int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); @@ -184,7 +184,7 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); // value - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { pBuilder->flag |= HAS_NULL; } else { pBuilder->flag |= HAS_VALUE; @@ -205,12 +205,12 @@ static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { / static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL int32_t code = 0; - if (!pColVal->isNull) { + if (!COL_VAL_IS_NULL(pColVal)) { int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); code = tRealloc(&pBuilder->pBitMap, nBit); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag |= HAS_NONE; memset(pBuilder->pBitMap, 255, nBit); @@ -238,12 +238,12 @@ _exit: static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL|HAS_NONE int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; @@ -279,8 +279,8 @@ _exit: static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE int32_t code = 0; - if (pColVal->isNone || pColVal->isNull) { - if (pColVal->isNone) { + if (!COL_VAL_IS_VALUE(pColVal)) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag |= HAS_NONE; } else { pBuilder->flag |= HAS_NULL; @@ -306,7 +306,7 @@ _exit: static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NONE int32_t code = 0; - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { pBuilder->flag |= HAS_NULL; uint8_t *pBitMap = NULL; @@ -324,7 +324,7 @@ static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { / code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); @@ -339,7 +339,7 @@ _exit: static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL int32_t code = 0; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { pBuilder->flag |= HAS_NONE; uint8_t *pBitMap = NULL; @@ -357,7 +357,7 @@ static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { / code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - if (pColVal->isNull) { + if (COL_VAL_IS_NULL(pColVal)) { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); } else { SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); @@ -375,9 +375,9 @@ static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { / code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); if (code) goto _exit; - if (pColVal->isNone) { + if (COL_VAL_IS_NONE(pColVal)) { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (pColVal->isNull) { + } else if (COL_VAL_IS_NULL(pColVal)) { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); } else { SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index ad99b04e5f..1d7eff9a08 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1589,7 +1589,7 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { tColDataGetValue(pColData, iVal, &cv); - if (cv.isNone || cv.isNull) { + if (!COL_VAL_IS_VALUE(&cv)) { pColAgg->numOfNull++; } else { tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); From 64434678cafe142410172543947b0c61d07b32c0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 25 Sep 2022 18:09:54 +0800 Subject: [PATCH 26/64] refact more code --- source/util/src/tcompression.c | 87 ++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 20 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 619ae8360d..3a6dbd4acc 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1001,38 +1001,49 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co *************************************************************************/ #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static struct { int8_t type; int32_t bytes; int8_t isVarLen; + int32_t (*initFn)(SCompressor *, int8_t type, int8_t cmprAlg); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); } DATA_TYPE_INFO[] = { - {TSDB_DATA_TYPE_NULL, 0, 0, NULL}, // TSDB_DATA_TYPE_NULL - {TSDB_DATA_TYPE_BOOL, 1, 0, tCompBool}, // TSDB_DATA_TYPE_BOOL - {TSDB_DATA_TYPE_TINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_TINYINT - {TSDB_DATA_TYPE_SMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_SMALLINT - {TSDB_DATA_TYPE_INT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_INT - {TSDB_DATA_TYPE_BIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_BIGINT - {TSDB_DATA_TYPE_FLOAT, 4, 0, tCompFloat}, // TSDB_DATA_TYPE_FLOAT - {TSDB_DATA_TYPE_DOUBLE, 8, 0, tCompDouble}, // TSDB_DATA_TYPE_DOUBLE - {TSDB_DATA_TYPE_VARCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARCHAR - {TSDB_DATA_TYPE_TIMESTAMP, 8, 0, tCompTimestamp}, // pTSDB_DATA_TYPE_TIMESTAMP - {TSDB_DATA_TYPE_NCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_NCHAR - {TSDB_DATA_TYPE_UTINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_UTINYINT - {TSDB_DATA_TYPE_USMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_USMALLINT - {TSDB_DATA_TYPE_UINT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_UINT - {TSDB_DATA_TYPE_UBIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_UBIGINT - {TSDB_DATA_TYPE_JSON, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_JSON - {TSDB_DATA_TYPE_VARBINARY, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARBINARY - {TSDB_DATA_TYPE_DECIMAL, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_DECIMAL - {TSDB_DATA_TYPE_BLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_BLOB - {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_MEDIUMBLOB + {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .initFn = NULL, .cmprFn = NULL}, // TSDB_DATA_TYPE_NULL + {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .initFn = tCompBoolInit, .cmprFn = tCompBool}, + {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .initFn = tCompFloatInit, .cmprFn = tCompFloat}, + {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .initFn = tCompDoubleInit, .cmprFn = tCompDouble}, + {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_TIMESTAMP, + .bytes = 8, + .isVarLen = 0, + .initFn = tCompTimestampInit, + .cmprFn = tCompTimestamp}, + {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, + {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, }; struct SCompressor { @@ -1072,6 +1083,12 @@ struct SCompressor { }; // Timestamp ===================================================== +static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompSetCopyMode(SCompressor *pCmprsor) { int32_t code = 0; @@ -1207,6 +1224,12 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; +static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1314,6 +1337,12 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } // Float ===================================================== +static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1377,6 +1406,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat } // Double ===================================================== +static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1440,6 +1475,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa } // Binary ===================================================== +static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1460,6 +1501,12 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nDa // Bool ===================================================== static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; +static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + // TODO + return code; +} + static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; From efacafde33d065c9942ad58d02d100b4d03f1afa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 10:47:24 +0800 Subject: [PATCH 27/64] refact more code --- include/util/tcompression.h | 4 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 20 +- source/util/src/tcompression.c | 316 ++++++++++++++------- 3 files changed, 219 insertions(+), 121 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index f22205d144..8f3d9634ec 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -130,8 +130,8 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); -int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); +int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 2c1d6d2149..9e7e4cefe0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -65,12 +65,12 @@ static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_ if (IS_VAR_DATA_TYPE(type)) { if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code; - code = tCompressorInit(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); + code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); if (code) return code; } if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code; - code = tCompressorInit(pBuilder->pValC, type, cmprAlg); + code = tCompressStart(pBuilder->pValC, type, cmprAlg); if (code) return code; if (pBuilder->calcSma) { @@ -116,13 +116,13 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { // OFFSET if (IS_VAR_DATA_TYPE(pBuilder->type)) { - code = tCompGen(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); + code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); if (code) return code; } // VALUE if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { - code = tCompGen(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); + code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); if (code) return code; } @@ -457,15 +457,15 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB pBuilder->calcSma = calcSma; if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; - code = tCompressorInit(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code; - code = tCompressorInit(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); + code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); if (code) return code; if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code; - code = tCompressorInit(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); + code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); if (code) return code; if (pBuilder->aBuilder == NULL) { @@ -586,16 +586,16 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { // UID if (pBuilder->uid == 0) { - code = tCompGen(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); + code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); if (code) return code; } // VERSION - code = tCompGen(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); + code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); if (code) return code; // TSKEY - code = tCompGen(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); + code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); if (code) return code; int32_t offset = 0; diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 3a6dbd4acc..f1d9b860b8 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1001,49 +1001,157 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co *************************************************************************/ #define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) -static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData); -static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); +static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); +static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); static struct { int8_t type; int32_t bytes; int8_t isVarLen; - int32_t (*initFn)(SCompressor *, int8_t type, int8_t cmprAlg); + int32_t (*startFn)(SCompressor *, int8_t type, int8_t cmprAlg); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); + int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); } DATA_TYPE_INFO[] = { - {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .initFn = NULL, .cmprFn = NULL}, // TSDB_DATA_TYPE_NULL - {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .initFn = tCompBoolInit, .cmprFn = tCompBool}, - {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .initFn = tCompFloatInit, .cmprFn = tCompFloat}, - {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .initFn = tCompDoubleInit, .cmprFn = tCompDouble}, - {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + {.type = TSDB_DATA_TYPE_NULL, + .bytes = 0, + .isVarLen = 0, + .startFn = NULL, + .cmprFn = NULL, + .endFn = NULL}, // TSDB_DATA_TYPE_NULL + {.type = TSDB_DATA_TYPE_BOOL, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompBoolStart, + .cmprFn = tCompBool, + .endFn = tCompBoolEnd}, + {.type = TSDB_DATA_TYPE_TINYINT, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_SMALLINT, + .bytes = 2, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_INT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_BIGINT, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_FLOAT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompFloatStart, + .cmprFn = tCompFloat, + .endFn = tCompFloatEnd}, + {.type = TSDB_DATA_TYPE_DOUBLE, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompDoubleStart, + .cmprFn = tCompDouble, + .endFn = tCompDoubleEnd}, + {.type = TSDB_DATA_TYPE_VARCHAR, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8, .isVarLen = 0, - .initFn = tCompTimestampInit, - .cmprFn = tCompTimestamp}, - {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .initFn = tCompIntInit, .cmprFn = tCompInt}, - {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, - {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .initFn = tCompBinaryInit, .cmprFn = tCompBinary}, + .startFn = tCompTimestampStart, + .cmprFn = tCompTimestamp, + .endFn = tCompTimestampEnd}, + {.type = TSDB_DATA_TYPE_NCHAR, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_UTINYINT, + .bytes = 1, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_USMALLINT, + .bytes = 2, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_UINT, + .bytes = 4, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_UBIGINT, + .bytes = 8, + .isVarLen = 0, + .startFn = tCompIntStart, + .cmprFn = tCompInt, + .endFn = tCompIntEnd}, + {.type = TSDB_DATA_TYPE_JSON, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_VARBINARY, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_DECIMAL, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_BLOB, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, + {.type = TSDB_DATA_TYPE_MEDIUMBLOB, + .bytes = 1, + .isVarLen = 1, + .startFn = tCompBinaryStart, + .cmprFn = tCompBinary, + .endFn = tCompBinaryEnd}, }; struct SCompressor { @@ -1083,9 +1191,13 @@ struct SCompressor { }; // Timestamp ===================================================== -static int32_t tCompTimestampInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - // TODO + pCmprsor->ts_prev_val = 0; + pCmprsor->ts_prev_delta = 0; + pCmprsor->ts_flag_p = NULL; + pCmprsor->aBuf[0][0] = 1; + pCmprsor->nBuf[0] = 1; return code; } @@ -1215,6 +1327,12 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t return code; } +static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + // TODO + return code; +} + // Integer ===================================================== #define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; @@ -1224,9 +1342,14 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; -static int32_t tCompIntInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - // TODO + pCmprsor->i_prev = 0; + pCmprsor->i_selector = 0; + pCmprsor->i_start = 0; + pCmprsor->i_end = 0; + pCmprsor->aBuf[0][0] = 0; + pCmprsor->nBuf[0] = 1; return code; } @@ -1336,13 +1459,22 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) return code; } -// Float ===================================================== -static int32_t tCompFloatInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; // TODO return code; } +// Float ===================================================== +static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + pCmprsor->f_prev = 0; + pCmprsor->f_flag_p = NULL; + pCmprsor->aBuf[0][0] = 0; + pCmprsor->nBuf[0] = 1; + return code; +} + static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1405,13 +1537,22 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat return code; } -// Double ===================================================== -static int32_t tCompDoubleInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; // TODO return code; } +// Double ===================================================== +static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + pCmprsor->d_prev = 0; + pCmprsor->d_flag_p = NULL; + pCmprsor->aBuf[0][0] = 0; + pCmprsor->nBuf[0] = 1; + return code; +} + static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1474,13 +1615,19 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa return code; } -// Binary ===================================================== -static int32_t tCompBinaryInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; // TODO return code; } +// Binary ===================================================== +static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { + int32_t code = 0; + pCmprsor->nBuf[0] = 0; + return code; +} + static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1498,12 +1645,18 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nDa return code; } +static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + // TODO + return code; +} + // Bool ===================================================== static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; -static int32_t tCompBoolInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - // TODO + pCmprsor->nBuf[0] = 0; return code; } @@ -1531,6 +1684,12 @@ static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData return code; } +static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { + int32_t code = 0; + // TODO + return code; +} + // SCompressor ===================================================== int32_t tCompressorCreate(SCompressor **ppCmprsor) { int32_t code = 0; @@ -1565,7 +1724,7 @@ int32_t tCompressorDestroy(SCompressor *pCmprsor) { return code; } -int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { +int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; pCmprsor->type = type; @@ -1573,82 +1732,21 @@ int32_t tCompressorInit(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { pCmprsor->autoAlloc = 1; pCmprsor->nVal = 0; - switch (type) { - case TSDB_DATA_TYPE_TIMESTAMP: - pCmprsor->ts_prev_val = 0; - pCmprsor->ts_prev_delta = 0; - pCmprsor->ts_flag_p = NULL; - pCmprsor->aBuf[0][0] = 1; // For timestamp, 1 means compressed, 0 otherwise - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_BOOL: - pCmprsor->nBuf[0] = 0; - break; - case TSDB_DATA_TYPE_BINARY: - pCmprsor->nBuf[0] = 0; - break; - case TSDB_DATA_TYPE_FLOAT: - pCmprsor->f_prev = 0; - pCmprsor->f_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_DOUBLE: - pCmprsor->d_prev = 0; - pCmprsor->d_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - pCmprsor->i_prev = 0; - pCmprsor->i_selector = 0; - pCmprsor->i_start = 0; - pCmprsor->i_end = 0; - pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility) - pCmprsor->nBuf[0] = 1; - break; - default: - break; + if (DATA_TYPE_INFO[type].startFn) { + DATA_TYPE_INFO[type].startFn(pCmprsor, type, cmprAlg); } return code; } -int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - if (pCmprsor->nVal == 0) { - *ppData = NULL; - *nData = 0; - return code; - } + *ppData = NULL; + *nData = 0; - if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) { - code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1); - if (code) return code; - - int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]); - if (ret) { - pCmprsor->aBuf[1][0] = 0; - pCmprsor->nBuf[1] = ret + 1; - } else { - pCmprsor->aBuf[1][0] = 1; - memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]); - pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1; - } - - *ppData = pCmprsor->aBuf[1]; - *nData = pCmprsor->nBuf[1]; - } else { - *ppData = pCmprsor->aBuf[0]; - *nData = pCmprsor->nBuf[0]; + if (DATA_TYPE_INFO[pCmprsor->type].endFn) { + return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppData, nData); } return code; From a34cf50a83c39bfde85e0dee60ae9c543cc68f5d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 13:43:23 +0800 Subject: [PATCH 28/64] refact more code --- source/common/src/tdataformat.c | 609 +++++++++++++++++++------------- 1 file changed, 359 insertions(+), 250 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index a5e8d4be66..e461e06158 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -1135,288 +1135,397 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa _exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 +static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) { int32_t code = 0; - - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag = HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag = HAS_NULL; - } else { - pColData->flag = HAS_VALUE; - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE - int32_t code = 0; - - if (!COL_VAL_IS_NONE(pColVal)) { - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - - if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag |= HAS_NULL; - } else { - pColData->flag |= HAS_VALUE; - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL - int32_t code = 0; - - if (!COL_VAL_IS_NULL(pColVal)) { - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; - - memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else { - pColData->flag |= HAS_VALUE; - - memset(pColData->pBitMap, 0, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE - int32_t code = 0; - - if (COL_VAL_IS_NONE(pColVal)) { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - } else { - pColData->flag |= HAS_VALUE; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); - } - SET_BIT2(pBitMap, pColData->nVal, 2); - - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - - if (pColData->nVal) { - if (IS_VAR_DATA_TYPE(pColData->type)) { - int32_t nOffset = sizeof(int32_t) * pColData->nVal; - code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); - if (code) goto _exit; - memset(pColData->aOffset, 0, nOffset); - } else { - pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; - code = tRealloc(&pColData->pData, pColData->nData); - if (code) goto _exit; - memset(pColData->pData, 0, pColData->nData); - } - } - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE - int32_t code = 0; - - if (!COL_VAL_IS_VALUE(pColVal)) { - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; - } else { - pColData->flag |= HAS_NULL; - } - - int32_t nBit = BIT1_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, nBit); - if (code) goto _exit; - - memset(pColData->pBitMap, 255, nBit); - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } else { - code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - } - pColData->nVal++; - -_exit: - return code; -} -static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE - int32_t code = 0; - - if (COL_VAL_IS_NULL(pColVal)) { - pColData->flag |= HAS_NULL; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); - } - SET_BIT2(pBitMap, pColData->nVal, 1); - - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - } else { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); - } else { - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - } - } + pColData->flag = HAS_VALUE; code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; - + if (code) return code; pColData->nVal++; - -_exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL +static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->flag = HAS_NONE; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->flag = HAS_NULL; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - pColData->flag |= HAS_NONE; + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); - } - SET_BIT2(pBitMap, pColData->nVal, 0); + pColData->flag |= HAS_VALUE; - tFree(pColData->pBitMap); - pColData->pBitMap = pBitMap; - } else { - code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); } else { - SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); } } + code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + pColData->flag |= HAS_NULL; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + pColData->flag |= HAS_VALUE; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + pColData->flag |= HAS_NONE; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + pColData->nVal++; + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_VALUE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pColData->nVal, 2); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) return code; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) return code; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NONE; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; pColData->nVal++; -_exit: return code; } -static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, - SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NULL; + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) return code; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pColData->nVal, 1); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + pColData->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pColData->nVal, 0); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) { int32_t code = 0; code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); - if (code) goto _exit; + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - } else { - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - } code = tColDataPutValue(pColData, pColVal); - if (code) goto _exit; + if (code) return code; pColData->nVal++; -_exit: return code; } -static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = { - tColDataAppendValue0, // 0 - tColDataAppendValue1, // HAS_NONE - tColDataAppendValue2, // HAS_NULL - tColDataAppendValue3, // HAS_NULL|HAS_NONE - tColDataAppendValue4, // HAS_VALUE - tColDataAppendValue5, // HAS_VALUE|HAS_NONE - tColDataAppendValue6, // HAS_VALUE|HAS_NULL - tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) return code; + SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + + code = tColDataPutValue(pColData, pColVal); + if (code) return code; + + pColData->nVal++; + + return code; +} +static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = { + {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0 + {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE + {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL + {tColDataAppendValue30, tColDataAppendValue31, tColDataAppendValue32}, // HAS_NULL|HAS_NONE + {tColDataAppendValue40, tColDataAppendValue41, tColDataAppendValue42}, // HAS_VALUE + {tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE + {tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL + {tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE }; int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); - return tColDataAppendValueImpl[pColData->flag](pColData, pColVal); + return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal); } static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE From 30937189aa9ec7d854a54058a93a8f0c87ed6d02 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 14:10:58 +0800 Subject: [PATCH 29/64] refact more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 416 +++++++++++---------- 1 file changed, 221 insertions(+), 195 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 9e7e4cefe0..11f5c398ba 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -138,7 +138,7 @@ static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *p } } -static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { +static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (IS_VAR_DATA_TYPE(pColVal->type)) { @@ -155,256 +155,282 @@ static int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { return code; } -static int32_t tDiskColAddVal0(SDiskColBuilder *pBuilder, SColVal *pColVal) { // 0 +static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_VALUE; + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_NONE; + return 0; +} +static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) { + pBuilder->flag = HAS_NULL; + return 0; +} +static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag = HAS_NONE; - } else if (COL_VAL_IS_NULL(pColVal)) { - pBuilder->flag = HAS_NULL; - } else { - pBuilder->flag = HAS_VALUE; - code = tDiskColPutValue(pBuilder, pColVal); + // bit map + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + // value + pBuilder->flag |= HAS_VALUE; + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); if (code) return code; } - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal1(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (!COL_VAL_IS_NONE(pColVal)) { - // bit map - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; - code = tRealloc(&pBuilder->pBitMap, nBit); + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + pBuilder->flag |= HAS_NULL; + + return code; +} +static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + pBuilder->flag |= HAS_VALUE; + + memset(pBuilder->pBitMap, 0, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); if (code) return code; - - memset(pBuilder->pBitMap, 0, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - // value - if (COL_VAL_IS_NULL(pColVal)) { - pBuilder->flag |= HAS_NULL; - } else { - pBuilder->flag |= HAS_VALUE; - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) return code; - } - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) return code; - } } - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal2(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL +static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (!COL_VAL_IS_NULL(pColVal)) { - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) goto _exit; + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; - memset(pBuilder->pBitMap, 255, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else { - pBuilder->flag |= HAS_VALUE; + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - memset(pBuilder->pBitMap, 0, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) goto _exit; - } - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - } - } - -_exit: return code; } -static int32_t tDiskColAddVal3(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + pBuilder->flag |= HAS_VALUE; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - } else { - pBuilder->flag |= HAS_VALUE; + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pBuilder->nVal, 2); - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); - } - SET_BIT2(pBitMap, pBuilder->nVal, 2); - - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) goto _exit; - } - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; + SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + code = tDiskColPutValue(pBuilder, &cv); + if (code) return code; } -_exit: - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal4(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE +static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (!COL_VAL_IS_VALUE(pColVal)) { - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag |= HAS_NONE; - } else { - pBuilder->flag |= HAS_NULL; - } + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) goto _exit; - - memset(pBuilder->pBitMap, 255, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - } else { - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - } - -_exit: return code; } -static int32_t tDiskColAddVal5(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NULL(pColVal)) { - pBuilder->flag |= HAS_NULL; + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; - - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); - } - SET_BIT2(pBitMap, pBuilder->nVal, 1); - - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - } else { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - } - } - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - -_exit: return code; } -static int32_t tDiskColAddVal6(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL +static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) { + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (COL_VAL_IS_NONE(pColVal)) { - pBuilder->flag |= HAS_NONE; + pBuilder->flag |= HAS_NONE; - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); - } - SET_BIT2(pBitMap, pBuilder->nVal, 0); + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - } else { - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; - - if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - } else { - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - } - } - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - -_exit: - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t tDiskColAddVal7(SDiskColBuilder *pBuilder, SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NULL; + + int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); + code = tRealloc(&pBuilder->pBitMap, nBit); + if (code) return code; + + memset(pBuilder->pBitMap, 255, nBit); + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pBuilder->nVal, 1); + + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + pBuilder->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + + for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pBuilder->nVal, 0); + + tFree(pBuilder->pBitMap); + pBuilder->pBitMap = pBitMap; + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) goto _exit; + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); - if (COL_VAL_IS_NONE(pColVal)) { - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); - } else if (COL_VAL_IS_NULL(pColVal)) { - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); - } else { - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); - } - code = tDiskColPutValue(pBuilder, pColVal); - if (code) goto _exit; - -_exit: - return code; + return tDiskColPutValue(pBuilder, pColVal); } -static int32_t (*tDiskColAddValImpl[])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { - tDiskColAddVal0, // 0 - tDiskColAddVal1, // HAS_NONE - tDiskColAddVal2, // HAS_NULL - tDiskColAddVal3, // HAS_NULL|HAS_NONE - tDiskColAddVal4, // HAS_VALUE - tDiskColAddVal5, // HAS_VALUE|HAS_NONE - tDiskColAddVal6, // HAS_VALUE|HAS_NULL - tDiskColAddVal7, // HAS_VALUE|HAS_NULL|HAS_NONE +static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); + + return tDiskColPutValue(pBuilder, pColVal); +} +static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + + code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); + if (code) return code; + SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); + + return tDiskColPutValue(pBuilder, pColVal); +} +static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { + {tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0 + {tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE + {tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL + {tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE + {tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE + {tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE + {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL + {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE }; static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; if (pBuilder->calcSma) tDiskColUpdateSma(pBuilder, pColVal); - code = tDiskColAddValImpl[pBuilder->type](pBuilder, pColVal); - if (code) return code; + if (tDiskColAddValImpl[pBuilder->type][pColVal->type]) { + code = tDiskColAddValImpl[pBuilder->type][pColVal->type](pBuilder, pColVal); + if (code) return code; + } pBuilder->nVal++; From 08feade5fe3a9da96268008b9ef9a98a4e857d04 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 14:24:35 +0800 Subject: [PATCH 30/64] more code --- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 18 ++++++++---------- source/dnode/vnode/src/tsdb/tsdbUtil.c | 6 +++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 11f5c398ba..bbf4e3337c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -129,15 +129,6 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { return code; } -extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); -static FORCE_INLINE void tDiskColUpdateSma(SDiskColBuilder *pBuilder, SColVal *pColVal) { - if (!COL_VAL_IS_VALUE(pColVal)) { - pBuilder->sma.numOfNull++; - } else { - tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); - } -} - static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; @@ -422,10 +413,17 @@ static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *p {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE }; +extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { int32_t code = 0; - if (pBuilder->calcSma) tDiskColUpdateSma(pBuilder, pColVal); + if (pBuilder->calcSma) { + if (COL_VAL_IS_VALUE(pColVal)) { + tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); + } else { + pBuilder->sma.numOfNull++; + } + } if (tDiskColAddValImpl[pBuilder->type][pColVal->type]) { code = tDiskColAddValImpl[pBuilder->type][pColVal->type](pBuilder, pColVal); diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 1d7eff9a08..cae113dc4b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1589,10 +1589,10 @@ void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) { for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { tColDataGetValue(pColData, iVal, &cv); - if (!COL_VAL_IS_VALUE(&cv)) { - pColAgg->numOfNull++; - } else { + if (COL_VAL_IS_VALUE(&cv)) { tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet); + } else { + pColAgg->numOfNull++; } } } From 787e166215551d657884a4e063f2fd30ee8e3361 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 17:45:37 +0800 Subject: [PATCH 31/64] refact more code --- source/util/src/tcompression.c | 556 ++++++++++++++++++++++++--------- 1 file changed, 409 insertions(+), 147 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index f1d9b860b8..65e8cabe32 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1159,8 +1159,9 @@ struct SCompressor { int8_t cmprAlg; int8_t autoAlloc; int32_t nVal; - uint8_t *aBuf[2]; - int64_t nBuf[2]; + uint8_t *pBuf; + int32_t nBuf; + uint8_t *aBuf[1]; union { // Timestamp ---- struct { @@ -1193,93 +1194,91 @@ struct SCompressor { // Timestamp ===================================================== static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; - pCmprsor->ts_prev_val = 0; - pCmprsor->ts_prev_delta = 0; - pCmprsor->ts_flag_p = NULL; - pCmprsor->aBuf[0][0] = 1; - pCmprsor->nBuf[0] = 1; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 1; + return code; } -static int32_t tCompSetCopyMode(SCompressor *pCmprsor) { +static int32_t tCompSwitchToCopyMode(SCompressor *pCmprsor) { int32_t code = 0; - if (pCmprsor->nVal) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->nVal); - if (code) return code; - } - pCmprsor->nBuf[1] = 0; + if (pCmprsor->nVal == 0) goto _exit; - int64_t n = 1; - int64_t value; - int64_t delta; - uint64_t vZigzag; - while (n < pCmprsor->nBuf[0]) { - uint8_t aN[2]; - aN[0] = pCmprsor->aBuf[0][n] & 0xf; - aN[1] = pCmprsor->aBuf[0][n] >> 4; - - n++; - - for (int32_t i = 0; i < 2; i++) { - vZigzag = 0; - for (uint8_t j = 0; j < aN[i]; j++) { - vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (8 * j)); - n++; - } - - int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); - if (pCmprsor->nBuf[1] == 0) { - delta = 0; - value = delta_of_delta; - } else { - delta = delta_of_delta + delta; - value = delta + value; - } - - memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &value, sizeof(int64_t)); - pCmprsor->nBuf[1] += sizeof(int64_t); - - if (n >= pCmprsor->nBuf[0]) break; - } - } - - ASSERT(n == pCmprsor->nBuf[0]); - - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[1] + 1); - if (code) return code; - } - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]); - pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1]; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(int64_t) * pCmprsor->nVal + 1))) { + return code; } - pCmprsor->aBuf[0][0] = 0; + int32_t n = 1; + int32_t nBuf = 1; + int64_t value; + int64_t delta; + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t aN[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int32_t i = 0; i < 2; i++) { + uint64_t vZigzag = 0; + for (uint8_t j = 0; j < aN[i]; j++) { + vZigzag |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag); + if (iVal) { + delta = delta_of_delta + delta; + value = delta + value; + } else { + delta = 0; + value = delta_of_delta; + } + + memcpy(pCmprsor->aBuf[0] + nBuf, &value, sizeof(value)); + nBuf += sizeof(int64_t); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + + ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nBuf + 1); + + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 0; return code; } static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; int64_t ts = *(int64_t *)pData; - ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP); ASSERT(nData == 8); - if (pCmprsor->aBuf[0][0] == 1) { + if (pCmprsor->pBuf[0] == 1) { if (pCmprsor->nVal == 0) { pCmprsor->ts_prev_val = ts; pCmprsor->ts_prev_delta = -ts; } if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) { - code = tCompSetCopyMode(pCmprsor); + code = tCompSwitchToCopyMode(pCmprsor); if (code) return code; goto _copy_cmpr; } int64_t delta = ts - pCmprsor->ts_prev_val; if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) { - code = tCompSetCopyMode(pCmprsor); + code = tCompSwitchToCopyMode(pCmprsor); if (code) return code; goto _copy_cmpr; } @@ -1290,37 +1289,35 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t pCmprsor->ts_prev_delta = delta; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) { + return code; } - pCmprsor->ts_flag_p = pCmprsor->aBuf[0] + pCmprsor->nBuf[0]; - pCmprsor->nBuf[0]++; + pCmprsor->ts_flag_p = pCmprsor->pBuf + pCmprsor->nBuf; + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0] = 0; while (vZigzag) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff); + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0]++; vZigzag >>= 8; } } else { while (vZigzag) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff); + pCmprsor->nBuf++; pCmprsor->ts_flag_p[0] += 0x10; vZigzag >>= 8; } } } else { _copy_cmpr: - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(ts)); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(ts)))) { + return code; } - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts)); - pCmprsor->nBuf[0] += sizeof(ts); + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, &ts, sizeof(ts)); + pCmprsor->nBuf += sizeof(ts); } pCmprsor->nVal++; @@ -1329,7 +1326,34 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= sizeof(int64_t) * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 1) { + code = tCompSwitchToCopyMode(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } @@ -1344,12 +1368,24 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; + pCmprsor->i_prev = 0; pCmprsor->i_selector = 0; pCmprsor->i_start = 0; pCmprsor->i_end = 0; - pCmprsor->aBuf[0][0] = 0; - pCmprsor->nBuf[0] = 1; + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + + return code; +} + +static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + // ASSERT(0); return code; } @@ -1363,26 +1399,18 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) switch (pCmprsor->type) { case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: val = *(int8_t *)pData; break; case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: val = *(int16_t *)pData; break; case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: val = *(int32_t *)pData; break; case TSDB_DATA_TYPE_BIGINT: - val = *(int64_t *)pData; - break; - case TSDB_DATA_TYPE_UTINYINT: - val = *(uint8_t *)pData; - break; - case TSDB_DATA_TYPE_USMALLINT: - val = *(uint16_t *)pData; - break; - case TSDB_DATA_TYPE_UINT: - val = *(uint32_t *)pData; - break; case TSDB_DATA_TYPE_UBIGINT: val = *(int64_t *)pData; break; @@ -1392,14 +1420,16 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { - // TODO + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; goto _copy_cmpr; } int64_t diff = val - pCmprsor->i_prev; uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff); if (vZigzag >= SIMPLE8B_MAX) { - // TODO + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; goto _copy_cmpr; } @@ -1424,12 +1454,12 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t)); + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)); if (code) return code; } - uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]); - pCmprsor->nBuf[0] += sizeof(uint64_t); + uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); + pCmprsor->nBuf += sizeof(uint64_t); bp[0] = pCmprsor->i_selector; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; for (int32_t iVal = 0; iVal < nEle; iVal++) { @@ -1448,11 +1478,11 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } } else { _copy_cmpr: - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData); if (code) return code; - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); - pCmprsor->nBuf[0] += nData; + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData); + pCmprsor->nBuf += nData; } pCmprsor->nVal++; @@ -1461,20 +1491,107 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 0) { + code = tCompIntSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } // Float ===================================================== static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; + pCmprsor->f_prev = 0; pCmprsor->f_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; - pCmprsor->nBuf[0] = 1; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + return code; } +static int32_t tCompFloatSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(float) * pCmprsor->nVal + 1))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + union { + float f; + uint32_t u; + } val = {.u = 0}; + + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int8_t i = 0; i < 2; i++) { + uint8_t flag = flags[i]; + + uint32_t diff = 0; + int8_t nBytes = (flag & 0x7) + 1; + for (int j = 0; j < nBytes; j++) { + diff |= (((uint32_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + if (flag & 0x8) { + diff <<= (32 - nBytes * 8); + } + + val.u ^= diff; + + memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val)); + nBuf += sizeof(val); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 1; + return code; +} static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1507,13 +1624,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat if (nBytes == 0) nBytes++; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 9))) { + return code; } - pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; - pCmprsor->nBuf[0]++; + pCmprsor->f_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf]; + pCmprsor->nBuf++; if (clz < ctz) { pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1)); @@ -1528,8 +1644,8 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat } } for (; nBytes; nBytes--) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff); + pCmprsor->nBuf++; diff >>= BITS_PER_BYTE; } pCmprsor->nVal++; @@ -1539,20 +1655,107 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= sizeof(float) * pCmprsor->nVal + 1) { + code = tCompFloatSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } // Double ===================================================== static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; + pCmprsor->d_prev = 0; pCmprsor->d_flag_p = NULL; - pCmprsor->aBuf[0][0] = 0; - pCmprsor->nBuf[0] = 1; + + pCmprsor->nBuf = 1; + + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); + if (code) return code; + + pCmprsor->pBuf[0] = 0; + return code; } +static int32_t tCompDoubleSwitchToCopy(SCompressor *pCmprsor) { + int32_t code = 0; + + if (pCmprsor->nVal == 0) goto _exit; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(double) * pCmprsor->nVal + 1))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + union { + double f; + uint64_t u; + } val = {.u = 0}; + + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)}; + + n++; + + for (int8_t i = 0; i < 2; i++) { + uint8_t flag = flags[i]; + + uint64_t diff = 0; + int8_t nBytes = (flag & 0x7) + 1; + for (int j = 0; j < nBytes; j++) { + diff |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j)); + n++; + } + + if (flag & 0x8) { + diff <<= (64 - nBytes * 8); + } + + val.u ^= diff; + + memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val)); + nBuf += sizeof(val); + + iVal++; + if (iVal >= pCmprsor->nVal) break; + } + } + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = nBuf; + +_exit: + pCmprsor->pBuf[0] = 1; + return code; +} static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; @@ -1585,13 +1788,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa if (nBytes == 0) nBytes++; if ((pCmprsor->nVal & 0x1) == 0) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) { + return code; } - pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]]; - pCmprsor->nBuf[0]++; + pCmprsor->d_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf]; + pCmprsor->nBuf++; if (clz < ctz) { pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1)); @@ -1606,8 +1808,8 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa } } for (; nBytes; nBytes--) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff); - pCmprsor->nBuf[0]++; + pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff); + pCmprsor->nBuf++; diff >>= BITS_PER_BYTE; } pCmprsor->nVal++; @@ -1617,28 +1819,53 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf >= sizeof(double) * pCmprsor->nVal + 1) { + code = tCompDoubleSwitchToCopy(pCmprsor); + if (code) return code; + } + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } // Binary ===================================================== static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { - int32_t code = 0; - pCmprsor->nBuf[0] = 0; - return code; + pCmprsor->nBuf = 1; + return 0; } static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) { int32_t code = 0; if (nData) { - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData))) { + return code; } - memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData); - pCmprsor->nBuf[0] += nData; + memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData); + pCmprsor->nBuf += nData; } pCmprsor->nVal++; @@ -1647,7 +1874,25 @@ static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nDa static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->nBuf == 1) return code; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf))) { + return code; + } + + int32_t szComp = + LZ4_compress_default(pCmprsor->pBuf + 1, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf - 1, pCmprsor->nBuf - 1); + if (szComp && szComp < pCmprsor->nBuf - 1) { + pCmprsor->aBuf[0][0] = 1; + *ppData = pCmprsor->aBuf[0]; + *nData = szComp + 1; + } else { + pCmprsor->pBuf[0] = 0; + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } + return code; } @@ -1655,9 +1900,8 @@ static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000}; static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { - int32_t code = 0; - pCmprsor->nBuf[0] = 0; - return code; + pCmprsor->nBuf = 0; + return 0; } static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) { @@ -1665,19 +1909,18 @@ static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData bool vBool = *(int8_t *)pData; - int32_t mod4 = pCmprsor->nVal & 3; + int32_t mod4 = (pCmprsor->nVal & 3); if (mod4 == 0) { - pCmprsor->nBuf[0]++; + pCmprsor->nBuf++; - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf))) { + return code; } - pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0; + pCmprsor->pBuf[pCmprsor->nBuf - 1] = 0; } if (vBool) { - pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4]; + pCmprsor->pBuf[pCmprsor->nBuf - 1] |= BOOL_CMPR_TABLE[mod4]; } pCmprsor->nVal++; @@ -1686,7 +1929,30 @@ static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - // TODO + + if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *ppData = pCmprsor->aBuf[0]; + + int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (szComp && szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *nData = szComp + 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *nData = pCmprsor->nBuf + 1; + } + } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { + *ppData = pCmprsor->pBuf; + *nData = pCmprsor->nBuf; + } else { + ASSERT(0); + } + return code; } @@ -1697,23 +1963,17 @@ int32_t tCompressorCreate(SCompressor **ppCmprsor) { *ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor)); if ((*ppCmprsor) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; + return code; } - code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024); - if (code) { - taosMemoryFree(*ppCmprsor); - *ppCmprsor = NULL; - goto _exit; - } - -_exit: return code; } int32_t tCompressorDestroy(SCompressor *pCmprsor) { int32_t code = 0; + tFree(pCmprsor->pBuf); + int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]); for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) { tFree(pCmprsor->aBuf[iBuf]); @@ -1745,6 +2005,8 @@ int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nDa *ppData = NULL; *nData = 0; + if (pCmprsor->nVal == 0) return code; + if (DATA_TYPE_INFO[pCmprsor->type].endFn) { return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppData, nData); } From 4fdb37e887d005cc7c605b51d0bc6f8909532425 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 26 Sep 2022 17:56:47 +0800 Subject: [PATCH 32/64] more code --- source/util/src/tcompression.c | 104 +++++++++++---------------------- 1 file changed, 34 insertions(+), 70 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 65e8cabe32..ef0a45f3a5 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1191,6 +1191,26 @@ struct SCompressor { }; }; +static int32_t tTwoStageComp(SCompressor *pCmprsor, int32_t *szComp) { + int32_t code = 0; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { + return code; + } + + *szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); + if (*szComp && *szComp < pCmprsor->nBuf) { + pCmprsor->aBuf[0][0] = 1; + *szComp += 1; + } else { + pCmprsor->aBuf[0][0] = 0; + memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); + *szComp = pCmprsor->nBuf + 1; + } + + return code; +} + // Timestamp ===================================================== static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; @@ -1205,7 +1225,7 @@ static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cm return code; } -static int32_t tCompSwitchToCopyMode(SCompressor *pCmprsor) { +static int32_t tCompTSSwitchToCopy(SCompressor *pCmprsor) { int32_t code = 0; if (pCmprsor->nVal == 0) goto _exit; @@ -1271,14 +1291,14 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t } if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) { - code = tCompSwitchToCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; goto _copy_cmpr; } int64_t delta = ts - pCmprsor->ts_prev_val; if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) { - code = tCompSwitchToCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; goto _copy_cmpr; } @@ -1328,25 +1348,14 @@ static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t code = 0; if (pCmprsor->nBuf >= sizeof(int64_t) * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 1) { - code = tCompSwitchToCopyMode(pCmprsor); + code = tCompTSSwitchToCopy(pCmprsor); if (code) return code; } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1498,20 +1507,9 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_ } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1662,20 +1660,9 @@ static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int3 } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1826,20 +1813,9 @@ static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int } if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; @@ -1931,21 +1907,9 @@ static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32 int32_t code = 0; if (pCmprsor->cmprAlg == TWO_STAGE_COMP) { - if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) { - return code; - } - + code = tTwoStageComp(pCmprsor, nData); + if (code) return code; *ppData = pCmprsor->aBuf[0]; - - int32_t szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf); - if (szComp && szComp < pCmprsor->nBuf) { - pCmprsor->aBuf[0][0] = 1; - *nData = szComp + 1; - } else { - pCmprsor->aBuf[0][0] = 0; - memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf); - *nData = pCmprsor->nBuf + 1; - } } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) { *ppData = pCmprsor->pBuf; *nData = pCmprsor->nBuf; From 24e899f88b0a43e9cd618ad9126e2ee04e6b7b8d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 15:40:06 +0800 Subject: [PATCH 33/64] more code --- source/util/src/tcompression.c | 49 ++++++++++------------------------ 1 file changed, 14 insertions(+), 35 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index ef0a45f3a5..7da55a7bdc 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1032,12 +1032,7 @@ static struct { int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); } DATA_TYPE_INFO[] = { - {.type = TSDB_DATA_TYPE_NULL, - .bytes = 0, - .isVarLen = 0, - .startFn = NULL, - .cmprFn = NULL, - .endFn = NULL}, // TSDB_DATA_TYPE_NULL + {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .startFn = NULL, .cmprFn = NULL, .endFn = NULL}, {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, @@ -1394,7 +1389,13 @@ static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { int32_t code = 0; - // ASSERT(0); + + if (pCmprsor->nVal == 0) goto _exit; + + // todo + +_exit: + pCmprsor->pBuf[0] = 1; return code; } @@ -1404,29 +1405,8 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); if (pCmprsor->aBuf[0][0] == 0) { - int64_t val; - - switch (pCmprsor->type) { - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - val = *(int8_t *)pData; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - val = *(int16_t *)pData; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - val = *(int32_t *)pData; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - val = *(int64_t *)pData; - break; - default: - ASSERT(0); - break; - } + int64_t val = 0; + memcpy(&val, pData, nData); // little-endian only if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { code = tCompIntSwitchToCopy(pCmprsor); @@ -1445,16 +1425,16 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0; pCmprsor->i_prev = val; - while (1) { + for (;;) { int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) { pCmprsor->i_selector = BIT_TO_SELECTOR[nBit]; } - pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag; pCmprsor->i_aBitN[pCmprsor->i_end] = nBit; + pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; break; } else { while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { @@ -1462,9 +1442,8 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) } nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; - if (pCmprsor->autoAlloc) { - code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)); - if (code) return code; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { + return code; } uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); From 3be4b1d4dfdf1061d0192216ba40a51fd506a3a4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 16:16:02 +0800 Subject: [PATCH 34/64] finish stream compression --- source/util/src/tcompression.c | 60 +++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 7da55a7bdc..133d1b29b7 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1392,7 +1392,40 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { if (pCmprsor->nVal == 0) goto _exit; - // todo + int32_t size = DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1; + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], size))) { + return code; + } + + int32_t n = 1; + int32_t nBuf = 1; + int64_t vPrev = 0; + for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + uint64_t b; + memcpy(&b, pCmprsor->pBuf + n, sizeof(b)); + n += sizeof(b); + + int32_t i_selector = (b & 0xf); + int32_t nEle = SELECTOR_TO_ELEMS[i_selector]; + uint8_t bits = BIT_PER_INTEGER[i_selector]; + for (int32_t iEle = 0; iEle < nEle; iEle++) { + uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); + int64_t diff = ZIGZAG_DECODE(int64_t, vZigzag); + vPrev = diff + vPrev; + + memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); + nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; + } + iVal += nEle; + ASSERT(iVal <= pCmprsor->nVal); + } + + ASSERT(n == pCmprsor->nBuf && nBuf == size); + + uint8_t *pBuf = pCmprsor->pBuf; + pCmprsor->pBuf = pCmprsor->aBuf[0]; + pCmprsor->aBuf[0] = pBuf; + pCmprsor->nBuf = size; _exit: pCmprsor->pBuf[0] = 1; @@ -1480,6 +1513,31 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; + for (;;) { + int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; + if (nEle == 0) break; + + while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + pCmprsor->i_selector++; + } + nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + + if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { + return code; + } + + uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf); + pCmprsor->nBuf += sizeof(uint64_t); + bp[0] = pCmprsor->i_selector; + uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; + for (int32_t iVal = 0; iVal < nEle; iVal++) { + bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); + pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + } + + pCmprsor->i_selector = 0; + } + if (pCmprsor->nBuf >= DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 0) { code = tCompIntSwitchToCopy(pCmprsor); if (code) return code; From dc0d687a06dca97045f9c69ffee258138a06a446 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 16:26:16 +0800 Subject: [PATCH 35/64] make it compile --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a76ce76b85..09b790ee81 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -440,7 +440,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { pIter->iSttBlk = 0; SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); pIter->iRow = 0; pIter->r.suid = pIter->bData.suid; @@ -1370,8 +1370,9 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; - code = tBlockDataInit(pBDatal, &tid, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _exit; + code = + tDiskDataBuilderInit(pCommitter->dWriter.pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); + TSDB_CHECK_CODE(code, lino, _exit); } _exit: @@ -1519,11 +1520,11 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { // impl code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0); - if (code) goto _err; + TSDB_CHECK_CODE(code, lino, _exit); /* merge with data in .data file */ code = tsdbMergeTableData(pCommitter, id); From a4d16f1c0029c245446fba6009b14baca9af1c0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 17:36:02 +0800 Subject: [PATCH 36/64] more code --- include/util/tcompression.h | 2 +- source/dnode/vnode/src/inc/tsdb.h | 6 +- source/dnode/vnode/src/tsdb/tsdbCommit.c | 7 +- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 101 +++++++++++++-------- source/util/src/tcompression.c | 15 ++- 5 files changed, 80 insertions(+), 51 deletions(-) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 8f3d9634ec..84c2922c9e 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -131,7 +131,7 @@ typedef struct SCompressor SCompressor; int32_t tCompressorCreate(SCompressor **ppCmprsor); int32_t tCompressorDestroy(SCompressor *pCmprsor); int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); -int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin); int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 264746e8f2..ca16b87e17 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -326,9 +326,9 @@ int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder); int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, uint8_t calcSma); -int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData); -int32_t tDiskDataDestroy(SDiskData *pDiskData); +int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); +int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData); // structs ======================= struct STsdbFS { diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 09b790ee81..7bc335d034 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -665,7 +665,7 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde .nRow = pBuilder->nRow}; // gnrt - code = tGnrtDiskData(pBuilder, &pBuilder->dd); + // code = tGnrtDiskData(pBuilder, &pBuilder->dd); TSDB_CHECK_CODE(code, lino, _exit); // write @@ -1369,7 +1369,6 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; code = tDiskDataBuilderInit(pCommitter->dWriter.pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); @@ -1396,7 +1395,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); - code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); + code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); TSDB_CHECK_CODE(code, lino, _exit); if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { @@ -1435,7 +1434,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { pTSchema = pCommitter->skmRow.pTSchema; } - code = tDiskDataBuilderAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); + code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index bbf4e3337c..8cba73d353 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -31,9 +31,16 @@ struct SDiskColBuilder { SColumnDataAgg sma; uint8_t minSet; uint8_t maxSet; - uint8_t *aBuf[1]; + uint8_t *aBuf[2]; }; +// SDiskData ================================================ +static int32_t tDiskDataDestroy(SDiskData *pDiskData) { + int32_t code = 0; + pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); + return code; +} + // SDiskColBuilder ================================================ #define tDiskColBuilderCreate() \ (SDiskColBuilder) { 0 } @@ -91,7 +98,7 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { .type = pBuilder->type, .smaOn = pBuilder->calcSma, .flag = pBuilder->flag, - .szOrigin = 0, // todo + .szOrigin = 0, .szBitmap = 0, .szOffset = 0, .szValue = 0, @@ -109,20 +116,27 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { nBit = BIT1_SIZE(pBuilder->nVal); } - pDiskCol->bCol.szBitmap = tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], 0, pBuilder->cmprAlg, - NULL, 0); // todo: alloc + code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES); + if (code) return code; + + code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); + if (code) return code; + + pDiskCol->bCol.szBitmap = + tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES, + pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); pDiskCol->pBit = pBuilder->aBuf[0]; } // OFFSET if (IS_VAR_DATA_TYPE(pBuilder->type)) { - code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset); + code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL); if (code) return code; } // VALUE if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { - code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue); + code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin); if (code) return code; } @@ -425,8 +439,8 @@ static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { } } - if (tDiskColAddValImpl[pBuilder->type][pColVal->type]) { - code = tDiskColAddValImpl[pBuilder->type][pColVal->type](pBuilder, pColVal); + if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) { + code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal); if (code) return code; } @@ -465,6 +479,7 @@ void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { tFree(pBuilder->aBuf[iBuf]); } + tDiskDataDestroy(&pBuilder->dd); taosMemoryFree(pBuilder); return NULL; @@ -474,6 +489,8 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB uint8_t calcSma) { int32_t code = 0; + ASSERT(pId->suid || pId->uid); + pBuilder->suid = pId->suid; pBuilder->uid = pId->uid; pBuilder->nRow = 0; @@ -512,9 +529,9 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB } } - SDiskColBuilder *pDiskColBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); + SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); - code = tDiskColBuilderInit(pDiskColBuilder, pTColumn->colId, pTColumn->type, cmprAlg, + code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg, (calcSma && (pTColumn->flags & COL_SMA_ON))); if (code) return code; @@ -524,14 +541,22 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB return code; } -int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { +int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) { + int32_t code = 0; + pBuilder->suid = 0; + pBuilder->uid = 0; + return code; +} + +int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { int32_t code = 0; + ASSERT(pBuilder->suid || pBuilder->uid); ASSERT(pId->suid == pBuilder->suid); // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { - ASSERT(!pBuilder->calcSma); + ASSERT(pBuilder->suid); for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); if (code) return code; @@ -564,14 +589,13 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche pColVal = tRowIterNext(&iter); } - if (pColVal == NULL || pColVal->cid > pDCBuilder->cid) { - SColVal cv = COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type); - code = tDiskColAddVal(pDCBuilder, &cv); - if (code) return code; - } else { + if (pColVal && pColVal->cid == pDCBuilder->cid) { code = tDiskColAddVal(pDCBuilder, pColVal); if (code) return code; pColVal = tRowIterNext(&iter); + } else { + code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); + if (code) return code; } } pBuilder->nRow++; @@ -579,11 +603,14 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche return code; } -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData) { int32_t code = 0; ASSERT(pBuilder->nRow); + *ppDiskData = NULL; + + SDiskData *pDiskData = &pBuilder->dd; // reset SDiskData pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, .fmtVer = 0, @@ -598,6 +625,22 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { pDiskData->pUid = NULL; pDiskData->pVer = NULL; pDiskData->pKey = NULL; + + // UID + if (pBuilder->uid == 0) { + code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL); + if (code) return code; + } + + // VERSION + code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL); + if (code) return code; + + // TSKEY + code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL); + if (code) return code; + + // aDiskCol if (pDiskData->aDiskCol) { taosArrayClear(pDiskData->aDiskCol); } else { @@ -608,20 +651,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { } } - // UID - if (pBuilder->uid == 0) { - code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid); - if (code) return code; - } - - // VERSION - code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer); - if (code) return code; - - // TSKEY - code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey); - if (code) return code; - int32_t offset = 0; for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); @@ -644,12 +673,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) { pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); } - return code; -} - -// SDiskData ================================================ -int32_t tDiskDataDestroy(SDiskData *pDiskData) { - int32_t code = 0; - pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); + *ppDiskData = pDiskData; return code; } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 133d1b29b7..8c38b4f20d 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -2000,16 +2000,23 @@ int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { return code; } -int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { +int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin) { int32_t code = 0; - *ppData = NULL; - *nData = 0; + *ppOut = NULL; + *nOut = 0; + if (nOrigin) { + if (DATA_TYPE_INFO[pCmprsor->type].isVarLen) { + *nOrigin = pCmprsor->nBuf - 1; + } else { + *nOrigin = pCmprsor->nVal * DATA_TYPE_INFO[pCmprsor->type].bytes; + } + } if (pCmprsor->nVal == 0) return code; if (DATA_TYPE_INFO[pCmprsor->type].endFn) { - return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppData, nData); + return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppOut, nOut); } return code; From a53575a1524a1f777a2bb180acd339312e76b917 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 17:57:57 +0800 Subject: [PATCH 37/64] more code --- source/dnode/vnode/src/inc/tsdb.h | 13 +++++++++++- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 24 +++++++++++++++++----- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ca16b87e17..a928cc725d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -328,7 +328,8 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB uint8_t calcSma); int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo *pBlkInfo); +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo); // structs ======================= struct STsdbFS { @@ -450,6 +451,15 @@ struct SSmaInfo { int32_t size; }; +typedef struct { + int64_t minUid; + int64_t maxUid; + TSDBKEY minKey; + TSDBKEY maxKey; + int64_t minVer; + int64_t maxVer; +} SBlkInfo; + struct SDataBlk { TSDBKEY minKey; TSDBKEY maxKey; @@ -702,6 +712,7 @@ struct SDiskDataBuilder { SArray *aBuilder; // SArray uint8_t *aBuf[2]; SDiskData dd; + SBlkInfo bi; }; int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index 8cba73d353..d3ea4f1dda 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -496,6 +496,12 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB pBuilder->nRow = 0; pBuilder->cmprAlg = cmprAlg; pBuilder->calcSma = calcSma; + pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX, + .maxUid = INT64_MIN, + .minKey = TSDBKEY_MAX, + .maxKey = TSDBKEY_MIN, + .minVer = VERSION_MAX, + .maxVer = VERSION_MIN}; if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); @@ -554,6 +560,8 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS ASSERT(pBuilder->suid || pBuilder->uid); ASSERT(pId->suid == pBuilder->suid); + TSDBKEY key = TSDBROW_KEY(pRow); + // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { ASSERT(pBuilder->suid); @@ -567,16 +575,20 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); if (code) return code; } + if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid; + if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid; // version - int64_t version = TSDBROW_VERSION(pRow); - code = tCompress(pBuilder->pVerC, &version, sizeof(int64_t)); + code = tCompress(pBuilder->pVerC, &key.version, sizeof(int64_t)); if (code) return code; + if (pBuilder->bi.minVer > key.version) pBuilder->bi.minVer = key.version; + if (pBuilder->bi.maxVer < key.version) pBuilder->bi.maxVer = key.version; // TSKEY - TSKEY ts = TSDBROW_TS(pRow); - code = tCompress(pBuilder->pKeyC, &ts, sizeof(int64_t)); + code = tCompress(pBuilder->pKeyC, &key.ts, sizeof(int64_t)); if (code) return code; + if (tsdbKeyCmprFn(&pBuilder->bi.minKey, &key) > 0) pBuilder->bi.minKey = key; + if (tsdbKeyCmprFn(&pBuilder->bi.maxKey, &key) < 0) pBuilder->bi.maxKey = key; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); @@ -603,12 +615,13 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS return code; } -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData) { +int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) { int32_t code = 0; ASSERT(pBuilder->nRow); *ppDiskData = NULL; + *ppBlkInfo = NULL; SDiskData *pDiskData = &pBuilder->dd; // reset SDiskData @@ -674,5 +687,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData) } *ppDiskData = pDiskData; + *ppBlkInfo = &pBuilder->bi; return code; } From 8bd8ff152390f1d7100c316342f2ca339076eb00 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 18:59:29 +0800 Subject: [PATCH 38/64] more code --- source/dnode/vnode/src/inc/tsdb.h | 15 ++- source/dnode/vnode/src/tsdb/tsdbCommit.c | 27 ++--- source/dnode/vnode/src/tsdb/tsdbDiskData.c | 25 +++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 106 ++++++++++++------ source/dnode/vnode/src/tsdb/tsdbUtil.c | 5 +- 5 files changed, 111 insertions(+), 67 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a928cc725d..2a627af324 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -74,6 +74,7 @@ typedef struct SLDataIter SLDataIter; typedef struct SDiskCol SDiskCol; typedef struct SDiskData SDiskData; typedef struct SDiskDataBuilder SDiskDataBuilder; +typedef struct SBlkInfo SBlkInfo; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 @@ -173,7 +174,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut int32_t aBufN[]); int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr -int32_t tPutDiskDataHdr(uint8_t *p, void *ph); +int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); // SDelIdx int32_t tPutDelIdx(uint8_t *p, void *ph); @@ -270,6 +271,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo); int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); // SDataFReader @@ -328,7 +330,6 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB uint8_t calcSma); int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo *pBlkInfo); int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo); // structs ======================= @@ -451,14 +452,16 @@ struct SSmaInfo { int32_t size; }; -typedef struct { +struct SBlkInfo { int64_t minUid; int64_t maxUid; - TSDBKEY minKey; - TSDBKEY maxKey; + TSKEY minKey; + TSKEY maxKey; int64_t minVer; int64_t maxVer; -} SBlkInfo; + TSDBKEY minTKey; + TSDBKEY maxTKey; +}; struct SDataBlk { TSDBKEY minKey; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 7bc335d034..1a0534df5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -542,6 +542,7 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); + tDiskDataBuilderClear(pCommitter->dWriter.pBuilder); // open iter code = tsdbOpenCommitIter(pCommitter); @@ -655,21 +656,21 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde if (pBuilder->nRow == 0) return code; - SSttBlk sttBlk = {.suid = pBuilder->suid, - .minUid = 0, // todo - .maxUid = 0, // todo - .minKey = 0, // todo - .maxKey = 0, // todo - .minVer = 0, // todo - .maxVer = 0, // todo - .nRow = pBuilder->nRow}; - // gnrt - // code = tGnrtDiskData(pBuilder, &pBuilder->dd); + const SDiskData *pDiskData; + const SBlkInfo *pBlkInfo; + code = tGnrtDiskData(pBuilder, &pDiskData, &pBlkInfo); TSDB_CHECK_CODE(code, lino, _exit); + SSttBlk sttBlk = {.suid = pBuilder->suid, + .minUid = pBlkInfo->minUid, + .maxUid = pBlkInfo->maxUid, + .minKey = pBlkInfo->minKey, + .maxKey = pBlkInfo->maxKey, + .minVer = pBlkInfo->minVer, + .maxVer = pBlkInfo->maxVer}; // write - // code = tsdbWriteDiskData(pWriter, &pBuilder->dd); + code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL); TSDB_CHECK_CODE(code, lino, _exit); // push @@ -678,8 +679,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde TSDB_CHECK_CODE(code, lino, _exit); } - // clear (todo) - // tDiskDataBuilderClear(pBuilder); + // clear + tDiskDataBuilderClear(pBuilder); _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c index d3ea4f1dda..43be51a694 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -498,10 +498,12 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB pBuilder->calcSma = calcSma; pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX, .maxUid = INT64_MIN, - .minKey = TSDBKEY_MAX, - .maxKey = TSDBKEY_MIN, + .minKey = TSKEY_MAX, + .maxKey = TSKEY_MIN, .minVer = VERSION_MAX, - .maxVer = VERSION_MIN}; + .maxVer = VERSION_MIN, + .minTKey = TSDBKEY_MAX, + .maxTKey = TSDBKEY_MIN}; if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); @@ -551,6 +553,7 @@ int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) { int32_t code = 0; pBuilder->suid = 0; pBuilder->uid = 0; + pBuilder->nRow = 0; return code; } @@ -560,7 +563,9 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS ASSERT(pBuilder->suid || pBuilder->uid); ASSERT(pId->suid == pBuilder->suid); - TSDBKEY key = TSDBROW_KEY(pRow); + TSDBKEY kRow = TSDBROW_KEY(pRow); + if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow; + if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow; // uid if (pBuilder->uid && pBuilder->uid != pId->uid) { @@ -579,16 +584,16 @@ int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTS if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid; // version - code = tCompress(pBuilder->pVerC, &key.version, sizeof(int64_t)); + code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t)); if (code) return code; - if (pBuilder->bi.minVer > key.version) pBuilder->bi.minVer = key.version; - if (pBuilder->bi.maxVer < key.version) pBuilder->bi.maxVer = key.version; + if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version; + if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version; // TSKEY - code = tCompress(pBuilder->pKeyC, &key.ts, sizeof(int64_t)); + code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t)); if (code) return code; - if (tsdbKeyCmprFn(&pBuilder->bi.minKey, &key) > 0) pBuilder->bi.minKey = key; - if (tsdbKeyCmprFn(&pBuilder->bi.maxKey, &key) < 0) pBuilder->bi.maxKey = key; + if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; + if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; SRowIter iter = {0}; tRowIterInit(&iter, pRow, pTSchema); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 4352595d88..00b6830fbc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -522,9 +522,6 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, // write if (pSmaInfo->size) { - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size); - if (code) goto _err; - code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); if (code) goto _err; @@ -607,12 +604,20 @@ _err: return code; } -int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInfo *pBlkInfo) { +int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) { int32_t code = 0; int32_t lino = 0; - STsdbFD *pFD = pWriter->pDataFD; // todo - int64_t offset = pWriter->fData.size; + STsdbFD *pFD = NULL; + if (pSmaInfo) { + pFD = pWriter->pDataFD; + pBlkInfo->offset = pWriter->fData.size; + } else { + pFD = pWriter->pSttFD; + pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; + } + pBlkInfo->szBlock = 0; + pBlkInfo->szKey = 0; // hdr int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr); @@ -621,26 +626,30 @@ int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInf tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr); - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], n); + code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n); TSDB_CHECK_CODE(code, lino, _exit); - offset += n; + pBlkInfo->szKey += n; + pBlkInfo->szBlock += n; // uid + ver + key - if (pDiskData->hdr.szUid) { - code = tsdbWriteFile(pFD, offset, pDiskData->pUid, pDiskData->hdr.szUid); + if (pDiskData->pUid) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szUid; + pBlkInfo->szKey += pDiskData->hdr.szUid; + pBlkInfo->szBlock += pDiskData->hdr.szUid; } - code = tsdbWriteFile(pFD, offset, pDiskData->pVer, pDiskData->hdr.szVer); + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szVer; + pBlkInfo->szKey += pDiskData->hdr.szVer; + pBlkInfo->szBlock += pDiskData->hdr.szVer; - code = tsdbWriteFile(pFD, offset, pDiskData->pKey, pDiskData->hdr.szKey); + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szKey; + pBlkInfo->szKey += pDiskData->hdr.szKey; + pBlkInfo->szBlock += pDiskData->hdr.szKey; - // SBlockCol + // aBlockCol if (pDiskData->hdr.szBlkCol) { code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol); TSDB_CHECK_CODE(code, lino, _exit); @@ -648,48 +657,75 @@ int32_t tsdbWriteDiskData(SDataFWriter *pWriter, SDiskData *pDiskData, SBlockInf n = 0; for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - - n += tPutBlockCol(pWriter->aBuf[0] + n, &pDiskCol->bCol); + n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol); } - ASSERT(n == pDiskData->hdr.szBlkCol); - code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskData->hdr.szBlkCol; + pBlkInfo->szBlock += pDiskData->hdr.szBlkCol; } - // pData + // aDiskCol for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); - if (pDiskCol->bCol.flag == HAS_NULL) continue; - - if (pDiskCol->bCol.szBitmap) { - code = tsdbWriteFile(pFD, offset, pDiskCol->pBit, pDiskCol->bCol.szBitmap); + if (pDiskCol->pBit) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskCol->bCol.szBitmap; + + pBlkInfo->szBlock += pDiskCol->bCol.szBitmap; } - if (pDiskCol->bCol.szOffset) { - code = tsdbWriteFile(pFD, offset, pDiskCol->pOff, pDiskCol->bCol.szOffset); + if (pDiskCol->pOff) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskCol->bCol.szOffset; + + pBlkInfo->szBlock += pDiskCol->bCol.szOffset; } - if (pDiskCol->bCol.szValue) { - code = tsdbWriteFile(pFD, offset, pDiskCol->pVal, pDiskCol->bCol.szValue); + if (pDiskCol->pVal) { + code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue); TSDB_CHECK_CODE(code, lino, _exit); - offset += pDiskCol->bCol.szValue; + + pBlkInfo->szBlock += pDiskCol->bCol.szValue; } } + if (pSmaInfo) { + pWriter->fData.size += pBlkInfo->szBlock; + } else { + pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; + goto _exit; + } + + pSmaInfo->offset = 0; + pSmaInfo->size = 0; + for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) { + SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol); + + if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue; + if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue; + if (!pDiskCol->bCol.smaOn) continue; + + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg)); + TSDB_CHECK_CODE(code, lino, _exit); + pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg); + } + + if (pSmaInfo->size) { + pSmaInfo->offset = pWriter->fSma.size; + + code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size); + TSDB_CHECK_CODE(code, lino, _exit); + + pWriter->fSma.size += pSmaInfo->size; + } + _exit: if (code) { tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code)); - } else { - tsdbDebug("vgId:%d %s", TD_VID(pWriter->pTsdb->pVnode), __func__); } return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 8b95d15407..47c836d9c1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -1454,9 +1454,8 @@ _exit: } // SDiskDataHdr ============================== -int32_t tPutDiskDataHdr(uint8_t *p, void *ph) { - int32_t n = 0; - SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; +int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) { + int32_t n = 0; n += tPutU32(p ? p + n : p, pHdr->delimiter); n += tPutU32v(p ? p + n : p, pHdr->fmtVer); From 18f3bd53c577c8ce50d5daa2fd90872d602d1652 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 27 Sep 2022 23:34:06 +0800 Subject: [PATCH 39/64] fix code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 1a0534df5c..99603500ae 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1364,14 +1364,15 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); + + tDiskDataBuilderClear(pBuilder); } } if (!pBuilder->suid && !pBuilder->uid) { ASSERT(pCommitter->skmTable.suid == id.suid); ASSERT(pCommitter->skmTable.uid == id.uid); - code = - tDiskDataBuilderInit(pCommitter->dWriter.pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); + code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); } @@ -1388,8 +1389,8 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { int32_t lino = 0; SBlockData *pBData = &pCommitter->dWriter.bData; + TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; - TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); @@ -1402,6 +1403,9 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); } } @@ -1425,7 +1429,8 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pRowInfo == NULL) goto _exit; if (pCommitter->toLastOnly) { - // init the data if need + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); while (pRowInfo) { STSchema *pTSchema = NULL; @@ -1449,6 +1454,9 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); + TSDB_CHECK_CODE(code, lino, _exit); } } } else { From 27b6290f6bb5e50294abc85a452d681f93b1f4b6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 09:56:28 +0800 Subject: [PATCH 40/64] fix compression bug --- source/util/src/tcompression.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 8c38b4f20d..003843c5bb 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1379,7 +1379,7 @@ static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) pCmprsor->i_end = 0; pCmprsor->nBuf = 1; - code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf); + code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); if (code) return code; pCmprsor->pBuf[0] = 0; @@ -1437,7 +1437,7 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); - if (pCmprsor->aBuf[0][0] == 0) { + if (pCmprsor->pBuf[0] == 0) { int64_t val = 0; memcpy(&val, pData, nData); // little-endian only From df84baad12b559810083fadc111385299ad1a1fe Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 10:12:10 +0800 Subject: [PATCH 41/64] fix another bug --- source/util/src/tcompression.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 003843c5bb..dc473922d7 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -999,7 +999,7 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co /************************************************************************* * STREAM COMPRESSION *************************************************************************/ -#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (a)) || ((a) < 0 && (b) >= INT64_MIN - (a))) static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData); From a4d426aa13a9b3621e2357e2ca080f5e34d8ab79 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 13:39:48 +0800 Subject: [PATCH 42/64] more fix --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 00b6830fbc..167b4a104c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -1069,8 +1069,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(pBlockData->suid == hdr.suid); - ASSERT(pBlockData->uid == hdr.uid); + pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; // uid From 5be361ee087dfee67ed8645a4017ddd569827357 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 14:01:34 +0800 Subject: [PATCH 43/64] more fix --- source/util/src/tcompression.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index dc473922d7..7d1c6b02ad 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1400,7 +1400,7 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { int32_t n = 1; int32_t nBuf = 1; int64_t vPrev = 0; - for (int32_t iVal = 0; iVal < pCmprsor->nVal;) { + while (n < pCmprsor->nBuf) { uint64_t b; memcpy(&b, pCmprsor->pBuf + n, sizeof(b)); n += sizeof(b); @@ -1410,14 +1410,18 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { uint8_t bits = BIT_PER_INTEGER[i_selector]; for (int32_t iEle = 0; iEle < nEle; iEle++) { uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); - int64_t diff = ZIGZAG_DECODE(int64_t, vZigzag); - vPrev = diff + vPrev; + vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; } - iVal += nEle; - ASSERT(iVal <= pCmprsor->nVal); + } + + for (; pCmprsor->i_start != pCmprsor->i_end; pCmprsor->i_start = (pCmprsor->i_start + 1) % 241) { + vPrev = ZIGZAG_DECODE(int64_t, pCmprsor->i_aZigzag[pCmprsor->i_start]) + vPrev; + + memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); + nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; } ASSERT(n == pCmprsor->nBuf && nBuf == size); From 0ca19fb42bd2a072eaed048060c203104b8f7245 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 15:46:56 +0800 Subject: [PATCH 44/64] fix more --- source/util/src/tcompression.c | 74 ++++++++++++++++++++++++---------- 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 7d1c6b02ad..a6167a1a75 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1024,6 +1024,11 @@ static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg); static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); + +static FORCE_INLINE int64_t tI8ToI64(const void *pData) { return *(int8_t *)pData; } +static FORCE_INLINE int64_t tI16ToI64(const void *pData) { return *(int16_t *)pData; } +static FORCE_INLINE int64_t tI32ToI64(const void *pData) { return *(int32_t *)pData; } +static FORCE_INLINE int64_t tI64ToI64(const void *pData) { return *(int64_t *)pData; } static struct { int8_t type; int32_t bytes; @@ -1031,122 +1036,148 @@ static struct { int32_t (*startFn)(SCompressor *, int8_t type, int8_t cmprAlg); int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); + int64_t (*getI64)(const void *pData); } DATA_TYPE_INFO[] = { - {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, .isVarLen = 0, .startFn = NULL, .cmprFn = NULL, .endFn = NULL}, + {.type = TSDB_DATA_TYPE_NULL, + .bytes = 0, + .isVarLen = 0, + .startFn = NULL, + .cmprFn = NULL, + .endFn = NULL, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .startFn = tCompBoolStart, .cmprFn = tCompBool, - .endFn = tCompBoolEnd}, + .endFn = tCompBoolEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI8ToI64}, {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI16ToI64}, {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI32ToI64}, {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI64ToI64}, {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .startFn = tCompFloatStart, .cmprFn = tCompFloat, - .endFn = tCompFloatEnd}, + .endFn = tCompFloatEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .startFn = tCompDoubleStart, .cmprFn = tCompDouble, - .endFn = tCompDoubleEnd}, + .endFn = tCompDoubleEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8, .isVarLen = 0, .startFn = tCompTimestampStart, .cmprFn = tCompTimestamp, - .endFn = tCompTimestampEnd}, + .endFn = tCompTimestampEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI8ToI64}, {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI16ToI64}, {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI32ToI64}, {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, - .endFn = tCompIntEnd}, + .endFn = tCompIntEnd, + .getI64 = tI64ToI64}, {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, - .endFn = tCompBinaryEnd}, + .endFn = tCompBinaryEnd, + .getI64 = NULL}, }; struct SCompressor { @@ -1442,8 +1473,7 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes); if (pCmprsor->pBuf[0] == 0) { - int64_t val = 0; - memcpy(&val, pData, nData); // little-endian only + int64_t val = DATA_TYPE_INFO[pCmprsor->type].getI64(pData); if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) { code = tCompIntSwitchToCopy(pCmprsor); From 82a227b9c74ad0ec7bac2cb59fe1f68d5cfccbd3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Sep 2022 18:48:35 +0800 Subject: [PATCH 45/64] more fix --- include/util/tcoding.h | 6 +- source/util/src/tcompression.c | 117 ++++++++++++++++++++++++--------- 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/include/util/tcoding.h b/include/util/tcoding.h index 5962949a70..38eb0d8fc6 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -23,8 +23,8 @@ extern "C" { #endif #define ENCODE_LIMIT (((uint8_t)1) << 7) -#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode -#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode +#define ZIGZAGE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode +#define ZIGZAGD(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode /* ------------------------ LEGACY CODES ------------------------ */ #if 1 @@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) { } static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) { - *value = ( (((int8_t *)buf)[0] == 0) ? false : true ); + *value = ((((int8_t *)buf)[0] == 0) ? false : true); return POINTER_SHIFT(buf, sizeof(int8_t)); } diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index a6167a1a75..9cac1fb957 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1; #define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) -#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode -#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode +#define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode +#define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode #ifdef TD_TSZ bool lossyFloat = false; @@ -1025,10 +1025,16 @@ static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprA static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData); static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData); -static FORCE_INLINE int64_t tI8ToI64(const void *pData) { return *(int8_t *)pData; } -static FORCE_INLINE int64_t tI16ToI64(const void *pData) { return *(int16_t *)pData; } -static FORCE_INLINE int64_t tI32ToI64(const void *pData) { return *(int32_t *)pData; } -static FORCE_INLINE int64_t tI64ToI64(const void *pData) { return *(int64_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI8(const void *pData) { return *(int8_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI16(const void *pData) { return *(int16_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI32(const void *pData) { return *(int32_t *)pData; } +static FORCE_INLINE int64_t tGetI64OfI64(const void *pData) { return *(int64_t *)pData; } + +static FORCE_INLINE void tPutI64OfI8(int64_t v, void *pData) { *(int8_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI16(int64_t v, void *pData) { *(int16_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI32(int64_t v, void *pData) { *(int32_t *)pData = v; } +static FORCE_INLINE void tPutI64OfI64(int64_t v, void *pData) { *(int64_t *)pData = v; } + static struct { int8_t type; int32_t bytes; @@ -1037,6 +1043,7 @@ static struct { int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData); int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *); int64_t (*getI64)(const void *pData); + void (*putI64)(int64_t v, void *pData); } DATA_TYPE_INFO[] = { {.type = TSDB_DATA_TYPE_NULL, .bytes = 0, @@ -1044,140 +1051,160 @@ static struct { .startFn = NULL, .cmprFn = NULL, .endFn = NULL, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_BOOL, .bytes = 1, .isVarLen = 0, .startFn = tCompBoolStart, .cmprFn = tCompBool, .endFn = tCompBoolEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_TINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI8ToI64}, + .getI64 = tGetI64OfI8, + .putI64 = tPutI64OfI8}, {.type = TSDB_DATA_TYPE_SMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI16ToI64}, + .getI64 = tGetI64OfI16, + .putI64 = tPutI64OfI16}, {.type = TSDB_DATA_TYPE_INT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI32ToI64}, + .getI64 = tGetI64OfI32, + .putI64 = tPutI64OfI32}, {.type = TSDB_DATA_TYPE_BIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI64ToI64}, + .getI64 = tGetI64OfI64, + .putI64 = tPutI64OfI64}, {.type = TSDB_DATA_TYPE_FLOAT, .bytes = 4, .isVarLen = 0, .startFn = tCompFloatStart, .cmprFn = tCompFloat, .endFn = tCompFloatEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_DOUBLE, .bytes = 8, .isVarLen = 0, .startFn = tCompDoubleStart, .cmprFn = tCompDouble, .endFn = tCompDoubleEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_VARCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_TIMESTAMP, .bytes = 8, .isVarLen = 0, .startFn = tCompTimestampStart, .cmprFn = tCompTimestamp, .endFn = tCompTimestampEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_NCHAR, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_UTINYINT, .bytes = 1, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI8ToI64}, + .getI64 = tGetI64OfI8, + .putI64 = tPutI64OfI8}, {.type = TSDB_DATA_TYPE_USMALLINT, .bytes = 2, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI16ToI64}, + .getI64 = tGetI64OfI16, + .putI64 = tPutI64OfI16}, {.type = TSDB_DATA_TYPE_UINT, .bytes = 4, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI32ToI64}, + .getI64 = tGetI64OfI32, + .putI64 = tPutI64OfI32}, {.type = TSDB_DATA_TYPE_UBIGINT, .bytes = 8, .isVarLen = 0, .startFn = tCompIntStart, .cmprFn = tCompInt, .endFn = tCompIntEnd, - .getI64 = tI64ToI64}, + .getI64 = tGetI64OfI64, + .putI64 = tPutI64OfI64}, {.type = TSDB_DATA_TYPE_JSON, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_VARBINARY, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_DECIMAL, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_BLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, {.type = TSDB_DATA_TYPE_MEDIUMBLOB, .bytes = 1, .isVarLen = 1, .startFn = tCompBinaryStart, .cmprFn = tCompBinary, .endFn = tCompBinaryEnd, - .getI64 = NULL}, + .getI64 = NULL, + .putI64 = NULL}, }; struct SCompressor { @@ -1443,7 +1470,7 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; - memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); + DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf); nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; } } @@ -1504,8 +1531,22 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; break; } else { - while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { - pCmprsor->i_selector++; + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + int32_t lidx = pCmprsor->i_selector + 1; + int32_t ridx = 15; + while (lidx <= ridx) { + pCmprsor->i_selector = (lidx + ridx) >> 1; + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + lidx = pCmprsor->i_selector + 1; + } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + ridx = pCmprsor->i_selector - 1; + } else { + break; + } + } + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; @@ -1551,8 +1592,22 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_ int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; if (nEle == 0) break; - while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { - pCmprsor->i_selector++; + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + int32_t lidx = pCmprsor->i_selector + 1; + int32_t ridx = 15; + while (lidx <= ridx) { + pCmprsor->i_selector = (lidx + ridx) >> 1; + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + lidx = pCmprsor->i_selector + 1; + } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + ridx = pCmprsor->i_selector - 1; + } else { + break; + } + } + + if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; From cf0aecd3dd5701082f94eb73f94a0e9c692ecd44 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 00:02:09 +0800 Subject: [PATCH 46/64] more fix --- source/util/src/tcompression.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 9cac1fb957..b19bfbbd43 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1320,7 +1320,7 @@ static int32_t tCompTSSwitchToCopy(SCompressor *pCmprsor) { } } - ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nBuf + 1); + ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nVal + 1); uint8_t *pBuf = pCmprsor->pBuf; pCmprsor->pBuf = pCmprsor->aBuf[0]; From 768ec8942a8f7bdd6a49a0c3c1d46faa9a680a0d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 13:23:12 +0800 Subject: [PATCH 47/64] fix code --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 99603500ae..8992ee47c4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -668,7 +668,8 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde .minKey = pBlkInfo->minKey, .maxKey = pBlkInfo->maxKey, .minVer = pBlkInfo->minVer, - .maxVer = pBlkInfo->maxVer}; + .maxVer = pBlkInfo->maxVer, + .nRow = pBuilder->nRow}; // write code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL); TSDB_CHECK_CODE(code, lino, _exit); From ce087d3dbc8f7b86fe87e1fb36e440800762ef29 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 14:54:58 +0800 Subject: [PATCH 48/64] optimize more code --- source/util/src/tcompression.c | 73 +++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 28 deletions(-) diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index b19bfbbd43..be220c2585 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -1228,6 +1228,7 @@ struct SCompressor { int32_t i_selector; int32_t i_start; int32_t i_end; + int32_t i_nEle; uint64_t i_aZigzag[241]; int8_t i_aBitN[241]; }; @@ -1427,6 +1428,18 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; +static const int32_t NEXT_IDX[] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, + 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, + 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, + 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, + 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, + 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, + 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, + 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, + 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, + 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 0}; static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) { int32_t code = 0; @@ -1435,6 +1448,7 @@ static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) pCmprsor->i_selector = 0; pCmprsor->i_start = 0; pCmprsor->i_end = 0; + pCmprsor->i_nEle = 0; pCmprsor->nBuf = 1; code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf); @@ -1463,11 +1477,12 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { memcpy(&b, pCmprsor->pBuf + n, sizeof(b)); n += sizeof(b); - int32_t i_selector = (b & 0xf); - int32_t nEle = SELECTOR_TO_ELEMS[i_selector]; - uint8_t bits = BIT_PER_INTEGER[i_selector]; + int32_t i_selector = (b & 0xf); + int32_t nEle = SELECTOR_TO_ELEMS[i_selector]; + uint8_t bits = BIT_PER_INTEGER[i_selector]; + uint64_t mask = (((uint64_t)1) << bits) - 1; for (int32_t iEle = 0; iEle < nEle; iEle++) { - uint64_t vZigzag = (b >> (bits * iEle + 4)) & (((uint64_t)1 << bits) - 1); + uint64_t vZigzag = (b >> (bits * iEle + 4)) & mask; vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev; DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf); @@ -1475,11 +1490,14 @@ static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) { } } - for (; pCmprsor->i_start != pCmprsor->i_end; pCmprsor->i_start = (pCmprsor->i_start + 1) % 241) { + while (pCmprsor->i_nEle) { vPrev = ZIGZAG_DECODE(int64_t, pCmprsor->i_aZigzag[pCmprsor->i_start]) + vPrev; memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes); nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes; + + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; } ASSERT(n == pCmprsor->nBuf && nBuf == size); @@ -1520,35 +1538,35 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) pCmprsor->i_prev = val; for (;;) { - int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; - - if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { + if (pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && + pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) { pCmprsor->i_selector = BIT_TO_SELECTOR[nBit]; } pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag; pCmprsor->i_aBitN[pCmprsor->i_end] = nBit; - pCmprsor->i_end = (pCmprsor->i_end + 1) % 241; + pCmprsor->i_end = NEXT_IDX[pCmprsor->i_end]; + pCmprsor->i_nEle++; break; } else { - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { int32_t lidx = pCmprsor->i_selector + 1; int32_t ridx = 15; while (lidx <= ridx) { pCmprsor->i_selector = (lidx + ridx) >> 1; - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { lidx = pCmprsor->i_selector + 1; - } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { ridx = pCmprsor->i_selector - 1; } else { break; } } - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } - nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { return code; @@ -1559,13 +1577,14 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) bp[0] = pCmprsor->i_selector; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; for (int32_t iVal = 0; iVal < nEle; iVal++) { - bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); - pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4)); + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; } // reset and continue pCmprsor->i_selector = 0; - for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = (iVal + 1) % 241) { + for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = NEXT_IDX[iVal]) { if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) { pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]; } @@ -1588,28 +1607,25 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) { int32_t code = 0; - for (;;) { - int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241; - if (nEle == 0) break; - - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + for (; pCmprsor->i_nEle;) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { int32_t lidx = pCmprsor->i_selector + 1; int32_t ridx = 15; while (lidx <= ridx) { pCmprsor->i_selector = (lidx + ridx) >> 1; - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { lidx = pCmprsor->i_selector + 1; - } else if (nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { + } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) { ridx = pCmprsor->i_selector - 1; } else { break; } } - if (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; + if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++; } - nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; + int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector]; if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) { return code; @@ -1620,8 +1636,9 @@ static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_ bp[0] = pCmprsor->i_selector; uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector]; for (int32_t iVal = 0; iVal < nEle; iVal++) { - bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4)); - pCmprsor->i_start = (pCmprsor->i_start + 1) % 241; + bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4)); + pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start]; + pCmprsor->i_nEle--; } pCmprsor->i_selector = 0; From bdeb1ebcc5c18c376f78555a9dbf040fcbaddd33 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Sep 2022 17:06:18 +0800 Subject: [PATCH 49/64] compare --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 76 ++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 8992ee47c4..05e08b6bf4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -17,6 +17,8 @@ typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; +#define USE_STREAM_COMPRESSION 1 + typedef struct { SRBTreeNode n; SRowInfo r; @@ -70,12 +72,16 @@ typedef struct { int8_t toLastOnly; }; struct { - SDataFWriter *pWriter; - SArray *aBlockIdx; // SArray - SArray *aSttBlk; // SArray - SMapData mBlock; // SMapData - SBlockData bData; + SDataFWriter *pWriter; + SArray *aBlockIdx; // SArray + SArray *aSttBlk; // SArray + SMapData mBlock; // SMapData + SBlockData bData; +#if USE_STREAM_COMPRESSION SDiskDataBuilder *pBuilder; +#else + SBlockData bDatal; +#endif } dWriter; SSkmInfo skmTable; SSkmInfo skmRow; @@ -542,7 +548,11 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); +#if USE_STREAM_COMPRESSION tDiskDataBuilderClear(pCommitter->dWriter.pBuilder); +#else + tBlockDataReset(&pCommitter->dWriter.bDatal); +#endif // open iter code = tsdbOpenCommitIter(pCommitter); @@ -860,7 +870,11 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { code = tBlockDataCreate(&pCommitter->dWriter.bData); TSDB_CHECK_CODE(code, lino, _exit); +#if USE_STREAM_COMPRESSION code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder); +#else + code = tBlockDataCreate(&pCommitter->dWriter.bDatal); +#endif TSDB_CHECK_CODE(code, lino, _exit); _exit: @@ -889,7 +903,11 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) { taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); +#if USE_STREAM_COMPRESSION tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder); +#else + tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); +#endif tTSchemaDestroy(pCommitter->skmTable.pTSchema); tTSchemaDestroy(pCommitter->skmRow.pTSchema); } @@ -1360,6 +1378,7 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) int32_t code = 0; int32_t lino = 0; +#if USE_STREAM_COMPRESSION SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder; if (pBuilder->suid || pBuilder->uid) { if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) { @@ -1376,6 +1395,25 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0); TSDB_CHECK_CODE(code, lino, _exit); } +#else + SBlockData *pBData = &pCommitter->dWriter.bDatal; + if (pBData->suid || pBData->uid) { + if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + + tBlockDataReset(pBData); + } + } + + if (!pBData->suid && !pBData->uid) { + ASSERT(pCommitter->skmTable.suid == id.suid); + ASSERT(pCommitter->skmTable.uid == id.uid); + TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid}; + code = tBlockDataInit(pBData, &tid, pCommitter->skmTable.pTSchema, NULL, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif _exit: if (code) { @@ -1398,6 +1436,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { TSDBROW row = tsdbRowFromBlockData(pBData, iRow); +#if USE_STREAM_COMPRESSION code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id); TSDB_CHECK_CODE(code, lino, _exit); @@ -1408,6 +1447,16 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); } +#else + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &row, NULL, id.uid); + TSDB_CHECK_CODE(code, lino, _exit); + + if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif } _exit: @@ -1441,7 +1490,11 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { pTSchema = pCommitter->skmRow.pTSchema; } +#if USE_STREAM_COMPRESSION code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id); +#else + code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pRowInfo->row, pTSchema, id.uid); +#endif TSDB_CHECK_CODE(code, lino, _exit); code = tsdbNextCommitRow(pCommitter); @@ -1452,6 +1505,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { pRowInfo = NULL; } +#if USE_STREAM_COMPRESSION if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) { code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); TSDB_CHECK_CODE(code, lino, _exit); @@ -1459,6 +1513,13 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); } +#else + if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + TSDB_CHECK_CODE(code, lino, _exit); + } +#endif } } else { SBlockData *pBData = &pCommitter->dWriter.bData; @@ -1561,7 +1622,12 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { code = tsdbMoveCommitData(pCommitter, id); TSDB_CHECK_CODE(code, lino, _exit); +#if USE_STREAM_COMPRESSION code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk); +#else + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); +#endif TSDB_CHECK_CODE(code, lino, _exit); _exit: From 46c99915bc7ebc106394ce8e833df3966eb3ae29 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Thu, 29 Sep 2022 18:52:11 +0800 Subject: [PATCH 50/64] feat(stream): distribute interval disc buff --- include/libs/stream/streamState.h | 8 +- source/libs/executor/inc/executil.h | 28 +- source/libs/executor/inc/executorimpl.h | 50 +- source/libs/executor/src/executorimpl.c | 28 +- source/libs/executor/src/scanoperator.c | 6 +- source/libs/executor/src/tfill.c | 4 +- source/libs/executor/src/timewindowoperator.c | 745 +++++++----------- source/libs/stream/src/streamState.c | 132 +++- .../tsim/stream/distributeInterval0.sim | 3 +- 9 files changed, 478 insertions(+), 526 deletions(-) diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 0adcf976f0..c2b3a7977c 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -33,6 +33,7 @@ typedef struct { TTB* pFuncStateDb; TTB* pFillStateDb; // todo refactor TXN txn; + int32_t number; } SStreamState; SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath); @@ -42,7 +43,8 @@ int32_t streamStateCommit(SStreamState* pState); int32_t streamStateAbort(SStreamState* pState); typedef struct { - TBC* pCur; + TBC* pCur; + int64_t number; } SStreamStateCur; int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen); @@ -52,6 +54,8 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key); int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateDel(SStreamState* pState, const SWinKey* key); +int32_t streamStateClear(SStreamState* pState); +void streamStateSetNumber(SStreamState* pState, int32_t number); int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); @@ -63,6 +67,7 @@ void streamFreeVal(void* val); SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key); +SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key); SStreamStateCur* streamStateFillSeekKeyPrev(SStreamState* pState, const SWinKey* key); void streamStateFreeCur(SStreamStateCur* pCur); @@ -70,6 +75,7 @@ void streamStateFreeCur(SStreamStateCur* pCur); int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); +int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key); int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index baf7d447cc..9dba29811e 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -15,7 +15,6 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H -#include "vnode.h" #include "function.h" #include "nodes.h" #include "plannodes.h" @@ -23,6 +22,7 @@ #include "tcommon.h" #include "tpagedbuf.h" #include "tsimplehash.h" +#include "vnode.h" #define T_LONG_JMP(_obj, _c) \ do { \ @@ -93,7 +93,7 @@ void resetResultRow(SResultRow* pResultRow, size_t entrySize); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos, bool forUpdate) { - SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); + SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); if (forUpdate) { setBufPageDirty(bufPage, true); } @@ -101,11 +101,6 @@ static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRo return pRow; } -static FORCE_INLINE void setResultBufPageDirty(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { - void* pPage = getBufPage(pBuf, pos->pageId); - setBufPageDirty(pPage, true); -} - void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t order); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); @@ -117,17 +112,18 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo); -int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); -int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo); -size_t getTableTagsBufLen(const SNodeList* pGroups); +int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, + STableListInfo* pListInfo); +int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId); +int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableListInfo* pTableListInfo); +size_t getTableTagsBufLen(const SNodeList* pGroups); -SArray* createSortInfo(SNodeList* pNodeList); -SArray* extractPartitionColInfo(SNodeList* pNodeList); -SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, - int32_t type); +SArray* createSortInfo(SNodeList* pNodeList); +SArray* extractPartitionColInfo(SNodeList* pNodeList); +SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, + int32_t type); -void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 56470f0668..897015c4d3 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -577,13 +577,7 @@ typedef struct SIntervalAggOperatorInfo { int32_t inputOrder; // input data ts order EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] STimeWindowAggSupp twAggSup; - bool invertible; SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. - bool ignoreExpiredData; - SArray* pRecycledPages; - SArray* pDelWins; // SWinRes - int32_t delIndex; - SSDataBlock* pDelRes; SNode* pCondition; } SIntervalAggOperatorInfo; @@ -609,38 +603,21 @@ typedef struct SStreamIntervalOperatorInfo { STimeWindowAggSupp twAggSup; bool invertible; bool ignoreExpiredData; - SArray* pRecycledPages; SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; - bool isFinal; -} SStreamIntervalOperatorInfo; - -typedef struct SStreamFinalIntervalOperatorInfo { - // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; // basic info - SAggSupporter aggSup; // aggregate supporter - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; // multiple results build supporter - SInterval interval; // interval info - int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SArray* pChildren; SSDataBlock* pUpdateRes; bool returnUpdate; - SPhysiNode* pPhyNode; // create new child - bool isFinal; + SPhysiNode* pPhyNode; // create new child SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo + SArray* pPullWins; // SPullWindowInfo int32_t pullIndex; SSDataBlock* pPullDataRes; - bool ignoreExpiredData; - SArray* pRecycledPages; - SArray* pDelWins; // SWinRes - int32_t delIndex; - SSDataBlock* pDelRes; -} SStreamFinalIntervalOperatorInfo; + bool isFinal; + SArray* pChildren; + SStreamState* pState; + SWinKey delKey; +} SStreamIntervalOperatorInfo; typedef struct SAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode @@ -1086,7 +1063,7 @@ bool functionNeedToExecute(SqlFunctionCtx* pCtx); bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup); +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup); void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp); void printDataBlock(SSDataBlock* pBlock, const char* flag); uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); @@ -1108,13 +1085,12 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol bool groupbyTbname(SNodeList* pGroupList); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); -int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo); -int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, - SExecTaskInfo* pTaskInfo); -int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); -int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); +int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup); +int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult); +int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize); void getNextIntervalWindow(SInterval* pInterval, STimeWindow* tw, int32_t order); #ifdef __cplusplus diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ff28199712..5db55b02f8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4183,9 +4183,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF return code; } -int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, - int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, - SExecTaskInfo* pTaskInfo) { +int32_t setOutputBuf(SStreamState* pState, STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, + SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup) { SWinKey key = { .ts = win->skey, .groupId = tableGroupId, @@ -4194,7 +4193,7 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI int32_t size = pAggSup->resultRowSize; tSimpleHashPut(pAggSup->pResultRowHashTable, &key, sizeof(SWinKey), NULL, 0); - if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) { + if (streamStateAddIfNotExist(pState, &key, (void**)&value, &size) < 0) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } *pResult = (SResultRow*)value; @@ -4205,18 +4204,17 @@ int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupI return TSDB_CODE_SUCCESS; } -int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) { - streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult); - /*taosMemoryFree((*(void**)pResult));*/ +int32_t releaseOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult) { + streamStateReleaseBuf(pState, pKey, pResult); return TSDB_CODE_SUCCESS; } -int32_t saveOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { - streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize); +int32_t saveOutputBuf(SStreamState* pState, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { + streamStatePut(pState, pKey, pResult, resSize); return TSDB_CODE_SUCCESS; } -int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup, SGroupResInfo* pGroupResInfo) { SExprInfo* pExprInfo = pSup->pExprInfo; int32_t numOfExprs = pSup->numOfExprs; @@ -4233,14 +4231,14 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock .ts = *(TSKEY*)pPos->key, .groupId = pPos->groupId, }; - int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size); + int32_t code = streamStateGet(pState, &key, &pVal, &size); ASSERT(code == 0); SResultRow* pRow = (SResultRow*)pVal; doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); // no results, continue to check the next one if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); continue; } @@ -4249,14 +4247,14 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock } else { // current value belongs to different group, it can't be packed into one datablock if (pBlock->info.groupId != pPos->groupId) { - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); break; } } if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { ASSERT(pBlock->info.rows > 0); - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); break; } @@ -4286,7 +4284,7 @@ int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock } pBlock->info.rows += pRow->numOfRows; - releaseOutputBuf(pTaskInfo, &key, pRow); + releaseOutputBuf(pState, &key, pRow); } blockDataUpdateTsWindow(pBlock, 0); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9694ccadb7..40fc8cb9a5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1357,7 +1357,8 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && - isDeletedStreamWindow(&win, pBlock->info.groupId, pInfo->pTableScanOp, &pInfo->twAggSup); + isDeletedStreamWindow(&win, pBlock->info.groupId, + pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, &pInfo->twAggSup); if ((update || closedWin) && out) { qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin); uint64_t gpId = 0; @@ -2135,6 +2136,9 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; + if (pInfo->pTableScanOp->pTaskInfo->streamInfo.pState) { + streamStateSetNumber(pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, -1); + } pInfo->readHandle = *pHandle; pInfo->tableUid = pScanPhyNode->uid; diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index ea0d26f4de..f6fd8c8920 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -1620,9 +1620,7 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi goto _error; } - SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL == downstream->operatorType - ? &((SStreamFinalIntervalOperatorInfo*)downstream->info)->interval - : &((SStreamIntervalOperatorInfo*)downstream->info)->interval; + SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; int32_t numOfFillCols = 0; SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b4e2b73889..9b4b7f1ee8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -867,10 +867,6 @@ static int32_t saveWinResult(int64_t ts, int32_t pageId, int32_t offset, uint64_ return TSDB_CODE_SUCCESS; } -static int32_t saveWinResultRow(SResultRow* result, uint64_t groupId, SHashObj* pUpdatedMap) { - return saveWinResult(result->win.skey, result->pageId, result->offset, groupId, pUpdatedMap); -} - static int32_t saveWinResultInfo(TSKEY ts, uint64_t groupId, SHashObj* pUpdatedMap) { return saveWinResult(ts, -1, -1, groupId, pUpdatedMap); } @@ -1390,7 +1386,7 @@ static bool doDeleteWindow(SOperatorInfo* pOperator, TSKEY ts, uint64_t groupId, SStreamIntervalOperatorInfo* pInfo = pOperator->info; SWinKey key = {.ts = ts, .groupId = groupId}; tSimpleHashRemove(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey)); - streamStateDel(pOperator->pTaskInfo->streamInfo.pState, &key); + streamStateDel(pInfo->pState, &key); return true; } @@ -1406,7 +1402,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); - while (win.ekey <= endTsCols[i]) { + while (win.skey <= endTsCols[i]) { uint64_t winGpId = pGpDatas[i]; bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput); SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; @@ -1505,64 +1501,13 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { return TSDB_CODE_SUCCESS; } -static int32_t closeIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, - SHashObj* pPullDataMap, SHashObj* closeWins, SArray* pRecyPages, - SDiskbasedBuf* pDiscBuf) { - qDebug("===stream===close interval window"); - void* pIte = NULL; - size_t keyLen = 0; - int32_t iter = 0; - while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { - void* key = tSimpleHashGetKey(pIte, &keyLen); - uint64_t groupId = *(uint64_t*)key; - ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); - TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); - STimeWindow win; - win.skey = ts; - win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - SWinKey winRe = { - .ts = win.skey, - .groupId = groupId, - }; - void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinKey)); - if (isCloseWindow(&win, pSup)) { - if (chIds && pPullDataMap) { - SArray* chAy = *(SArray**)chIds; - int32_t size = taosArrayGetSize(chAy); - qDebug("===stream===window %" PRId64 " wait child size:%d", win.skey, size); - for (int32_t i = 0; i < size; i++) { - qDebug("===stream===window %" PRId64 " wait child id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); - } - continue; - } else if (pPullDataMap) { - qDebug("===stream===close window %" PRId64, win.skey); - } - SResultRowPosition* pPos = (SResultRowPosition*)pIte; - if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { - int32_t code = saveWinResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - ASSERT(pRecyPages != NULL); - taosArrayPush(pRecyPages, &pPos->pageId); - } else { - // SFilePage* bufPage = getBufPage(pDiscBuf, pPos->pageId); - // dBufSetBufPageRecycled(pDiscBuf, bufPage); - } - char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; - SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - tSimpleHashIterateRemove(pHashMap, keyBuf, keyLen, &pIte, &iter); - } - } - return TSDB_CODE_SUCCESS; -} - static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SInterval* pInterval, SHashObj* pPullDataMap, SHashObj* closeWins, SOperatorInfo* pOperator) { qDebug("===stream===close interval window"); - void* pIte = NULL; - size_t keyLen = 0; - int32_t iter = 0; + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + SStreamIntervalOperatorInfo* pInfo = pOperator->info; while ((pIte = tSimpleHashIterate(pHashMap, pIte, &iter)) != NULL) { void* key = tSimpleHashGetKey(pIte, &keyLen); SWinKey* pWinKey = (SWinKey*)key; @@ -1591,24 +1536,85 @@ static int32_t closeStreamIntervalWindow(SSHashObj* pHashMap, STimeWindowAggSupp } } tSimpleHashIterateRemove(pHashMap, pWinKey, sizeof(SWinKey), &pIte, &iter); - - if (needDeleteWindowBuf(&win, pTwSup)) { - streamStateDel(pOperator->pTaskInfo->streamInfo.pState, pWinKey); - } } } return TSDB_CODE_SUCCESS; } -static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { +STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { + STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return w; +} + +static void deleteIntervalDiscBuf(SStreamState* pState, SHashObj* pPullDataMap, TSKEY mark, SInterval* pInterval, + SWinKey* key) { + STimeWindow tw = getFinalTimeWindow(key->ts, pInterval); + SWinKey next = {0}; + while (tw.ekey < mark) { + SStreamStateCur* pCur = streamStateSeekKeyNext(pState, key); + int32_t code = streamStateGetKVByCur(pCur, &next, NULL, 0); + streamStateFreeCur(pCur); + + void* chIds = taosHashGet(pPullDataMap, key, sizeof(SWinKey)); + if (chIds && pPullDataMap) { + SArray* chAy = *(SArray**)chIds; + int32_t size = taosArrayGetSize(chAy); + qDebug("===stream===window %" PRId64 " wait child size:%d", key->ts, size); + for (int32_t i = 0; i < size; i++) { + qDebug("===stream===window %" PRId64 " wait child id:%d", key->ts, *(int32_t*)taosArrayGet(chAy, i)); + } + break; + } + qDebug("===stream===delete window %" PRId64, key->ts); + int32_t codeDel = streamStateDel(pState, key); + if (codeDel != TSDB_CODE_SUCCESS) { + code = streamStateGetFirst(pState, key); + if (code != TSDB_CODE_SUCCESS) { + qDebug("===stream===stream state first key: empty-empty"); + return; + } + continue; + } + if (code == TSDB_CODE_SUCCESS) { + *key = next; + tw = getFinalTimeWindow(key->ts, pInterval); + } + } + + if (qDebugFlag & DEBUG_DEBUG) { + SStreamStateCur* pCur = streamStateGetCur(pState, key); + int32_t code = streamStateCurPrev(pState, pCur); + if (code == TSDB_CODE_SUCCESS) { + SWinKey tmpKey = {0}; + code = streamStateGetKVByCur(pCur, &tmpKey, NULL, 0); + if (code == TSDB_CODE_SUCCESS) { + STimeWindow tw = getFinalTimeWindow(tmpKey.ts, pInterval); + qDebug("===stream===error stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, + tw.ekey, tmpKey.groupId, mark); + } else { + STimeWindow tw = getFinalTimeWindow(key->ts, pInterval); + qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey, + key->groupId, mark); + } + } else { + STimeWindow tw = getFinalTimeWindow(key->ts, pInterval); + qDebug("===stream===stream state first key:%" PRId64 "-%" PRId64 ",%" PRId64 ",mark %" PRId64, tw.skey, tw.ekey, + key->groupId, mark); + } + streamStateFreeCur(pCur); + } +} + +static void closeChildIntervalWindow(SOperatorInfo* pOperator, SArray* pChildren, TSKEY maxTs) { int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { - SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); - SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); - closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, NULL, - pChInfo->aggSup.pResultBuf); + closeStreamIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL, + pOperator); } } @@ -1660,13 +1666,10 @@ void destroyIntervalOperatorInfo(void* param) { tdListFree(pInfo->binfo.resultRowInfo.openWindow); - pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); taosArrayDestroyEx(pInfo->pPrevValues, freeItem); pInfo->pPrevValues = NULL; - pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); cleanupGroupResInfo(&pInfo->groupResInfo); colDataDestroy(&pInfo->twAggSup.timeWindowData); @@ -1674,17 +1677,17 @@ void destroyIntervalOperatorInfo(void* param) { } void destroyStreamFinalIntervalOperatorInfo(void* param) { - SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); // it should be empty. taosHashCleanup(pInfo->pPullDataMap); taosArrayDestroy(pInfo->pPullWins); blockDataDestroy(pInfo->pPullDataRes); - taosArrayDestroy(pInfo->pRecycledPages); blockDataDestroy(pInfo->pUpdateRes); taosArrayDestroy(pInfo->pDelWins); blockDataDestroy(pInfo->pDelRes); + taosMemoryFreeClear(pInfo->pState); if (pInfo->pChildren) { int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -1794,7 +1797,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; pInfo->twAggSup = *pTwAggSupp; - pInfo->ignoreExpiredData = pPhyNode->window.igExpired; pInfo->pCondition = pPhyNode->window.node.pConditions; pInfo->binfo.mergeResultBlock = pPhyNode->window.mergeDataBlock; @@ -1827,9 +1829,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pInfo->win); - pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); - pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API - pInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, pInfo); if (pInfo->timeWindowInterpo) { pInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo)); @@ -1838,10 +1837,6 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* } } - pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); - pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); - pInfo->delIndex = 0; - pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); initResultRowInfo(&pInfo->binfo.resultRowInfo); pOperator->name = "TimeIntervalAggOperator"; @@ -2100,7 +2095,7 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { @@ -2356,13 +2351,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { @@ -2493,13 +2488,13 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { - colDataAppend(pDst, pResBlock->info.rows, (char *)&pSliceInfo->current, false); + colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); } else { - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); if (colDataIsNull_s(pSrc, i)) { @@ -2837,24 +2832,15 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } } -bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) { - int32_t bytes = sizeof(TSKEY); - SET_RES_WINDOW_KEY(pSup->keyBuf, &ts, bytes, groupId); - SResultRowPosition* p1 = - (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); - return p1 != NULL; +bool hasIntervalWindow(SStreamState* pState, SWinKey* pKey) { + return TSDB_CODE_SUCCESS == streamStateGet(pState, pKey, NULL, 0); } -STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { - STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - return w; -} - -static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, - SHashObj* pUpdatedMap) { - int32_t size = taosArrayGetSize(pWinArray); +static void rebuildIntervalWindow(SOperatorInfo* pOperator, SExprSupp* pSup, SArray* pWinArray, SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int32_t size = taosArrayGetSize(pWinArray); + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; if (!pInfo->pChildren) { return; } @@ -2862,31 +2848,35 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr SWinKey* pWinRes = taosArrayGet(pWinArray, i); SResultRow* pCurResult = NULL; STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval); - if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) { + if (isDeletedStreamWindow(&parentWin, pWinRes->groupId, pInfo->pState, &pInfo->twAggSup) && + isCloseWindow(&parentWin, &pInfo->twAggSup)) { continue; } - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t code = setOutputBuf(pInfo->pState, &parentWin, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup); + if (code != TSDB_CODE_SUCCESS || pCurResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); int32_t num = 0; for (int32_t j = 0; j < numOfChildren; j++) { - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); - SIntervalAggOperatorInfo* pChInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; - if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) { + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; + SExprSupp* pChildSup = &pChildOp->exprSupp; + if (!hasIntervalWindow(pChInfo->pState, pWinRes)) { continue; } num++; SResultRow* pChResult = NULL; - setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId, - pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, - pTaskInfo); + setOutputBuf(pChInfo->pState, &parentWin, &pChResult, pWinRes->groupId, pChildSup->pCtx, pChildSup->numOfExprs, + pChildSup->rowEntryInfoOffset, &pChInfo->aggSup); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); } if (num > 0 && pUpdatedMap) { - saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur); + saveWinResultInfo(pCurResult->win.skey, pWinRes->groupId, pUpdatedMap); + saveOutputBuf(pInfo->pState, pWinRes, pCurResult, pInfo->aggSup.resultRowSize); + releaseOutputBuf(pInfo->pState, pWinRes, pCurResult); } } } @@ -2898,15 +2888,15 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } -bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SOperatorInfo* pOperator, STimeWindowAggSupp* pTwSup) { +bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, SStreamState* pState, STimeWindowAggSupp* pTwSup) { if (pWin->ekey < pTwSup->maxTs - pTwSup->deleteMark) { SWinKey key = {.ts = pWin->skey, .groupId = groupId}; void* pVal = NULL; int32_t size = 0; - if (streamStateGet(pOperator->pTaskInfo->streamInfo.pState, &key, &pVal, &size) < 0) { + if (streamStateGet(pState, &key, &pVal, &size) == TSDB_CODE_SUCCESS) { return false; } - streamStateReleaseBuf(pOperator->pTaskInfo->streamInfo.pState, &key, pVal); + return true; } return false; } @@ -2929,109 +2919,12 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } -static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SHashObj* pUpdatedMap) { - SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; - SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t numOfOutput = pSup->numOfExprs; - int32_t step = 1; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t forwardRows = 0; - - ASSERT(pSDataBlock->pDataBlock != NULL); - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - - int32_t startPos = 0; - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = {0}; - if (IS_FINAL_OP(pInfo)) { - nextWin = getFinalTimeWindow(ts, &pInfo->interval); - } else { - nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->order); - } - while (1) { - bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { - startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); - if (startPos < 0) { - break; - } - continue; - } - if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) { - bool ignore = true; - SWinKey winRes = { - .ts = nextWin.skey, - .groupId = tableGroupId, - }; - void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); - if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { - SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; - // add pull data request - savePullWindow(&pull, pInfo->pPullWins); - int32_t size = taosArrayGetSize(pInfo->pChildren); - addPullWindow(pInfo->pPullDataMap, &winRes, size); - qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); - } else { - int32_t index = -1; - SArray* chArray = NULL; - int32_t chId = 0; - if (chIds) { - chArray = *(void**)chIds; - chId = getChildIndex(pSDataBlock); - index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); - } - if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) { - ignore = false; - } - } - - if (ignore) { - startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); - if (startPos < 0) { - break; - } - continue; - } - } - - int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (IS_FINAL_OP(pInfo)) { - forwardRows = 1; - } else { - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, - NULL, TSDB_ORDER_ASC); - } - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - pSDataBlock->info.rows, numOfOutput); - int32_t prevEndPos = (forwardRows - 1) * step + startPos; - ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); - if (startPos < 0) { - break; - } - } -} - -static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) { +static void clearStreamIntervalOperator(SStreamIntervalOperatorInfo* pInfo) { tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->aggSup.currentPageId = -1; + streamStateClear(pInfo->pState); } static void clearSpecialDataBlock(SSDataBlock* pBlock) { @@ -3115,12 +3008,12 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { } } -static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) { +static void addRetriveWindow(SArray* wins, SStreamIntervalOperatorInfo* pInfo) { int32_t size = taosArrayGetSize(wins); for (int32_t i = 0; i < size; i++) { SWinKey* winKey = taosArrayGet(wins, i); STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); - if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { + if (needDeleteWindowBuf(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); if (!chIds) { SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId}; @@ -3140,8 +3033,124 @@ static void clearFunctionContext(SExprSupp* pSup) { } } +void doBuildResult(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + return; + } + + // clear the existed group id + pBlock->info.groupId = 0; + buildDataBlockFromGroupRes(pTaskInfo, pState, pBlock, &pOperator->exprSupp, pGroupResInfo); +} + +static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, + SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pSup = &pOperatorInfo->exprSupp; + int32_t numOfOutput = pSup->numOfExprs; + int32_t step = 1; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; + + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + + int32_t startPos = 0; + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = + getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + while (1) { + bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); + if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } + + if (IS_FINAL_OP(pInfo) && isClosed && pInfo->pChildren) { + bool ignore = true; + SWinKey winRes = { + .ts = nextWin.skey, + .groupId = groupId, + }; + void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinKey)); + if (isDeletedStreamWindow(&nextWin, groupId, pInfo->pState, &pInfo->twAggSup) && !chIds) { + SPullWindowInfo pull = {.window = nextWin, .groupId = groupId}; + // add pull data request + savePullWindow(&pull, pInfo->pPullWins); + int32_t size = taosArrayGetSize(pInfo->pChildren); + addPullWindow(pInfo->pPullDataMap, &winRes, size); + qDebug("===stream===prepare retrive %" PRId64 ", size:%d", winRes.ts, size); + } else { + int32_t index = -1; + SArray* chArray = NULL; + int32_t chId = 0; + if (chIds) { + chArray = *(void**)chIds; + chId = getChildIndex(pSDataBlock); + index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + } + if (index == -1 || pSDataBlock->info.type == STREAM_PULL_DATA) { + ignore = false; + } + } + + if (ignore) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } + } + + int32_t code = setOutputBuf(pInfo->pState, &nextWin, &pResult, groupId, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { + saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); + } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + pSDataBlock->info.rows, numOfOutput); + SWinKey key = { + .ts = nextWin.skey, + .groupId = groupId, + }; + saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize); + releaseOutputBuf(pInfo->pState, &key, pResult); + if (pInfo->delKey.ts > key.ts) { + pInfo->delKey = key; + } + int32_t prevEndPos = (forwardRows - 1) * step + startPos; + ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); + startPos = + getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (startPos < 0) { + break; + } + } +} + static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { - SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; + SStreamIntervalOperatorInfo* pInfo = pOperator->info; SOperatorInfo* downstream = pOperator->pDownstream[0]; TSKEY maxTs = INT64_MIN; @@ -3169,7 +3178,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -3182,12 +3191,13 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearStreamIntervalOperator(pInfo); qDebug("===stream===clear semi operator"); } else { - freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, + &pInfo->interval, &pInfo->delKey); } return NULL; } else { if (!IS_FINAL_OP(pInfo)) { - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -3227,16 +3237,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins, NULL); if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; + int32_t childIndex = getChildIndex(pBlock); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; + SExprSupp* pChildSup = &pChildOp->exprSupp; - doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildSup->numOfExprs, pBlock, NULL); - rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, - pOperator->pTaskInfo, NULL); + doDeleteWindows(pChildOp, &pChildInfo->interval, pChildOp->exprSupp.numOfExprs, pBlock, NULL, NULL); + rebuildIntervalWindow(pOperator, pSup, pUpWins, pUpdatedMap); taosArrayDestroy(pUpWins); continue; } @@ -3247,15 +3256,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); - doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, delWins, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); - SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; - doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, &pInfo->twAggSup, pBlock, NULL, &pChildInfo->interval, NULL); - rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, - pOperator->pTaskInfo, pUpdatedMap); + int32_t childIndex = getChildIndex(pBlock); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + SStreamIntervalOperatorInfo* pChildInfo = pChildOp->info; + SExprSupp* pChildSup = &pChildOp->exprSupp; + doDeleteWindows(pChildOp, &pChildInfo->interval, pChildOp->exprSupp.numOfExprs, pBlock, NULL, NULL); + rebuildIntervalWindow(pOperator, pSup, delWins, pUpdatedMap); addRetriveWindow(delWins, pInfo); taosArrayAddAll(pInfo->pDelWins, delWins); taosArrayDestroy(delWins); @@ -3269,10 +3277,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { - SArray* pUpWins = taosArrayInit(8, sizeof(SWinKey)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); - removeResults(pUpWins, pUpdatedMap); - taosArrayDestroy(pUpWins); + doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, NULL, pUpdatedMap); if (taosArrayGetSize(pUpdated) > 0) { break; } @@ -3286,8 +3291,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SExprSupp* pExprSup = &pInfo->scalarSupp; projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } - setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); - doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); + setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3297,15 +3302,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { if (!pChildOp) { T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - SStreamFinalIntervalOperatorInfo* pTmpInfo = pChildOp->info; + SStreamIntervalOperatorInfo* pTmpInfo = pChildOp->info; pTmpInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; taosArrayPush(pInfo->pChildren, &pChildOp); qDebug("===stream===add child, id:%d", chIndex); } - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); - SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; - setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); - doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; + setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + doStreamIntervalAggImpl(pChildOp, pBlock, pBlock->info.groupId, NULL); } maxTs = TMAX(maxTs, pBlock->info.window.ekey); maxTs = TMAX(maxTs, pBlock->info.watermark); @@ -3315,12 +3320,11 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, - pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); - closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); - } else { - pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; + closeStreamIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, + pInfo->pPullDataMap, pUpdatedMap, pOperator); + closeChildIntervalWindow(pOperator, pInfo->pChildren, pInfo->twAggSup.maxTs); } + pInfo->binfo.pRes->info.watermark = pInfo->twAggSup.maxTs; void* pIte = NULL; while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { @@ -3329,7 +3333,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { taosHashCleanup(pUpdatedMap); taosArraySort(pUpdated, resultrowComparAsc); - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -3348,7 +3351,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); return pInfo->binfo.pRes; @@ -3366,15 +3369,14 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { - SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; - SStreamFinalIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamFinalIntervalOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; + SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } pOperator->pTaskInfo = pTaskInfo; - pInfo->order = TSDB_ORDER_ASC; pInfo->interval = (SInterval){.interval = pIntervalPhyNode->interval, .sliding = pIntervalPhyNode->sliding, .intervalUnit = pIntervalPhyNode->intervalUnit, @@ -3386,6 +3388,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, .minTs = INT64_MAX, + .deleteMark = INT64_MAX, }; ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -3415,6 +3418,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, ASSERT(numOfCols > 0); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + streamStateSetNumber(pInfo->pState, -1); + initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->pChildren = NULL; if (numOfChild > 0) { @@ -3422,9 +3429,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); if (pChildOp) { - SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; + SStreamIntervalOperatorInfo* pChInfo = pChildOp->info; pChInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; taosArrayPush(pInfo->pChildren, &pChildOp); + streamStateSetNumber(pChInfo->pState, i); continue; } goto _error; @@ -3458,7 +3466,8 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); pInfo->delIndex = 0; pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); - pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); + pInfo->delKey.ts = INT64_MAX; + pInfo->delKey.groupId = 0; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -5600,160 +5609,6 @@ _error: return NULL; } -static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SHashObj* pUpdatedMap) { - SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; - - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; - - int32_t startPos = 0; - int32_t numOfOutput = pSup->numOfExprs; - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; - uint64_t tableGroupId = pBlock->info.groupId; - bool ascScan = true; - TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); - SResultRow* pResult = NULL; - - STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); - int32_t ret = TSDB_CODE_SUCCESS; - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - } - - TSKEY ekey = ascScan ? win.ekey : win.skey; - int32_t forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - ASSERT(forwardRows > 0); - - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); - } - - STimeWindow nextWin = win; - while (1) { - int32_t prevEndPos = forwardRows - 1 + startPos; - startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); - if (startPos < 0) { - break; - } - if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) { - ekey = ascScan ? nextWin.ekey : nextWin.skey; - forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - continue; - } - - // null data, failed to allocate more memory buffer - int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, - pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - - ekey = ascScan ? nextWin.ekey : nextWin.skey; - forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); - } -} - -static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SHashObj* pUpdatedMap) { - SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; - - SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t numOfOutput = pSup->numOfExprs; - int32_t step = 1; - TSKEY* tsCols = NULL; - SResultRow* pResult = NULL; - int32_t forwardRows = 0; - - ASSERT(pSDataBlock->pDataBlock != NULL); - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - - int32_t startPos = 0; - TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); - while (1) { - bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); - if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { - startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); - if (startPos < 0) { - break; - } - continue; - } - - int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->aggSup, pTaskInfo); - if (code != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); - if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { - saveWinResultInfo(pResult->win.skey, tableGroupId, pUpdatedMap); - } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - pSDataBlock->info.rows, numOfOutput); - SWinKey key = { - .ts = nextWin.skey, - .groupId = tableGroupId, - }; - saveOutputBuf(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize); - releaseOutputBuf(pTaskInfo, &key, pResult); - int32_t prevEndPos = (forwardRows - 1) * step + startPos; - ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); - startPos = - getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); - if (startPos < 0) { - break; - } - } -} - -void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - // set output datablock version - pBlock->info.version = pTaskInfo->version; - - blockDataCleanup(pBlock); - if (!hasRemainResults(pGroupResInfo)) { - return; - } - - // clear the existed group id - pBlock->info.groupId = 0; - buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo); -} - static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SStreamIntervalOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -5772,12 +5627,13 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows > 0) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; } - + deleteIntervalDiscBuf(pInfo->pState, NULL, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, &pInfo->interval, + &pInfo->delKey); doSetOperatorCompleted(pOperator); return NULL; } @@ -5800,8 +5656,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - // doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval, - // pUpdatedMap); doDeleteWindows(pOperator, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, pInfo->pDelWins, pUpdatedMap); continue; @@ -5830,9 +5684,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { maxTs = TMAX(maxTs, pBlock->info.window.ekey); minTs = TMIN(minTs, pBlock->info.window.skey); - // doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); - // new disc buf - doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); + doStreamIntervalAggImpl(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); @@ -5857,7 +5709,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pDelRes; } - doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); + doBuildResult(pOperator, pInfo->pState, pInfo->binfo.pRes, &pInfo->groupResInfo); if (pInfo->binfo.pRes->info.rows > 0) { printDataBlock(pInfo->binfo.pRes, "single interval"); return pInfo->binfo.pRes; @@ -5866,20 +5718,6 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { return NULL; } -void destroyStreamIntervalOperatorInfo(void* param) { - SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; - cleanupBasicInfo(&pInfo->binfo); - cleanupAggSup(&pInfo->aggSup); - pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); - - pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); - pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); - - cleanupGroupResInfo(&pInfo->groupResInfo); - colDataDestroy(&pInfo->twAggSup.timeWindowData); - taosMemoryFreeClear(param); -} - SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); @@ -5938,20 +5776,35 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); - pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API - pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); + pInfo->invertible = false; pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); pInfo->delIndex = 0; pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); initResultRowInfo(&pInfo->binfo.resultRowInfo); + pInfo->pState = taosMemoryCalloc(1, sizeof(SStreamState)); + *(pInfo->pState) = *(pTaskInfo->streamInfo.pState); + streamStateSetNumber(pInfo->pState, -1); + + pInfo->pUpdateRes = NULL; + pInfo->returnUpdate = false; + pInfo->pPhyNode = NULL; // create new child + pInfo->pPullDataMap = NULL; + pInfo->pPullWins = NULL; // SPullWindowInfo + pInfo->pullIndex = 0; + pInfo->pPullDataRes = NULL; + pInfo->isFinal = false; + pInfo->pChildren = NULL; + pInfo->delKey.ts = INT64_MAX; + pInfo->delKey.groupId = 0; + pOperator->name = "StreamIntervalOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo, + createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, &pInfo->twAggSup); @@ -5963,7 +5816,7 @@ SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhys return pOperator; _error: - destroyStreamIntervalOperatorInfo(pInfo); + destroyStreamFinalIntervalOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 3428a85823..da0d0fbd6d 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -18,6 +18,37 @@ #include "tcommon.h" #include "ttimer.h" +// todo refactor +typedef struct SStateKey { + SWinKey key; + int64_t opNum; +} SStateKey; + +static inline int SStateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { + SStateKey* pWin1 = (SStateKey*)pKey1; + SStateKey* pWin2 = (SStateKey*)pKey2; + + if (pWin1->opNum > pWin2->opNum) { + return 1; + } else if (pWin1->opNum < pWin2->opNum) { + return -1; + } + + if (pWin1->key.ts > pWin2->key.ts) { + return 1; + } else if (pWin1->key.ts < pWin2->key.ts) { + return -1; + } + + if (pWin1->key.groupId > pWin2->key.groupId) { + return 1; + } else if (pWin1->key.groupId < pWin2->key.groupId) { + return -1; + } + + return 0; +} + SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); if (pState == NULL) { @@ -36,7 +67,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath) { } // open state storage backend - if (tdbTbOpen("state.db", sizeof(SWinKey), -1, SWinKeyCmpr, pState->db, &pState->pStateDb) < 0) { + if (tdbTbOpen("state.db", sizeof(SStateKey), -1, SStateKeyCmpr, pState->db, &pState->pStateDb) < 0) { goto _err; } @@ -130,8 +161,10 @@ int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn); } +// todo refactor int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { - return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); + SStateKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbUpsert(pState->pStateDb, &sKey, sizeof(SStateKey), value, vLen, &pState->txn); } // todo refactor @@ -139,8 +172,10 @@ int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* return tdbTbUpsert(pState->pFillStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); } +// todo refactor int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { - return tdbTbGet(pState->pStateDb, key, sizeof(SWinKey), pVal, pVLen); + SStateKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbGet(pState->pStateDb, &sKey, sizeof(SStateKey), pVal, pVLen); } // todo refactor @@ -148,10 +183,30 @@ int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal return tdbTbGet(pState->pFillStateDb, key, sizeof(SWinKey), pVal, pVLen); } +// todo refactor int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { - return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); + SStateKey sKey = {.key = *key, .opNum = pState->number}; + return tdbTbDelete(pState->pStateDb, &sKey, sizeof(SStateKey), &pState->txn); } +int32_t streamStateClear(SStreamState* pState) { + SWinKey key = {.ts = 0, .groupId = 0}; + streamStatePut(pState, &key, NULL, 0); + while (1) { + SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &key); + SWinKey delKey = {0}; + int32_t code = streamStateGetKVByCur(pCur, &delKey, NULL, 0); + if (code == 0) { + streamStateDel(pState, &delKey); + } else { + break; + } + } + return 0; +} + +void streamStateSetNumber(SStreamState* pState, int32_t number) { pState->number = number; } + // todo refactor int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key) { return tdbTbDelete(pState->pFillStateDb, key, sizeof(SWinKey), &pState->txn); @@ -179,12 +234,14 @@ SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { if (pCur == NULL) return NULL; tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL); - int32_t c; + int32_t c; + SStateKey sKey = {.key = *key, .opNum = pState->number}; tdbTbcMoveTo(pCur->pCur, key, sizeof(SWinKey), &c); if (c != 0) { taosMemoryFree(pCur); return NULL; } + pCur->number = pState->number; return pCur; } @@ -214,6 +271,25 @@ SStreamStateCur* streamStateGetAndCheckCur(SStreamState* pState, SWinKey* key) { } int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { + if (!pCur) { + return -1; + } + const SStateKey* pKTmp = NULL; + int32_t kLen; + if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { + return -1; + } + if (pKTmp->opNum != pCur->number) { + return -1; + } + *pKey = pKTmp->key; + return 0; +} + +int32_t streamStateFillGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { + if (!pCur) { + return -1; + } const SWinKey* pKTmp = NULL; int32_t kLen; if (tdbTbcGet(pCur->pCur, (const void**)&pKTmp, &kLen, pVal, pVLen) < 0) { @@ -225,7 +301,7 @@ int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen) { uint64_t groupId = pKey->groupId; - int32_t code = streamStateGetKVByCur(pCur, pKey, pVal, pVLen); + int32_t code = streamStateFillGetKVByCur(pCur, pKey, pVal, pVLen); if (code == 0) { if (pKey->groupId == groupId) { return 0; @@ -234,6 +310,16 @@ int32_t streamStateGetGroupKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const v return -1; } +int32_t streamStateGetFirst(SStreamState* pState, SWinKey* key) { + // todo refactor + SWinKey tmp = {.ts = 0, .groupId = 0}; + streamStatePut(pState, &tmp, NULL, 0); + SStreamStateCur* pCur = streamStateSeekKeyNext(pState, &tmp); + int32_t code = streamStateGetKVByCur(pCur, key, NULL, 0); + streamStateDel(pState, &tmp); + return code; +} + int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur) { // return tdbTbcMoveToFirst(pCur->pCur); @@ -244,6 +330,34 @@ int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur) { return tdbTbcMoveToLast(pCur->pCur); } +SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key) { + SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); + if (pCur == NULL) { + return NULL; + } + pCur->number = pState->number; + if (tdbTbcOpen(pState->pStateDb, &pCur->pCur, NULL) < 0) { + taosMemoryFree(pCur); + return NULL; + } + + SStateKey sKey = {.key = *key, .opNum = pState->number}; + int32_t c; + if (tdbTbcMoveTo(pCur->pCur, &sKey, sizeof(SStateKey), &c) < 0) { + tdbTbcClose(pCur->pCur); + taosMemoryFree(pCur); + return NULL; + } + if (c > 0) return pCur; + + if (tdbTbcMoveToNext(pCur->pCur) < 0) { + taosMemoryFree(pCur); + return NULL; + } + + return pCur; +} + SStreamStateCur* streamStateFillSeekKeyNext(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) { @@ -303,9 +417,15 @@ int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { // + if (!pCur) { + return -1; + } return tdbTbcMoveToPrev(pCur->pCur); } void streamStateFreeCur(SStreamStateCur* pCur) { + if (!pCur) { + return; + } tdbTbcClose(pCur->pCur); taosMemoryFree(pCur); } diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 9b2e940556..1d58922928 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -33,7 +33,8 @@ if $data(2)[4] != ready then endi print ===== step2 - +sql drop stream if exists stream_t1; +sql drop database if exists test; sql create database test vgroups 4; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); From 145abcb17fdb648eb69919411a6e7a73da1c1968 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 30 Sep 2022 10:52:17 +0800 Subject: [PATCH 51/64] more --- source/dnode/vnode/src/tsdb/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 05e08b6bf4..995d7a3155 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -17,7 +17,7 @@ typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; -#define USE_STREAM_COMPRESSION 1 +#define USE_STREAM_COMPRESSION 0 typedef struct { SRBTreeNode n; From 3dbb7c435a4c666e9857b5127c5ed322da6c7ab2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 30 Sep 2022 11:00:35 +0800 Subject: [PATCH 52/64] docs: add sub-tbname rule note for tcollector (#17167) --- docs/en/20-third-party/08-tcollector.md | 16 +++++++++++----- docs/zh/20-third-party/08-tcollector.md | 17 ++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/docs/en/20-third-party/08-tcollector.md b/docs/en/20-third-party/08-tcollector.md index 16b73c2394..206b9dd23c 100644 --- a/docs/en/20-third-party/08-tcollector.md +++ b/docs/en/20-third-party/08-tcollector.md @@ -33,11 +33,12 @@ Wait for a few seconds and then use the TDengine CLI to query whether the corres ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - tcollector | 2022-04-20 12:44:49.604 | 88 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.002679s) + name | +================================= + information_schema | + performance_schema | + tcollector | +Query OK, 3 rows in database (0.001647s) taos> use tcollector; Database changed. @@ -65,3 +66,8 @@ taos> show stables; proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | ... ``` + +:::note + +- TDengine will automatically create unique IDs for sub-table names by the rule. +::: diff --git a/docs/zh/20-third-party/08-tcollector.md b/docs/zh/20-third-party/08-tcollector.md index 78d0b4a5df..b79e3ed2d1 100644 --- a/docs/zh/20-third-party/08-tcollector.md +++ b/docs/zh/20-third-party/08-tcollector.md @@ -34,11 +34,13 @@ sudo systemctl restart taosadapter ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - tcollector | 2022-04-20 12:44:49.604 | 88 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.002679s) + name | +================================= + information_schema | + performance_schema | + tcollector | +Query OK, 3 rows in database (0.001647s) + taos> use tcollector; Database changed. @@ -66,3 +68,8 @@ taos> show stables; proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | ... ``` + +:::note + +- TDengine 默认生成的子表名是根据规则生成的唯一 ID 值。 +::: From 55ffdaecd474844ec40100628d2463d2fe330bcb Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 30 Sep 2022 11:11:06 +0800 Subject: [PATCH 53/64] docs: fix show database output (#17169) * docs: add prometheus sub-table naming by rule note * docs: fix show databases output --- docs/en/20-third-party/02-prometheus.md | 13 ++++++------- docs/zh/20-third-party/02-prometheus.md | 13 ++++++------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/docs/en/20-third-party/02-prometheus.md b/docs/en/20-third-party/02-prometheus.md index c46b0381b9..49d701bd20 100644 --- a/docs/en/20-third-party/02-prometheus.md +++ b/docs/en/20-third-party/02-prometheus.md @@ -30,13 +30,12 @@ After restarting Prometheus, you can refer to the following example to verify th ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - test | 2022-04-12 08:07:58.756 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - log | 2022-04-20 07:19:50.260 | 2 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - prometheus_data | 2022-04-20 07:21:09.202 | 158 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - db | 2022-04-15 06:37:08.512 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 4 row(s) in set (0.000585s) + name | +================================= + information_schema | + performance_schema | + prometheus_data | +Query OK, 3 row(s) in set (0.000585s) taos> use prometheus_data; Database changed. diff --git a/docs/zh/20-third-party/02-prometheus.md b/docs/zh/20-third-party/02-prometheus.md index 59bab1823a..8b8d078792 100644 --- a/docs/zh/20-third-party/02-prometheus.md +++ b/docs/zh/20-third-party/02-prometheus.md @@ -29,13 +29,12 @@ Prometheus 提供了 `remote_write` 和 `remote_read` 接口来利用其它数 ### 使用 TDengine CLI 查询写入数据 ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - test | 2022-04-12 08:07:58.756 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - log | 2022-04-20 07:19:50.260 | 2 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - prometheus_data | 2022-04-20 07:21:09.202 | 158 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - db | 2022-04-15 06:37:08.512 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 4 row(s) in set (0.000585s) + name | +================================= + information_schema | + performance_schema | + prometheus_data | +Query OK, 3 row(s) in set (0.000585s) taos> use prometheus_data; Database changed. From c6317e67f0c61ab85c9e0c814643c2217391e9a0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 30 Sep 2022 11:26:26 +0800 Subject: [PATCH 54/64] fix: coverity issues --- source/dnode/mnode/impl/src/mndCluster.c | 1 + source/dnode/mnode/impl/src/mndDb.c | 8 +++--- source/dnode/mnode/impl/src/mndDnode.c | 14 ++++++----- source/dnode/mnode/impl/src/mndFunc.c | 9 ++++--- source/dnode/mnode/impl/src/mndGrant.c | 28 ++++++++++----------- source/dnode/mnode/impl/src/mndMain.c | 2 +- source/dnode/mnode/impl/src/mndMnode.c | 11 +++++--- source/dnode/mnode/impl/src/mndPerfSchema.c | 6 ++--- source/dnode/mnode/impl/src/mndProfile.c | 4 +-- source/dnode/mnode/impl/src/mndSubscribe.c | 2 +- 10 files changed, 48 insertions(+), 37 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 96748b3620..3502ecf604 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -237,6 +237,7 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, NULL, "create-cluster"); if (pTrans == NULL) { + sdbFreeRaw(pRaw); mError("cluster:%" PRId64 ", failed to create since %s", clusterObj.id, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index a05d8dd739..1d70f43834 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -1594,7 +1594,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, break; } char precVstr[10] = {0}; - STR_WITH_SIZE_TO_VARSTR(precVstr, precStr, 2); + STR_WITH_MAXSIZE_TO_VARSTR(precVstr, precStr, 10); char *statusStr = "ready"; if (objStatus == SDB_STATUS_CREATING) { @@ -1607,7 +1607,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, } } char statusVstr[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(statusVstr, statusStr, strlen(statusStr)); + STR_WITH_MAXSIZE_TO_VARSTR(statusVstr, statusStr, 24); if (sysDb || !sysinfo) { for (int32_t i = 0; i < pShow->numOfColumns; ++i) { @@ -1644,7 +1644,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, const char *strictStr = pDb->cfg.strict ? "on" : "off"; char strictVstr[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(strictVstr, strictStr, strlen(strictStr)); + STR_WITH_MAXSIZE_TO_VARSTR(strictVstr, strictStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)strictVstr, false); @@ -1704,7 +1704,7 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, const char *cacheModelStr = getCacheModelStr(pDb->cfg.cacheLast); char cacheModelVstr[24] = {0}; - STR_WITH_SIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, strlen(cacheModelStr)); + STR_WITH_MAXSIZE_TO_VARSTR(cacheModelVstr, cacheModelStr, 24); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)cacheModelVstr, false); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4a8436513f..c8995eb6bc 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -102,7 +102,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) { dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.port = tsServerPort; memcpy(&dnodeObj.fqdn, tsLocalFqdn, TSDB_FQDN_LEN); - snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); + snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", dnodeObj.fqdn, dnodeObj.port); pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-dnode"); if (pTrans == NULL) goto _OVER; @@ -190,7 +190,7 @@ _OVER: static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode) { mTrace("dnode:%d, perform insert action, row:%p", pDnode->id, pDnode); pDnode->offlineReason = DND_REASON_STATUS_NOT_RECEIVED; - snprintf(pDnode->ep, TSDB_EP_LEN, "%s:%u", pDnode->fqdn, pDnode->port); + snprintf(pDnode->ep, TSDB_EP_LEN - 1, "%s:%u", pDnode->fqdn, pDnode->port); return 0; } @@ -253,7 +253,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SDnodeObj *pDnode, int64_t curMs) { int64_t interval = TABS(pDnode->lastAccessTime - curMs); - if (interval > 5000 * tsStatusInterval) { + if (interval > 5000 * (int64_t)tsStatusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } @@ -486,7 +486,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC dnodeObj.updateTime = dnodeObj.createdTime; dnodeObj.port = pCreate->port; memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN); - snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port); + snprintf(dnodeObj.ep, TSDB_EP_LEN - 1, "%s:%u", dnodeObj.fqdn, dnodeObj.port); pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_GLOBAL, pReq, "create-dnode"); if (pTrans == NULL) goto _OVER; @@ -673,12 +673,14 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM mInfo("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); pRaw = mndDnodeActionEncode(pDnode); - if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) goto _OVER; + if (mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; sdbSetRawStatus(pRaw, SDB_STATUS_DROPPING); pRaw = NULL; pRaw = mndDnodeActionEncode(pDnode); - if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; + if (pRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER; sdbSetRawStatus(pRaw, SDB_STATUS_DROPPED); pRaw = NULL; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 727f7b0cc9..ccab4774e2 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -257,15 +257,18 @@ static int32_t mndDropFunc(SMnode *pMnode, SRpcMsg *pReq, SFuncObj *pFunc) { mInfo("trans:%d, used to drop user:%s", pTrans->id, pFunc->name); SSdbRaw *pRedoRaw = mndFuncActionEncode(pFunc); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (pRedoRaw == NULL) goto _OVER; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING); SSdbRaw *pUndoRaw = mndFuncActionEncode(pFunc); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (pUndoRaw == NULL) goto _OVER; + if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY); SSdbRaw *pCommitRaw = mndFuncActionEncode(pFunc); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (pCommitRaw == NULL) goto _OVER; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED); if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndGrant.c b/source/dnode/mnode/impl/src/mndGrant.c index a608a1e40f..7262f23657 100644 --- a/source/dnode/mnode/impl/src/mndGrant.c +++ b/source/dnode/mnode/impl/src/mndGrant.c @@ -30,85 +30,85 @@ static int32_t mndRetrieveGrant(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols); const char *src = "community"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "false"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); cols++; pColInfo = taosArrayGet(pBlock->pDataBlock, cols); src = "unlimited"; - STR_WITH_SIZE_TO_VARSTR(tmp, src, strlen(src)); + STR_WITH_MAXSIZE_TO_VARSTR(tmp, src, 32); colDataAppend(pColInfo, numOfRows, tmp, false); numOfRows++; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index a628cefa65..b3b9927a34 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -649,7 +649,7 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { tmsg_t type = TMSG_INDEX(msgType); - if (type >= 0 && type < TDMT_MAX) { + if (type < TDMT_MAX) { pMnode->msgFp[type] = fp; } } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index a41f958c0f..cfb362c51c 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -93,6 +93,7 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, NULL, "create-mnode"); if (pTrans == NULL) { + sdbFreeRaw(pRaw); mError("mnode:%d, failed to create since %s", mnodeObj.id, terrstr()); return -1; } @@ -220,8 +221,12 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { SSdb *pSdb = pMnode->pSdb; int32_t totalMnodes = sdbGetSize(pSdb, SDB_MNODE); - void *pIter = NULL; + if (totalMnodes == 0) { + syncGetRetryEpSet(pMnode->syncMgmt.sync, pEpSet); + return; + } + void *pIter = NULL; while (1) { SMnodeObj *pObj = NULL; pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj); @@ -658,7 +663,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pObj->id, false); char b1[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(b1, pObj->pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, b1, false); @@ -667,7 +672,7 @@ static int32_t mndRetrieveMnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pObj->id == pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_LEADER); } - if (pObj->pDnode && mndIsDnodeOnline(pObj->pDnode, curMs)) { + if (mndIsDnodeOnline(pObj->pDnode, curMs)) { roles = syncStr(pObj->state); if (pObj->state == TAOS_SYNC_STATE_LEADER && pObj->id != pMnode->selfDnodeId) { roles = syncStr(TAOS_SYNC_STATE_ERROR); diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index fe8d1a0a0e..cb636dc037 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -26,7 +26,7 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S } for (int32_t i = 0; i < colNum; ++i) { - strcpy(schema[i].name, pSrc[i].name); + tstrncpy(schema[i].name, pSrc[i].name, sizeof(schema[i].name)); schema[i].type = pSrc[i].type; schema[i].colId = i + 1; @@ -40,7 +40,7 @@ int32_t mndInitPerfsTableSchema(const SSysDbTableSchema *pSrc, int32_t colNum, S int32_t mndPerfsInitMeta(SHashObj *hash) { STableMetaRsp meta = {0}; - strcpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB); + tstrncpy(meta.dbFName, TSDB_INFORMATION_SCHEMA_DB, sizeof(meta.dbFName)); meta.tableType = TSDB_SYSTEM_TABLE; meta.sversion = 1; meta.tversion = 1; @@ -50,7 +50,7 @@ int32_t mndPerfsInitMeta(SHashObj *hash) { getPerfDbMeta(&pSysDbTableMeta, &size); for (int32_t i = 0; i < size; ++i) { - strcpy(meta.tbName, pSysDbTableMeta[i].name); + tstrncpy(meta.tbName, pSysDbTableMeta[i].name, sizeof(meta.tbName)); meta.numOfColumns = pSysDbTableMeta[i].colNum; if (mndInitPerfsTableSchema(pSysDbTableMeta[i].schema, pSysDbTableMeta[i].colNum, &meta.pSchemas)) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e8737e30c9..1909b168bd 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -664,7 +664,7 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { int32_t code = -1; SServerVerRsp rsp = {0}; - strcpy(rsp.ver, version); + tstrncpy(rsp.ver, version, sizeof(rsp.ver)); int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); if (contLen < 0) goto _over; @@ -702,7 +702,7 @@ static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl break; } - if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > (keepTime * 1000)) { + if ((taosGetTimestampMs() - pConn->lastAccessTimeMs) > ((int64_t)keepTime * 1000)) { continue; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 21e45407fd..f2fec27bc5 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -123,7 +123,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.subType = pSub->subType; req.withMeta = pSub->withMeta; req.suid = pSub->stbUid; - strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); + tstrncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); void *buf = taosMemoryMalloc(tlen); From 86ca19f4d42fecdec6fcb4f73357c060cd05d4f9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 30 Sep 2022 11:58:30 +0800 Subject: [PATCH 55/64] docs: sub-tbname rule note and fix show stables output (#17171) --- .../03-insert-data/03-opentsdb-telnet.mdx | 8 ++-- .../03-insert-data/04-opentsdb-json.mdx | 8 ++-- .../03-connector/_verify_linux.mdx | 10 ++-- .../03-connector/_verify_windows.mdx | 11 ++--- docs/en/14-reference/11-docker/index.md | 9 ++-- docs/en/14-reference/_statsd.mdx | 1 + docs/en/20-third-party/02-prometheus.md | 6 +-- docs/en/20-third-party/03-telegraf.md | 20 ++++---- docs/en/20-third-party/05-collectd.md | 35 +++++++------- docs/en/20-third-party/06-statsd.md | 24 ++++++---- docs/en/20-third-party/07-icinga2.md | 48 +++++++++---------- docs/en/20-third-party/08-tcollector.md | 40 ++++++++-------- .../03-insert-data/03-opentsdb-telnet.mdx | 8 ++-- .../03-insert-data/04-opentsdb-json.mdx | 8 ++-- docs/zh/08-connector/_verify_linux.mdx | 10 ++-- docs/zh/14-reference/_statsd.mdx | 1 + docs/zh/20-third-party/02-prometheus.md | 6 +-- docs/zh/20-third-party/03-telegraf.md | 20 ++++---- docs/zh/20-third-party/05-collectd.md | 35 +++++++------- docs/zh/20-third-party/06-statsd.md | 24 ++++++---- docs/zh/20-third-party/07-icinga2.md | 48 +++++++++---------- docs/zh/20-third-party/08-tcollector.md | 40 ++++++++-------- 22 files changed, 218 insertions(+), 202 deletions(-) diff --git a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx index d88a6335cb..a71efde735 100644 --- a/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -64,10 +64,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-30 17:04:10.877 | 2 | 2 | 2 | - meters.voltage | 2022-03-30 17:04:10.882 | 2 | 2 | 2 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.002544s) taos> select tbname, * from `meters.current`; diff --git a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx index e2e6d6fc9f..c0a287e331 100644 --- a/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -81,10 +81,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-29 16:05:25.193 | 2 | 2 | 1 | - meters.voltage | 2022-03-29 16:05:25.200 | 2 | 2 | 1 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.001954s) taos> select * from `meters.current`; diff --git a/docs/en/14-reference/03-connector/_verify_linux.mdx b/docs/en/14-reference/03-connector/_verify_linux.mdx index 3438b1578e..220fccfef3 100644 --- a/docs/en/14-reference/03-connector/_verify_linux.mdx +++ b/docs/en/14-reference/03-connector/_verify_linux.mdx @@ -4,11 +4,11 @@ Execute TDengine CLI program `taos` directly from the Linux shell to connect to $ taos taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - db | 2022-08-04 14:14:49.385 | 2 | 4 | 1 | off | 14400m | 5254560m,5254560m,5254560m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | + name | +================================= + information_schema | + performance_schema | + db | Query OK, 3 rows in database (0.019154s) taos> diff --git a/docs/en/14-reference/03-connector/_verify_windows.mdx b/docs/en/14-reference/03-connector/_verify_windows.mdx index 402b170511..65d10f0875 100644 --- a/docs/en/14-reference/03-connector/_verify_windows.mdx +++ b/docs/en/14-reference/03-connector/_verify_windows.mdx @@ -2,12 +2,11 @@ Go to the `C:\TDengine` directory from `cmd` and execute TDengine CLI program `t ```text taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - test | 2022-08-04 16:46:40.506 | 2 | 0 | 1 | off | 14400m | 5256000m,5256000m,5256000m | 96 | 4 | 256 | -100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | + name | +================================= + information_schema | + performance_schema | + test | Query OK, 3 rows in database (0.123000s) taos> diff --git a/docs/en/14-reference/11-docker/index.md b/docs/en/14-reference/11-docker/index.md index 7cd1e810dc..89987c456f 100644 --- a/docs/en/14-reference/11-docker/index.md +++ b/docs/en/14-reference/11-docker/index.md @@ -25,10 +25,11 @@ The TDengine client taos can be executed in this container to access TDengine us $ docker exec -it tdengine taos taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-01-17 13:57:22.270 | 10 | 1 | 1 | 1 | 10 | 30 | 1 | 3 | 100 | 4096 | 1 | 3000 | 2 | 0 | us | 0 | ready | -Query OK, 1 row(s) in set (0.002843s) + name | +================================= + information_schema | + performance_schema | +Query OK, 2 row(s) in set (0.002843s) ``` The TDengine server running in the container uses the container's hostname to establish a connection. Using TDengine CLI or various connectors (such as JDBC-JNI) to access the TDengine inside the container from outside the container is more complicated. So the above is the simplest way to access the TDengine service in the container and is suitable for some simple scenarios. Please refer to the next section if you want to access the TDengine service in the container from outside the container using TDengine CLI or various connectors for complex scenarios. diff --git a/docs/en/14-reference/_statsd.mdx b/docs/en/14-reference/_statsd.mdx index d0721a16fb..d068304a1d 100644 --- a/docs/en/14-reference/_statsd.mdx +++ b/docs/en/14-reference/_statsd.mdx @@ -51,5 +51,6 @@ port: 8125 Start StatsD after adding the following (assuming the config file is modified to config.js) ``` +npm install node stats.js config.js & ``` diff --git a/docs/en/20-third-party/02-prometheus.md b/docs/en/20-third-party/02-prometheus.md index 49d701bd20..ef9b9cb637 100644 --- a/docs/en/20-third-party/02-prometheus.md +++ b/docs/en/20-third-party/02-prometheus.md @@ -41,9 +41,9 @@ taos> use prometheus_data; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - metrics | 2022-04-20 07:21:09.209 | 2 | 1 | 1389 | + name | +================================= + metrics | Query OK, 1 row(s) in set (0.000487s) taos> select * from metrics limit 10; diff --git a/docs/en/20-third-party/03-telegraf.md b/docs/en/20-third-party/03-telegraf.md index ab06ab0379..8f3cab0e57 100644 --- a/docs/en/20-third-party/03-telegraf.md +++ b/docs/en/20-third-party/03-telegraf.md @@ -43,16 +43,16 @@ taos> use telegraf; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - swap | 2022-04-20 08:47:53.532 | 7 | 1 | 1 | - cpu | 2022-04-20 08:48:03.488 | 11 | 2 | 5 | - system | 2022-04-20 08:47:53.512 | 8 | 1 | 1 | - diskio | 2022-04-20 08:47:53.550 | 12 | 2 | 15 | - kernel | 2022-04-20 08:47:53.503 | 6 | 1 | 1 | - mem | 2022-04-20 08:47:53.521 | 35 | 1 | 1 | - processes | 2022-04-20 08:47:53.555 | 12 | 1 | 1 | - disk | 2022-04-20 08:47:53.541 | 8 | 5 | 2 | + name | +================================= + swap | + cpu | + system | + diskio | + kernel | + mem | + processes | + disk | Query OK, 8 row(s) in set (0.000521s) taos> select * from telegraf.system limit 10; diff --git a/docs/en/20-third-party/05-collectd.md b/docs/en/20-third-party/05-collectd.md index 295c08c307..5b52e3b7bc 100644 --- a/docs/en/20-third-party/05-collectd.md +++ b/docs/en/20-third-party/05-collectd.md @@ -32,28 +32,29 @@ Use the TDengine CLI to verify that collectd's data is written to TDengine and c ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - collectd | 2022-04-20 09:27:45.460 | 95 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.003266s) + name | +================================= + information_schema | + performance_schema | + collectd | +Query OK, 3 row(s) in set (0.003266s) taos> use collectd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - load_1 | 2022-04-20 09:27:45.492 | 2 | 2 | 1 | - memory_value | 2022-04-20 09:27:45.463 | 2 | 3 | 6 | - df_value | 2022-04-20 09:27:45.463 | 2 | 4 | 25 | - load_2 | 2022-04-20 09:27:45.501 | 2 | 2 | 1 | - load_0 | 2022-04-20 09:27:45.485 | 2 | 2 | 1 | - interface_1 | 2022-04-20 09:27:45.488 | 2 | 3 | 12 | - irq_value | 2022-04-20 09:27:45.476 | 2 | 3 | 31 | - interface_0 | 2022-04-20 09:27:45.480 | 2 | 3 | 12 | - entropy_value | 2022-04-20 09:27:45.473 | 2 | 2 | 1 | - swap_value | 2022-04-20 09:27:45.477 | 2 | 3 | 5 | + name | +================================= + load_1 | + memory_value | + df_value | + load_2 | + load_0 | + interface_1 | + irq_value | + interface_0 | + entropy_value | + swap_value | Query OK, 10 row(s) in set (0.002236s) taos> select * from collectd.memory_value limit 10; diff --git a/docs/en/20-third-party/06-statsd.md b/docs/en/20-third-party/06-statsd.md index 32b1bbb97a..b861a48ecd 100644 --- a/docs/en/20-third-party/06-statsd.md +++ b/docs/en/20-third-party/06-statsd.md @@ -26,7 +26,7 @@ Start StatsD: ``` $ node stats.js config.js & [1] 8546 -$ 20 Apr 09:54:41 - [8546] reading config file: exampleConfig.js +$ 20 Apr 09:54:41 - [8546] reading config file: config.js 20 Apr 09:54:41 - server is up INFO ``` @@ -40,19 +40,20 @@ Use the TDengine CLI to verify that StatsD data is written to TDengine and can r ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - statsd | 2022-04-20 09:54:51.220 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | -Query OK, 2 row(s) in set (0.003142s) + name | +================================= + information_schema | + performance_schema | + statsd | +Query OK, 3 row(s) in set (0.003142s) taos> use statsd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - foo | 2022-04-20 09:54:51.234 | 2 | 1 | 1 | + name | +================================= + foo | Query OK, 1 row(s) in set (0.002161s) taos> select * from foo; @@ -63,3 +64,8 @@ Query OK, 1 row(s) in set (0.004179s) taos> ``` + +:::note + +- TDengine will automatically create unique IDs for sub-table names by the rule. +::: diff --git a/docs/en/20-third-party/07-icinga2.md b/docs/en/20-third-party/07-icinga2.md index e98f209803..167b6a4303 100644 --- a/docs/en/20-third-party/07-icinga2.md +++ b/docs/en/20-third-party/07-icinga2.md @@ -47,30 +47,30 @@ taos> use icinga2; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - icinga.service.users.state_... | 2022-04-20 12:11:39.726 | 2 | 1 | 1 | - icinga.service.users.acknow... | 2022-04-20 12:11:39.756 | 2 | 1 | 1 | - icinga.service.procs.downti... | 2022-04-20 12:11:44.541 | 2 | 1 | 1 | - icinga.service.users.users | 2022-04-20 12:11:39.770 | 2 | 1 | 1 | - icinga.service.procs.procs_min | 2022-04-20 12:11:44.599 | 2 | 1 | 1 | - icinga.service.users.users_min | 2022-04-20 12:11:39.809 | 2 | 1 | 1 | - icinga.check.max_check_atte... | 2022-04-20 12:11:39.847 | 2 | 3 | 2 | - icinga.service.procs.state_... | 2022-04-20 12:11:44.522 | 2 | 1 | 1 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.576 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.796 | 2 | 1 | 1 | - icinga.check.latency | 2022-04-20 12:11:39.869 | 2 | 3 | 2 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.588 | 2 | 1 | 1 | - icinga.service.users.downti... | 2022-04-20 12:11:39.746 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.783 | 2 | 1 | 1 | - icinga.service.users.reachable | 2022-04-20 12:11:39.736 | 2 | 1 | 1 | - icinga.service.procs.procs | 2022-04-20 12:11:44.565 | 2 | 1 | 1 | - icinga.service.procs.acknow... | 2022-04-20 12:11:44.554 | 2 | 1 | 1 | - icinga.service.procs.state | 2022-04-20 12:11:44.509 | 2 | 1 | 1 | - icinga.service.procs.reachable | 2022-04-20 12:11:44.532 | 2 | 1 | 1 | - icinga.check.current_attempt | 2022-04-20 12:11:39.825 | 2 | 3 | 2 | - icinga.check.execution_time | 2022-04-20 12:11:39.898 | 2 | 3 | 2 | - icinga.service.users.state | 2022-04-20 12:11:39.704 | 2 | 1 | 1 | + name | +================================= + icinga.service.users.state_... | + icinga.service.users.acknow... | + icinga.service.procs.downti... | + icinga.service.users.users | + icinga.service.procs.procs_min | + icinga.service.users.users_min | + icinga.check.max_check_atte... | + icinga.service.procs.state_... | + icinga.service.procs.procs_... | + icinga.service.users.users_... | + icinga.check.latency | + icinga.service.procs.procs_... | + icinga.service.users.downti... | + icinga.service.users.users_... | + icinga.service.users.reachable | + icinga.service.procs.procs | + icinga.service.procs.acknow... | + icinga.service.procs.state | + icinga.service.procs.reachable | + icinga.check.current_attempt | + icinga.check.execution_time | + icinga.service.users.state | Query OK, 22 row(s) in set (0.002317s) ``` diff --git a/docs/en/20-third-party/08-tcollector.md b/docs/en/20-third-party/08-tcollector.md index 206b9dd23c..b604a2d712 100644 --- a/docs/en/20-third-party/08-tcollector.md +++ b/docs/en/20-third-party/08-tcollector.md @@ -44,26 +44,26 @@ taos> use tcollector; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - proc.meminfo.hugepages_rsvd | 2022-04-20 12:44:53.945 | 2 | 1 | 1 | - proc.meminfo.directmap1g | 2022-04-20 12:44:54.110 | 2 | 1 | 1 | - proc.meminfo.vmallocchunk | 2022-04-20 12:44:53.724 | 2 | 1 | 1 | - proc.meminfo.hugepagesize | 2022-04-20 12:44:54.004 | 2 | 1 | 1 | - tcollector.reader.lines_dro... | 2022-04-20 12:44:49.675 | 2 | 1 | 1 | - proc.meminfo.sunreclaim | 2022-04-20 12:44:53.437 | 2 | 1 | 1 | - proc.stat.ctxt | 2022-04-20 12:44:55.363 | 2 | 1 | 1 | - proc.meminfo.swaptotal | 2022-04-20 12:44:53.158 | 2 | 1 | 1 | - proc.uptime.total | 2022-04-20 12:44:52.813 | 2 | 1 | 1 | - tcollector.collector.lines_... | 2022-04-20 12:44:49.895 | 2 | 2 | 51 | - proc.meminfo.vmallocused | 2022-04-20 12:44:53.704 | 2 | 1 | 1 | - proc.meminfo.memavailable | 2022-04-20 12:44:52.939 | 2 | 1 | 1 | - sys.numa.foreign_allocs | 2022-04-20 12:44:57.929 | 2 | 2 | 1 | - proc.meminfo.committed_as | 2022-04-20 12:44:53.639 | 2 | 1 | 1 | - proc.vmstat.pswpin | 2022-04-20 12:44:54.177 | 2 | 1 | 1 | - proc.meminfo.cmafree | 2022-04-20 12:44:53.865 | 2 | 1 | 1 | - proc.meminfo.mapped | 2022-04-20 12:44:53.349 | 2 | 1 | 1 | - proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | + name | +================================= + proc.meminfo.hugepages_rsvd | + proc.meminfo.directmap1g | + proc.meminfo.vmallocchunk | + proc.meminfo.hugepagesize | + tcollector.reader.lines_dro... | + proc.meminfo.sunreclaim | + proc.stat.ctxt | + proc.meminfo.swaptotal | + proc.uptime.total | + tcollector.collector.lines_... | + proc.meminfo.vmallocused | + proc.meminfo.memavailable | + sys.numa.foreign_allocs | + proc.meminfo.committed_as | + proc.vmstat.pswpin | + proc.meminfo.cmafree | + proc.meminfo.mapped | + proc.vmstat.pgmajfault | ... ``` diff --git a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx index 58bca7f843..8d097e3f65 100644 --- a/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx +++ b/docs/zh/07-develop/03-insert-data/03-opentsdb-telnet.mdx @@ -66,10 +66,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-30 17:04:10.877 | 2 | 2 | 2 | - meters.voltage | 2022-03-30 17:04:10.882 | 2 | 2 | 2 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.002544s) taos> select tbname, * from `meters.current`; diff --git a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx index aa3e5980cf..e2e7d7c8fa 100644 --- a/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx +++ b/docs/zh/07-develop/03-insert-data/04-opentsdb-json.mdx @@ -82,10 +82,10 @@ taos> use test; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - meters.current | 2022-03-29 16:05:25.193 | 2 | 2 | 1 | - meters.voltage | 2022-03-29 16:05:25.200 | 2 | 2 | 1 | + name | +================================= + meters.current | + meters.voltage | Query OK, 2 row(s) in set (0.001954s) taos> select * from `meters.current`; diff --git a/docs/zh/08-connector/_verify_linux.mdx b/docs/zh/08-connector/_verify_linux.mdx index 28957e77aa..4137b2880b 100644 --- a/docs/zh/08-connector/_verify_linux.mdx +++ b/docs/zh/08-connector/_verify_linux.mdx @@ -4,11 +4,11 @@ $ taos taos> show databases; - name | create_time | vgroups | ntables | replica | strict | duration | keep | buffer | pagesize | pages | minrows | maxrows | comp | precision | status | retention | single_stable | cachemodel | cachesize | wal_level | wal_fsync_period | wal_retention_period | wal_retention_size | wal_roll_period | wal_seg_size | -========================================================================================================================================================================================================================================================================================================================================================================================================================================================================= - information_schema | NULL | NULL | 14 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - performance_schema | NULL | NULL | 3 | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | ready | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | NULL | - db | 2022-08-04 14:14:49.385 | 2 | 4 | 1 | off | 14400m | 5254560m,5254560m,5254560m | 96 | 4 | 256 | 100 | 4096 | 2 | ms | ready | NULL | false | none | 1 | 1 | 3000 | 0 | 0 | 0 | 0 | + name | +================================= + information_schema | + performance_schema | + db | Query OK, 3 rows in database (0.019154s) taos> diff --git a/docs/zh/14-reference/_statsd.mdx b/docs/zh/14-reference/_statsd.mdx index b225c44267..dc4f19b37c 100644 --- a/docs/zh/14-reference/_statsd.mdx +++ b/docs/zh/14-reference/_statsd.mdx @@ -51,5 +51,6 @@ port: 8125 增加如下内容后启动 StatsD（假设配置文件修改为 config.js）。 ``` +npm install node stats.js config.js & ``` diff --git a/docs/zh/20-third-party/02-prometheus.md b/docs/zh/20-third-party/02-prometheus.md index 8b8d078792..0f95053a78 100644 --- a/docs/zh/20-third-party/02-prometheus.md +++ b/docs/zh/20-third-party/02-prometheus.md @@ -40,9 +40,9 @@ taos> use prometheus_data; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - metrics | 2022-04-20 07:21:09.209 | 2 | 1 | 1389 | + name | +================================= + metrics | Query OK, 1 row(s) in set (0.000487s) taos> select * from metrics limit 10; diff --git a/docs/zh/20-third-party/03-telegraf.md b/docs/zh/20-third-party/03-telegraf.md index 71bb1b3885..bb688a6ab6 100644 --- a/docs/zh/20-third-party/03-telegraf.md +++ b/docs/zh/20-third-party/03-telegraf.md @@ -44,16 +44,16 @@ taos> use telegraf; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - swap | 2022-04-20 08:47:53.532 | 7 | 1 | 1 | - cpu | 2022-04-20 08:48:03.488 | 11 | 2 | 5 | - system | 2022-04-20 08:47:53.512 | 8 | 1 | 1 | - diskio | 2022-04-20 08:47:53.550 | 12 | 2 | 15 | - kernel | 2022-04-20 08:47:53.503 | 6 | 1 | 1 | - mem | 2022-04-20 08:47:53.521 | 35 | 1 | 1 | - processes | 2022-04-20 08:47:53.555 | 12 | 1 | 1 | - disk | 2022-04-20 08:47:53.541 | 8 | 5 | 2 | + name | +================================= + swap | + cpu | + system | + diskio | + kernel | + mem | + processes | + disk | Query OK, 8 row(s) in set (0.000521s) taos> select * from telegraf.system limit 10; diff --git a/docs/zh/20-third-party/05-collectd.md b/docs/zh/20-third-party/05-collectd.md index d55b981fb9..bda66b3cce 100644 --- a/docs/zh/20-third-party/05-collectd.md +++ b/docs/zh/20-third-party/05-collectd.md @@ -32,28 +32,29 @@ sudo systemctl restart collectd ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - collectd | 2022-04-20 09:27:45.460 | 95 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | -Query OK, 2 row(s) in set (0.003266s) + name | +================================= + information_schema | + performance_schema | + collectd | +Query OK, 3 row(s) in set (0.003266s) taos> use collectd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - load_1 | 2022-04-20 09:27:45.492 | 2 | 2 | 1 | - memory_value | 2022-04-20 09:27:45.463 | 2 | 3 | 6 | - df_value | 2022-04-20 09:27:45.463 | 2 | 4 | 25 | - load_2 | 2022-04-20 09:27:45.501 | 2 | 2 | 1 | - load_0 | 2022-04-20 09:27:45.485 | 2 | 2 | 1 | - interface_1 | 2022-04-20 09:27:45.488 | 2 | 3 | 12 | - irq_value | 2022-04-20 09:27:45.476 | 2 | 3 | 31 | - interface_0 | 2022-04-20 09:27:45.480 | 2 | 3 | 12 | - entropy_value | 2022-04-20 09:27:45.473 | 2 | 2 | 1 | - swap_value | 2022-04-20 09:27:45.477 | 2 | 3 | 5 | + name | +================================= + load_1 | + memory_value | + df_value | + load_2 | + load_0 | + interface_1 | + irq_value | + interface_0 | + entropy_value | + swap_value | Query OK, 10 row(s) in set (0.002236s) taos> select * from collectd.memory_value limit 10; diff --git a/docs/zh/20-third-party/06-statsd.md b/docs/zh/20-third-party/06-statsd.md index 122c9fd94c..20b4d800be 100644 --- a/docs/zh/20-third-party/06-statsd.md +++ b/docs/zh/20-third-party/06-statsd.md @@ -27,7 +27,7 @@ StatsD 是汇总和总结应用指标的一个简单的守护进程，近些年 ``` $ node stats.js config.js & [1] 8546 -$ 20 Apr 09:54:41 - [8546] reading config file: exampleConfig.js +$ 20 Apr 09:54:41 - [8546] reading config file: config.js 20 Apr 09:54:41 - server is up INFO ``` @@ -41,19 +41,20 @@ $ echo "foo:1|c" | nc -u -w0 127.0.0.1 8125 ``` taos> show databases; - name | created_time | ntables | vgroups | replica | quorum | days | keep | cache(MB) | blocks | minrows | maxrows | wallevel | fsync | comp | cachelast | precision | update | status | -==================================================================================================================================================================================================================================================================================== - log | 2022-04-20 07:19:50.260 | 11 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ms | 0 | ready | - statsd | 2022-04-20 09:54:51.220 | 1 | 1 | 1 | 1 | 10 | 3650 | 16 | 6 | 100 | 4096 | 1 | 3000 | 2 | 0 | ns | 2 | ready | -Query OK, 2 row(s) in set (0.003142s) + name | +================================= + information_schema | + performance_schema | + statsd | +Query OK, 3 row(s) in set (0.003142s) taos> use statsd; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - foo | 2022-04-20 09:54:51.234 | 2 | 1 | 1 | + name | +================================= + foo | Query OK, 1 row(s) in set (0.002161s) taos> select * from foo; @@ -64,3 +65,8 @@ Query OK, 1 row(s) in set (0.004179s) taos> ``` + +:::note + +- TDengine will automatically create unique IDs for sub-table names by the rule. +::: diff --git a/docs/zh/20-third-party/07-icinga2.md b/docs/zh/20-third-party/07-icinga2.md index 03d733a326..d112a7a5e6 100644 --- a/docs/zh/20-third-party/07-icinga2.md +++ b/docs/zh/20-third-party/07-icinga2.md @@ -48,30 +48,30 @@ taos> use icinga2; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - icinga.service.users.state_... | 2022-04-20 12:11:39.726 | 2 | 1 | 1 | - icinga.service.users.acknow... | 2022-04-20 12:11:39.756 | 2 | 1 | 1 | - icinga.service.procs.downti... | 2022-04-20 12:11:44.541 | 2 | 1 | 1 | - icinga.service.users.users | 2022-04-20 12:11:39.770 | 2 | 1 | 1 | - icinga.service.procs.procs_min | 2022-04-20 12:11:44.599 | 2 | 1 | 1 | - icinga.service.users.users_min | 2022-04-20 12:11:39.809 | 2 | 1 | 1 | - icinga.check.max_check_atte... | 2022-04-20 12:11:39.847 | 2 | 3 | 2 | - icinga.service.procs.state_... | 2022-04-20 12:11:44.522 | 2 | 1 | 1 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.576 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.796 | 2 | 1 | 1 | - icinga.check.latency | 2022-04-20 12:11:39.869 | 2 | 3 | 2 | - icinga.service.procs.procs_... | 2022-04-20 12:11:44.588 | 2 | 1 | 1 | - icinga.service.users.downti... | 2022-04-20 12:11:39.746 | 2 | 1 | 1 | - icinga.service.users.users_... | 2022-04-20 12:11:39.783 | 2 | 1 | 1 | - icinga.service.users.reachable | 2022-04-20 12:11:39.736 | 2 | 1 | 1 | - icinga.service.procs.procs | 2022-04-20 12:11:44.565 | 2 | 1 | 1 | - icinga.service.procs.acknow... | 2022-04-20 12:11:44.554 | 2 | 1 | 1 | - icinga.service.procs.state | 2022-04-20 12:11:44.509 | 2 | 1 | 1 | - icinga.service.procs.reachable | 2022-04-20 12:11:44.532 | 2 | 1 | 1 | - icinga.check.current_attempt | 2022-04-20 12:11:39.825 | 2 | 3 | 2 | - icinga.check.execution_time | 2022-04-20 12:11:39.898 | 2 | 3 | 2 | - icinga.service.users.state | 2022-04-20 12:11:39.704 | 2 | 1 | 1 | + name | +================================= + icinga.service.users.state_... | + icinga.service.users.acknow... | + icinga.service.procs.downti... | + icinga.service.users.users | + icinga.service.procs.procs_min | + icinga.service.users.users_min | + icinga.check.max_check_atte... | + icinga.service.procs.state_... | + icinga.service.procs.procs_... | + icinga.service.users.users_... | + icinga.check.latency | + icinga.service.procs.procs_... | + icinga.service.users.downti... | + icinga.service.users.users_... | + icinga.service.users.reachable | + icinga.service.procs.procs | + icinga.service.procs.acknow... | + icinga.service.procs.state | + icinga.service.procs.reachable | + icinga.check.current_attempt | + icinga.check.execution_time | + icinga.service.users.state | Query OK, 22 row(s) in set (0.002317s) ``` diff --git a/docs/zh/20-third-party/08-tcollector.md b/docs/zh/20-third-party/08-tcollector.md index b79e3ed2d1..a6ed0f89e2 100644 --- a/docs/zh/20-third-party/08-tcollector.md +++ b/docs/zh/20-third-party/08-tcollector.md @@ -46,26 +46,26 @@ taos> use tcollector; Database changed. taos> show stables; - name | created_time | columns | tags | tables | -============================================================================================ - proc.meminfo.hugepages_rsvd | 2022-04-20 12:44:53.945 | 2 | 1 | 1 | - proc.meminfo.directmap1g | 2022-04-20 12:44:54.110 | 2 | 1 | 1 | - proc.meminfo.vmallocchunk | 2022-04-20 12:44:53.724 | 2 | 1 | 1 | - proc.meminfo.hugepagesize | 2022-04-20 12:44:54.004 | 2 | 1 | 1 | - tcollector.reader.lines_dro... | 2022-04-20 12:44:49.675 | 2 | 1 | 1 | - proc.meminfo.sunreclaim | 2022-04-20 12:44:53.437 | 2 | 1 | 1 | - proc.stat.ctxt | 2022-04-20 12:44:55.363 | 2 | 1 | 1 | - proc.meminfo.swaptotal | 2022-04-20 12:44:53.158 | 2 | 1 | 1 | - proc.uptime.total | 2022-04-20 12:44:52.813 | 2 | 1 | 1 | - tcollector.collector.lines_... | 2022-04-20 12:44:49.895 | 2 | 2 | 51 | - proc.meminfo.vmallocused | 2022-04-20 12:44:53.704 | 2 | 1 | 1 | - proc.meminfo.memavailable | 2022-04-20 12:44:52.939 | 2 | 1 | 1 | - sys.numa.foreign_allocs | 2022-04-20 12:44:57.929 | 2 | 2 | 1 | - proc.meminfo.committed_as | 2022-04-20 12:44:53.639 | 2 | 1 | 1 | - proc.vmstat.pswpin | 2022-04-20 12:44:54.177 | 2 | 1 | 1 | - proc.meminfo.cmafree | 2022-04-20 12:44:53.865 | 2 | 1 | 1 | - proc.meminfo.mapped | 2022-04-20 12:44:53.349 | 2 | 1 | 1 | - proc.vmstat.pgmajfault | 2022-04-20 12:44:54.251 | 2 | 1 | 1 | + name | +================================= + proc.meminfo.hugepages_rsvd | + proc.meminfo.directmap1g | + proc.meminfo.vmallocchunk | + proc.meminfo.hugepagesize | + tcollector.reader.lines_dro... | + proc.meminfo.sunreclaim | + proc.stat.ctxt | + proc.meminfo.swaptotal | + proc.uptime.total | + tcollector.collector.lines_... | + proc.meminfo.vmallocused | + proc.meminfo.memavailable | + sys.numa.foreign_allocs | + proc.meminfo.committed_as | + proc.vmstat.pswpin | + proc.meminfo.cmafree | + proc.meminfo.mapped | + proc.vmstat.pgmajfault | ... ``` From c9286fed25225b40e9b2184f3c080d5900c376e5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Sep 2022 13:03:25 +0800 Subject: [PATCH 56/64] fix(query): fix memory leak. --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/tsdb/tsdbCacheRead.c | 4 ++-- source/libs/executor/src/cachescanoperator.c | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6ba10641f5..3e086de097 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -153,7 +153,7 @@ uint64_t getReaderMaxVersion(STsdbReader *pReader); int32_t tsdbCacherowsReaderOpen(void *pVnode, int32_t type, SArray *pTableIdList, int32_t numOfCols, void **pReader); int32_t tsdbRetrieveCacheRows(void *pReader, SSDataBlock *pResBlock, const int32_t *slotIds, SArray *pTableUids); -int32_t tsdbCacherowsReaderClose(void *pReader); +void* tsdbCacherowsReaderClose(void *pReader); int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid); void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 5f981649f3..7a000253d5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -101,7 +101,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, SArray* pTableIdList return TSDB_CODE_SUCCESS; } -int32_t tsdbCacherowsReaderClose(void* pReader) { +void* tsdbCacherowsReaderClose(void* pReader) { SCacheRowsReader* p = pReader; if (p->pSchema != NULL) { @@ -114,7 +114,7 @@ int32_t tsdbCacherowsReaderClose(void* pReader) { } taosMemoryFree(pReader); - return TSDB_CODE_SUCCESS; + return NULL; } static int32_t doExtractCacheRow(SCacheRowsReader* pr, SLRUCache* lruCache, uint64_t uid, STSRow** pRow, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 94d9d0cadb..1b7448712e 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -207,7 +207,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } } - tsdbCacherowsReaderClose(pInfo->pLastrowReader); + pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); return pInfo->pRes; } } @@ -220,6 +220,15 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { void destroyLastrowScanOperator(void* param) { SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; blockDataDestroy(pInfo->pRes); + blockDataDestroy(pInfo->pBufferredRes); + taosMemoryFree(pInfo->pSlotIds); + taosArrayDestroy(pInfo->pUidList); + taosArrayDestroy(pInfo->pColMatchInfo); + + if (pInfo->pLastrowReader != NULL) { + pInfo->pLastrowReader = tsdbCacherowsReaderClose(pInfo->pLastrowReader); + } + taosMemoryFreeClear(param); } From dfb2b0678f2faae8a717aa3df183fdb68f77bc50 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 30 Sep 2022 14:03:46 +0800 Subject: [PATCH 57/64] fix: coverity issues --- source/dnode/mnode/impl/src/mndDef.c | 20 +++++++++++-- source/dnode/mnode/impl/src/mndStb.c | 22 +++++++-------- source/dnode/mnode/impl/src/mndStream.c | 37 ++++++++++++------------- source/dnode/vnode/src/vnd/vnodeSync.c | 12 ++++---- 4 files changed, 51 insertions(+), 40 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 50d26ce9a5..42e56cd488 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -42,9 +42,23 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI64(pEncoder, pObj->targetStbUid) < 0) return -1; if (tEncodeI32(pEncoder, pObj->fixedSinkVgId) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; - if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + if (pObj->sql != NULL) { + if (tEncodeCStr(pEncoder, pObj->sql) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } + + if (pObj->ast != NULL) { + if (tEncodeCStr(pEncoder, pObj->ast) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } + + if (pObj->physicalPlan != NULL) { + if (tEncodeCStr(pEncoder, pObj->physicalPlan) < 0) return -1; + } else { + if (tEncodeCStr(pEncoder, "") < 0) return -1; + } int32_t sz = taosArrayGetSize(pObj->tasks); if (tEncodeI32(pEncoder, sz) < 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 38c7850e6c..8cd8431c34 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -554,7 +554,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfColumns; ++i) { SField *pField1 = taosArrayGet(pCreate->pColumns, i); - if (pField1->type < 0) { + if (pField1->type >= TSDB_DATA_TYPE_MAX) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -570,7 +570,7 @@ int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { for (int32_t i = 0; i < pCreate->numOfTags; ++i) { SField *pField1 = taosArrayGet(pCreate->pTags, i); - if (pField1->type < 0) { + if (pField1->type >= TSDB_DATA_TYPE_MAX) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; } @@ -982,8 +982,8 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } } else { - mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid", createReq.name, - createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); + mError("stb:%s, already exist while create, input tagVer:%d colVer:%d is invalid, origin tagVer:%d colVer:%d", + createReq.name, createReq.tagVer, createReq.colVer, pStb->tagVer, pStb->colVer); terrno = TSDB_CODE_MND_INVALID_SCHEMA_VER; goto _OVER; } @@ -1603,9 +1603,9 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return -1; } - strcpy(pRsp->dbFName, pStb->db); - strcpy(pRsp->tbName, tbName); - strcpy(pRsp->stbName, tbName); + tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); + tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName)); + tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName)); pRsp->dbId = pDb->uid; pRsp->numOfTags = pStb->numOfTags; pRsp->numOfColumns = pStb->numOfColumns; @@ -1649,9 +1649,9 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, return -1; } - strcpy(pRsp->dbFName, pStb->db); - strcpy(pRsp->tbName, tbName); - strcpy(pRsp->stbName, tbName); + tstrncpy(pRsp->dbFName, pStb->db, sizeof(pRsp->dbFName)); + tstrncpy(pRsp->tbName, tbName, sizeof(pRsp->tbName)); + tstrncpy(pRsp->stbName, tbName, sizeof(pRsp->stbName)); pRsp->numOfTags = pStb->numOfTags; pRsp->numOfColumns = pStb->numOfColumns; pRsp->tableType = TSDB_SUPER_TABLE; @@ -2551,7 +2551,7 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)maxDelay, false); - char rollup[128 + VARSTR_HEADER_SIZE] = {0}; + char rollup[160 + VARSTR_HEADER_SIZE] = {0}; int32_t rollupNum = (int32_t)taosArrayGetSize(pStb->pFuncs); for (int32_t i = 0; i < rollupNum; ++i) { char *funcName = taosArrayGet(pStb->pFuncs, i); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index c6f46b28fc..03d7d8e825 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -425,8 +425,10 @@ static int32_t mndSetStreamRecover(SMnode *pMnode, STrans *pTrans, const SStream SStreamObj streamObj = {0}; memcpy(streamObj.name, pStream->name, TSDB_STREAM_FNAME_LEN); streamObj.status = STREAM_STATUS__RECOVER; + SSdbRaw *pCommitRaw = mndStreamActionEncode(&streamObj); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { mError("stream trans:%d, failed to append commit log since %s", pTrans->id, terrstr()); mndTransDrop(pTrans); return -1; @@ -771,12 +773,14 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); return -1; } // drop stream if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); return -1; } @@ -945,10 +949,8 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SName n; int32_t cols = 0; - char streamName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->name, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(streamName)); - varDataSetLen(streamName, strlen(varDataVal(streamName))); + char streamName[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(streamName, mndGetDbStr(pStream->name), sizeof(streamName)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)streamName, false); @@ -956,28 +958,24 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(&sql[VARSTR_HEADER_SIZE], pStream->sql, TSDB_SHOW_SQL_LEN); - varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); + STR_WITH_MAXSIZE_TO_VARSTR(streamName, pStream->sql, sizeof(sql)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); char status[20 + VARSTR_HEADER_SIZE] = {0}; - mndShowStreamStatus(&status[VARSTR_HEADER_SIZE], pStream); - varDataSetLen(status, strlen(varDataVal(status))); + char status2[20] = {0}; + mndShowStreamStatus(status2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(status, status2, sizeof(status)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&status, false); char sourceDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->sourceDb, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(sourceDB)); - varDataSetLen(sourceDB, strlen(varDataVal(sourceDB))); + STR_WITH_MAXSIZE_TO_VARSTR(sourceDB, mndGetDbStr(pStream->sourceDb), sizeof(sourceDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&sourceDB, false); char targetDB[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetDb, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&n, varDataVal(targetDB)); - varDataSetLen(targetDB, strlen(varDataVal(targetDB))); + STR_WITH_MAXSIZE_TO_VARSTR(targetDB, mndGetDbStr(pStream->targetDb), sizeof(targetDB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetDB, false); @@ -986,9 +984,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, NULL, true); } else { char targetSTB[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&n, pStream->targetSTbName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - strcpy(&targetSTB[VARSTR_HEADER_SIZE], tNameGetTableName(&n)); - varDataSetLen(targetSTB, strlen(varDataVal(targetSTB))); + STR_WITH_MAXSIZE_TO_VARSTR(targetSTB, mndGetStbStr(pStream->targetSTbName), sizeof(targetSTB)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&targetSTB, false); } @@ -997,8 +993,9 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->watermark, false); char trigger[20 + VARSTR_HEADER_SIZE] = {0}; - mndShowStreamTrigger(&trigger[VARSTR_HEADER_SIZE], pStream); - varDataSetLen(trigger, strlen(varDataVal(trigger))); + char trigger2[20] = {0}; + mndShowStreamTrigger(trigger2, pStream); + STR_WITH_MAXSIZE_TO_VARSTR(trigger, trigger2, sizeof(trigger)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&trigger, false); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index aa3f72d281..f773de5280 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -328,14 +328,14 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { const STraceId *trace = &pMsg->info.traceId; if (!syncEnvIsStart()) { - vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId); + vGError("vgId:%d, msg:%p failed to process since sync env not start", pVnode->config.vgId, pMsg); terrno = TSDB_CODE_APP_ERROR; return -1; } SSyncNode *pSyncNode = syncNodeAcquire(pVnode->sync); if (pSyncNode == NULL) { - vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId); + vGError("vgId:%d, msg:%p failed to process since invalid sync node", pVnode->config.vgId, pMsg); terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -394,7 +394,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); + vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); code = -1; } @@ -459,7 +459,7 @@ int32_t vnodeProcessSyncMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SRpcMsg rsp = {.code = code, .info = pMsg->info}; tmsgSendRsp(&rsp); } else { - vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg->msgType); + vGError("vgId:%d, msg:%p failed to process since error msg type:%d", pVnode->config.vgId, pMsg, pMsg->msgType); code = -1; } } @@ -630,7 +630,7 @@ static int32_t vnodeSnapshotStartWrite(struct SSyncFSM *pFsm, void *pParam, void vInfo("vgId:%d, start write vnode snapshot since apply queue is empty", pVnode->config.vgId); break; } else { - vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId); + vInfo("vgId:%d, write vnode snapshot later since %d items in apply queue", pVnode->config.vgId, itemSize); taosMsleep(10); } } while (true); @@ -683,7 +683,7 @@ static void vnodeRestoreFinish(struct SSyncFSM *pFsm) { vInfo("vgId:%d, apply queue is empty, restore finish", pVnode->config.vgId); break; } else { - vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId); + vInfo("vgId:%d, restore not finish since %d items in apply queue", pVnode->config.vgId, itemSize); taosMsleep(10); } } while (true); From 3ed27805c37d5bbb2ca15d6b44700289444d0204 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 30 Sep 2022 14:43:51 +0800 Subject: [PATCH 58/64] fix: query information_schema.ins_tags error --- source/libs/parser/inc/parUtil.h | 1 + source/libs/parser/src/parAstParser.c | 21 +++++ source/libs/parser/src/parTranslater.c | 110 +++++++++++----------- source/libs/parser/src/parUtil.c | 67 ++++++++++++- source/libs/parser/test/parSelectTest.cpp | 6 +- source/libs/parser/test/parTestMain.cpp | 2 +- source/libs/planner/test/planTestMain.cpp | 2 +- 7 files changed, 149 insertions(+), 60 deletions(-) diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 308afd467f..75b631ec9d 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -91,6 +91,7 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); +int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName); int32_t buildCatalogReq(SParseContext* pCxt, const SParseMetaCache* pMetaCache, SCatalogReq* pCatalogReq); int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SParseMetaCache* pMetaCache, diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index ce90bd3da0..27adb79e3c 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -125,6 +125,23 @@ static bool needGetTableIndex(SNode* pStmt) { return false; } +static int32_t collectMetaKeyFromInsTagsImpl(SCollectMetaKeyCxt* pCxt, SName* pName) { + if (TSDB_DB_NAME_T == pName->type) { + return reserveDbVgInfoInCache(pName->acctId, pName->dbname, pCxt->pMetaCache); + } + return reserveTableVgroupInCacheExt(pName, pCxt->pMetaCache); +} + +static int32_t collectMetaKeyFromInsTags(SCollectMetaKeyCxt* pCxt) { + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pStmt; + SName name = {0}; + int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &name); + if (TSDB_CODE_SUCCESS == code) { + code = collectMetaKeyFromInsTagsImpl(pCxt, &name); + } + return code; +} + static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const char* pDb, const char* pTable, AUTH_TYPE authType) { int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); @@ -143,6 +160,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES))) { code = reserveDnodeRequiredInCache(pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pTable, TSDB_INS_TABLE_TAGS)) && + QUERY_NODE_SELECT_STMT == nodeType(pCxt->pStmt)) { + code = collectMetaKeyFromInsTags(pCxt); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 9d805b8e33..b58626e6fb 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2198,70 +2198,70 @@ static int32_t getTagsTableVgroupListImpl(STranslateContext* pCxt, SName* pTarge return code; } -static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) { - if (OP_TYPE_EQUAL != pOper->opType) { - return TSDB_CODE_SUCCESS; - } +// static int32_t getTagsTableTargetNameFromOp(STranslateContext* pCxt, SOperatorNode* pOper, SName* pName) { +// if (OP_TYPE_EQUAL != pOper->opType) { +// return TSDB_CODE_SUCCESS; +// } - SColumnNode* pCol = NULL; - SValueNode* pVal = NULL; - if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { - pCol = (SColumnNode*)pOper->pLeft; - } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { - pVal = (SValueNode*)pOper->pLeft; - } - if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { - pCol = (SColumnNode*)pOper->pRight; - } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { - pVal = (SValueNode*)pOper->pRight; - } - if (NULL == pCol || NULL == pVal) { - return TSDB_CODE_SUCCESS; - } +// SColumnNode* pCol = NULL; +// SValueNode* pVal = NULL; +// if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { +// pCol = (SColumnNode*)pOper->pLeft; +// } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { +// pVal = (SValueNode*)pOper->pLeft; +// } +// if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { +// pCol = (SColumnNode*)pOper->pRight; +// } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { +// pVal = (SValueNode*)pOper->pRight; +// } +// if (NULL == pCol || NULL == pVal) { +// return TSDB_CODE_SUCCESS; +// } - if (0 == strcmp(pCol->colName, "db_name")) { - return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal)); - } else if (0 == strcmp(pCol->colName, "table_name")) { - return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); - } +// if (0 == strcmp(pCol->colName, "db_name")) { +// return tNameSetDbName(pName, pCxt->pParseCxt->acctId, pVal->literal, strlen(pVal->literal)); +// } else if (0 == strcmp(pCol->colName, "table_name")) { +// return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); +// } - return TSDB_CODE_SUCCESS; -} +// return TSDB_CODE_SUCCESS; +// } -static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) { - if (QUERY_NODE_OPERATOR == nodeType(pNode)) { - getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName); - } -} +// static void getTagsTableTargetObjName(STranslateContext* pCxt, SNode* pNode, SName* pName) { +// if (QUERY_NODE_OPERATOR == nodeType(pNode)) { +// getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pNode, pName); +// } +// } -static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) { - if (LOGIC_COND_TYPE_AND != pCond->condType) { - return TSDB_CODE_SUCCESS; - } +// static int32_t getTagsTableTargetNameFromCond(STranslateContext* pCxt, SLogicConditionNode* pCond, SName* pName) { +// if (LOGIC_COND_TYPE_AND != pCond->condType) { +// return TSDB_CODE_SUCCESS; +// } - SNode* pNode = NULL; - FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } - if ('\0' == pName->dbname[0]) { - pName->type = 0; - } - return TSDB_CODE_SUCCESS; -} +// SNode* pNode = NULL; +// FOREACH(pNode, pCond->pParameterList) { getTagsTableTargetObjName(pCxt, pNode, pName); } +// if ('\0' == pName->dbname[0]) { +// pName->type = 0; +// } +// return TSDB_CODE_SUCCESS; +// } -static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) { - if (NULL == pWhere) { - return TSDB_CODE_SUCCESS; - } +// static int32_t getTagsTableTargetName(STranslateContext* pCxt, SNode* pWhere, SName* pName) { +// if (NULL == pWhere) { +// return TSDB_CODE_SUCCESS; +// } - if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { - return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName); - } +// if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { +// return getTagsTableTargetNameFromOp(pCxt, (SOperatorNode*)pWhere, pName); +// } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { - return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName); - } +// if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { +// return getTagsTableTargetNameFromCond(pCxt, (SLogicConditionNode*)pWhere, pName); +// } - return TSDB_CODE_SUCCESS; -} +// return TSDB_CODE_SUCCESS; +// } static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SArray** pVgroupList) { if (!isSelectStmt(pCxt->pCurrStmt)) { @@ -2269,7 +2269,7 @@ static int32_t getTagsTableVgroupList(STranslateContext* pCxt, SName* pName, SAr } SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SName targetName = {0}; - int32_t code = getTagsTableTargetName(pCxt, pSelect->pWhere, &targetName); + int32_t code = getInsTagsTableTargetName(pCxt->pParseCxt->acctId, pSelect->pWhere, &targetName); if (TSDB_CODE_SUCCESS == code) { code = getTagsTableVgroupListImpl(pCxt, &targetName, pName, pVgroupList); } diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 3075be1a00..c6a51f22e8 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -420,6 +420,71 @@ end: return retCode; } +static int32_t getInsTagsTableTargetNameFromOp(int32_t acctId, SOperatorNode* pOper, SName* pName) { + if (OP_TYPE_EQUAL != pOper->opType) { + return TSDB_CODE_SUCCESS; + } + + SColumnNode* pCol = NULL; + SValueNode* pVal = NULL; + if (QUERY_NODE_COLUMN == nodeType(pOper->pLeft)) { + pCol = (SColumnNode*)pOper->pLeft; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pLeft)) { + pVal = (SValueNode*)pOper->pLeft; + } + if (QUERY_NODE_COLUMN == nodeType(pOper->pRight)) { + pCol = (SColumnNode*)pOper->pRight; + } else if (QUERY_NODE_VALUE == nodeType(pOper->pRight)) { + pVal = (SValueNode*)pOper->pRight; + } + if (NULL == pCol || NULL == pVal) { + return TSDB_CODE_SUCCESS; + } + + if (0 == strcmp(pCol->colName, "db_name")) { + return tNameSetDbName(pName, acctId, pVal->literal, strlen(pVal->literal)); + } else if (0 == strcmp(pCol->colName, "table_name")) { + return tNameAddTbName(pName, pVal->literal, strlen(pVal->literal)); + } + + return TSDB_CODE_SUCCESS; +} + +static void getInsTagsTableTargetObjName(int32_t acctId, SNode* pNode, SName* pName) { + if (QUERY_NODE_OPERATOR == nodeType(pNode)) { + getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pNode, pName); + } +} + +static int32_t getInsTagsTableTargetNameFromCond(int32_t acctId, SLogicConditionNode* pCond, SName* pName) { + if (LOGIC_COND_TYPE_AND != pCond->condType) { + return TSDB_CODE_SUCCESS; + } + + SNode* pNode = NULL; + FOREACH(pNode, pCond->pParameterList) { getInsTagsTableTargetObjName(acctId, pNode, pName); } + if ('\0' == pName->dbname[0]) { + pName->type = 0; + } + return TSDB_CODE_SUCCESS; +} + +int32_t getInsTagsTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) { + if (NULL == pWhere) { + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_OPERATOR == nodeType(pWhere)) { + return getInsTagsTableTargetNameFromOp(acctId, (SOperatorNode*)pWhere, pName); + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(pWhere)) { + return getInsTagsTableTargetNameFromCond(acctId, (SLogicConditionNode*)pWhere, pName); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) { return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type); } @@ -1173,7 +1238,7 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) { taosArrayDestroy(p->pTableVgroupReq); p = taosHashIterate(pMetaCache->pInsertTables, p); - } + } taosHashCleanup(pMetaCache->pInsertTables); taosHashCleanup(pMetaCache->pDbVgroup); taosHashCleanup(pMetaCache->pDbCfg); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 79a7038401..0027ac9ca1 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -420,9 +420,11 @@ TEST_F(ParserSelectTest, setOperatorSemanticCheck) { } TEST_F(ParserSelectTest, informationSchema) { - useDb("root", "test"); + useDb("root", "information_schema"); - run("SELECT * FROM information_schema.ins_databases WHERE name = 'information_schema'"); + run("SELECT * FROM ins_databases WHERE name = 'information_schema'"); + + run("SELECT * FROM ins_tags WHERE db_name = 'test' and table_name = 'st1'"); } TEST_F(ParserSelectTest, withoutFrom) { diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 9e67249238..de2ce55459 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -53,7 +53,7 @@ class ParserEnv : public testing::Environment { private: void initLog(const char* path) { - int32_t logLevel = getLogLevel(); + int32_t logLevel = getLogLevel() | DEBUG_SCREEN; dDebugFlag = logLevel; vDebugFlag = logLevel; mDebugFlag = logLevel; diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index df6e72ce46..4e013c44b8 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -48,7 +48,7 @@ class PlannerEnv : public testing::Environment { private: void initLog(const char* path) { - int32_t logLevel = getLogLevel(); + int32_t logLevel = getLogLevel() | DEBUG_SCREEN; dDebugFlag = logLevel; vDebugFlag = logLevel; mDebugFlag = logLevel; From 6faf1af5f5f3c610b97827465bcaa1147380a480 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Sep 2022 14:44:42 +0800 Subject: [PATCH 59/64] fix(query): handle the block read case. --- source/dnode/vnode/src/tsdb/tsdbRead.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index c3cb5f9eb8..de64b8a61e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -926,6 +926,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn } } + // fill the mis-matched columns with null value while (i < numOfOutputCols) { pColData = taosArrayGet(pResBlock->pDataBlock, i); colDataAppendNNULL(pColData, 0, remain); @@ -935,12 +936,15 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pResBlock->info.rows = remain; pDumpInfo->rowIndex += step * remain; + // check if current block are all handled if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { -// int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; -// setBlockAllDumped(pDumpInfo, ts, pReader->order); + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + if (outOfTimeWindow(ts, &pReader->window)) { // the remain data has out of query time window, ignore current block + setBlockAllDumped(pDumpInfo, ts, pReader->order); + } } else { - int64_t k = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; - setBlockAllDumped(pDumpInfo, k, pReader->order); + int64_t ts = asc ? pBlock->maxKey.ts : pBlock->minKey.ts; + setBlockAllDumped(pDumpInfo, ts, pReader->order); } double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; From 31e424d5aba662b2922fe3a53d2b5dd4ba3c8f2a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 30 Sep 2022 14:59:31 +0800 Subject: [PATCH 60/64] fix: coverity issues --- source/dnode/mnode/impl/src/mndStream.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 03d7d8e825..ea889e7001 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -958,7 +958,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colDataAppend(pColInfo, numOfRows, (const char *)&pStream->createTime, false); char sql[TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_WITH_MAXSIZE_TO_VARSTR(streamName, pStream->sql, sizeof(sql)); + STR_WITH_MAXSIZE_TO_VARSTR(sql, pStream->sql, sizeof(sql)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); From 99befe4a8915f9bc01328bcb0c0babf1547ba2d6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 30 Sep 2022 15:08:21 +0800 Subject: [PATCH 61/64] fix: ins_tags table name optimization error --- source/libs/executor/src/scanoperator.c | 78 ++++++++++++++----------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 40fc8cb9a5..2251d09776 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -41,9 +41,9 @@ static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const S static bool processBlockWithProbability(const SSampleExecInfo* pInfo); -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, - const char* tableName, int32_t* pNumOfRows, - const SSDataBlock* dataBlock); +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock); static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { @@ -2415,11 +2415,10 @@ static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && nodeType(node->pRight) == QUERY_NODE_VALUE) { SValueNode* pValue = (SValueNode*)node->pRight; - if (pValue->node.type == TSDB_DATA_TYPE_NCHAR || pValue->node.type == TSDB_DATA_TYPE_VARCHAR || - pValue->node.type == TSDB_DATA_TYPE_BINARY) { - char* value = nodesGetStrValueFromNode(pValue); - strncpy(condTable, value, TSDB_TABLE_NAME_LEN); - taosMemoryFree(value); + if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || + pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { + char* value = nodesGetValueFromNode(pValue); + strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); return true; } } @@ -2480,18 +2479,28 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tableName, condTableName); - SMetaReader smr = {0}; - metaReaderInit(&smr, pInfo->readHandle.meta, 0); - metaGetTableEntryByName(&smr, condTableName); - sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smr); + SMetaReader smrChildTable = {0}; + metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); + metaGetTableEntryByName(&smrChildTable, condTableName); + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { + metaReaderClear(&smrChildTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); + metaReaderClear(&smrSuperTable); + metaReaderClear(&smrChildTable); if (numOfRows > 0) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); numOfRows = 0; } blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + doSetOperatorCompleted(pOperator); return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -2508,23 +2517,22 @@ static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - SMetaReader smr = {0}; - metaReaderInit(&smr, pInfo->readHandle.meta, 0); - + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smr, suid); + int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); if (code != TSDB_CODE_SUCCESS) { qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&smr); + metaReaderClear(&smrSuperTable); metaCloseTbCursor(pInfo->pCur); pInfo->pCur = NULL; T_LONG_JMP(pTaskInfo->env, terrno); } - sysTableUserTagsFillOneTableTags(pInfo, &smr, dbname, tableName, &numOfRows, dataBlock); + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smr); + metaReaderClear(&smrSuperTable); if (numOfRows >= pOperator->resultInfo.capacity) { relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock); @@ -2562,15 +2570,15 @@ static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t blockDataCleanup(dataBlock); } -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smr, const char* dbname, - const char* tableName, int32_t* pNumOfRows, - const SSDataBlock* dataBlock) { +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock) { char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(stableName, (*smr).me.name); + STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); int32_t numOfRows = *pNumOfRows; - int32_t numOfTags = (*smr).me.stbEntry.schemaTag.nCols; + int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; for (int32_t i = 0; i < numOfTags; ++i) { SColumnInfoData* pColInfoData = NULL; @@ -2588,35 +2596,35 @@ static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, // tag name char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tagName, (*smr).me.stbEntry.schemaTag.pSchema[i].name); + STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); colDataAppend(pColInfoData, numOfRows, tagName, false); // tag type - int8_t tagType = (*smr).me.stbEntry.schemaTag.pSchema[i].type; + int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); char tagTypeStr[VARSTR_HEADER_SIZE + 32]; int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); if (tagType == TSDB_DATA_TYPE_VARCHAR) { tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); + (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); } else if (tagType == TSDB_DATA_TYPE_NCHAR) { - tagTypeLen += - sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)(((*smr).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + tagTypeLen += sprintf( + varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(tagTypeStr, tagTypeLen); colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); STagVal tagVal = {0}; - tagVal.cid = (*smr).me.stbEntry.schemaTag.pSchema[i].colId; + tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; char* tagData = NULL; uint32_t tagLen = 0; if (tagType == TSDB_DATA_TYPE_JSON) { - tagData = (char*)pInfo->pCur->mr.me.ctbEntry.pTags; + tagData = (char*)smrChildTable->me.ctbEntry.pTags; } else { - bool exist = tTagGet((STag*)pInfo->pCur->mr.me.ctbEntry.pTags, &tagVal); + bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); if (exist) { if (IS_VAR_DATA_TYPE(tagType)) { tagData = (char*)tagVal.pData; From f3b2067675522118938d27f70b4bcf492a1e04df Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Fri, 30 Sep 2022 14:41:30 +0800 Subject: [PATCH 62/64] fix(stream): sliding error --- source/libs/executor/src/timewindowoperator.c | 20 +++++--- tests/script/tsim/stream/sliding.sim | 48 ++++++++++++++++++- 2 files changed, 61 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 9b4b7f1ee8..dceb696d54 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -1402,7 +1402,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); - while (win.skey <= endTsCols[i]) { + do { uint64_t winGpId = pGpDatas[i]; bool res = doDeleteWindow(pOperator, win.skey, winGpId, numOfOutput); SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; @@ -1413,7 +1413,7 @@ static void doDeleteWindows(SOperatorInfo* pOperator, SInterval* pInterval, int3 taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); } getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); - } + } while (win.ekey <= endTsCols[i]); } } @@ -3067,8 +3067,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); - STimeWindow nextWin = - getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + STimeWindow nextWin = {0}; + if (IS_FINAL_OP(pInfo)) { + nextWin = getFinalTimeWindow(ts, &pInfo->interval); + } else { + nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + } while (1) { bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { @@ -3122,8 +3126,12 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); + if (IS_FINAL_OP(pInfo)) { + forwardRows = 1; + } else { + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, + NULL, TSDB_ORDER_ASC); + } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { saveWinResultInfo(pResult->win.skey, groupId, pUpdatedMap); } diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index bd8d3b0579..b7477fe36c 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -369,9 +369,10 @@ endi #$loop_all = 0 #=looptest: -sql drop database IF EXISTS test2; sql drop stream IF EXISTS streams21; sql drop stream IF EXISTS streams22; +sql drop stream IF EXISTS streams23; +sql drop database IF EXISTS test2; sql create database test2 vgroups 6; sql use test2; @@ -381,6 +382,7 @@ sql create table t2 using st tags(2,2,2); sql create stream streams21 trigger at_once into streamt as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from t1 interval(10s, 5s); sql create stream streams22 trigger at_once into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); +sql create stream streams23 trigger at_once into streamt3 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(20s) sliding(10s); sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); @@ -508,6 +510,50 @@ if $data32 != 8 then goto loop3 endi + +$loop_count = 0 + +loop4: +sleep 100 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt3; + +# row 0 +if $rows != 5 then + print =====rows=$rows + goto loop4 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop4 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop4 +endi + +if $data21 != 4 then + print =====data21=$data21 + goto loop4 +endi + +if $data31 != 4 then + print =====data31=$data31 + goto loop4 +endi + +if $data41 != 2 then + print =====data41=$data41 + goto loop4 +endi + $loop_all = $loop_all + 1 print ============loop_all=$loop_all From 4b1e8e0cec107981bfc187d38634b242e68a8f9b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Sep 2022 16:02:28 +0800 Subject: [PATCH 63/64] fix(query): fix memory leak. --- source/dnode/vnode/src/tsdb/tsdbCache.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 3a921349e6..845b045360 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -447,7 +447,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - // tMergeTreeClose(&state->mergeTree); + tMergeTreeClose(&state->mergeTree); *ppRow = NULL; return code; @@ -463,7 +463,7 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { bool hasVal = tMergeTreeNext(&state->mergeTree); if (!hasVal) { state->state = SFSLASTNEXTROW_FILESET; - // tMergeTreeClose(&state->mergeTree); + tMergeTreeClose(&state->mergeTree); goto _next_fileset; } state->state = SFSLASTNEXTROW_BLOCKROW; From 877c707ce6096a66460b7c1194d092231c4003bd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Sep 2022 17:10:55 +0800 Subject: [PATCH 64/64] fix(query): fix memory leak in last_row query --- source/dnode/vnode/src/tsdb/tsdbCache.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 845b045360..6e27d70096 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -590,7 +590,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { goto _next_fileset; } - tMapDataReset(&state->blockMap); + if (state->blockMap.pData != NULL) { + tMapDataClear(&state->blockMap); + } + code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap); if (code) goto _err; @@ -695,6 +698,10 @@ int32_t clearNextRowFromFS(void *iter) { state->pBlockData = NULL; } + if (state->blockMap.pData != NULL) { + tMapDataClear(&state->blockMap); + } + return code; }