diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index ac94b0af7d..10cb72f5d0 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -162,17 +162,7 @@ struct STSRowBuilder {
struct SValue {
union {
- int8_t i8; // TSDB_DATA_TYPE_BOOL||TSDB_DATA_TYPE_TINYINT
- uint8_t u8; // TSDB_DATA_TYPE_UTINYINT
- int16_t i16; // TSDB_DATA_TYPE_SMALLINT
- uint16_t u16; // TSDB_DATA_TYPE_USMALLINT
- int32_t i32; // TSDB_DATA_TYPE_INT
- uint32_t u32; // TSDB_DATA_TYPE_UINT
- int64_t i64; // TSDB_DATA_TYPE_BIGINT
- uint64_t u64; // TSDB_DATA_TYPE_UBIGINT
- TSKEY ts; // TSDB_DATA_TYPE_TIMESTAMP
- float f; // TSDB_DATA_TYPE_FLOAT
- double d; // TSDB_DATA_TYPE_DOUBLE
+ int64_t val;
struct {
uint32_t nData;
uint8_t *pData;
diff --git a/include/common/ttypes.h b/include/common/ttypes.h
index a88f65f6ac..14e9a5af87 100644
--- a/include/common/ttypes.h
+++ b/include/common/ttypes.h
@@ -333,10 +333,10 @@ typedef struct tDataTypeDescriptor {
char *name;
int64_t minValue;
int64_t maxValue;
- int32_t (*compFunc)(const char *const input, int32_t inputSize, const int32_t nelements, char *const output,
- int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
- int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output,
- int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
+ int32_t (*compFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+ int32_t (*decompFunc)(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min,
int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
} tDataTypeDescriptor;
@@ -356,7 +356,6 @@ void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type);
void *getDataMin(int32_t type);
void *getDataMax(int32_t type);
-
#ifdef __cplusplus
}
#endif
diff --git a/include/util/tcoding.h b/include/util/tcoding.h
index 5962949a70..38eb0d8fc6 100644
--- a/include/util/tcoding.h
+++ b/include/util/tcoding.h
@@ -23,8 +23,8 @@ extern "C" {
#endif
#define ENCODE_LIMIT (((uint8_t)1) << 7)
-#define ZIGZAGE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode
-#define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode
+#define ZIGZAGE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode
+#define ZIGZAGD(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode
/* ------------------------ LEGACY CODES ------------------------ */
#if 1
@@ -70,7 +70,7 @@ static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
}
static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
- *value = ( (((int8_t *)buf)[0] == 0) ? false : true );
+ *value = ((((int8_t *)buf)[0] == 0) ? false : true);
return POINTER_SHIFT(buf, sizeof(int8_t));
}
diff --git a/include/util/tcompression.h b/include/util/tcompression.h
index d7ba4dfa3f..84c2922c9e 100644
--- a/include/util/tcompression.h
+++ b/include/util/tcompression.h
@@ -51,287 +51,12 @@ extern "C" {
#define HEAD_MODE(x) x % 2
#define HEAD_ALGO(x) x / 2
-extern int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type);
-extern int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output,
- const char type);
-extern int32_t tsCompressBoolImp(const char *const input, const int32_t nelements, char *const output);
-extern int32_t tsDecompressBoolImp(const char *const input, const int32_t nelements, char *const output);
-extern int32_t tsCompressStringImp(const char *const input, int32_t inputSize, char *const output, int32_t outputSize);
-extern int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, char *const output,
- int32_t outputSize);
-extern int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output);
-extern int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output);
-extern int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, char *const output);
-extern int32_t tsDecompressDoubleImp(const char *const input, const int32_t nelements, char *const output);
-extern int32_t tsCompressFloatImp(const char *const input, const int32_t nelements, char *const output);
-extern int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output);
-// lossy
-extern int32_t tsCompressFloatLossyImp(const char *input, const int32_t nelements, char *const output);
-extern int32_t tsDecompressFloatLossyImp(const char *input, int32_t compressedSize, const int32_t nelements,
- char *const output);
-extern int32_t tsCompressDoubleLossyImp(const char *input, const int32_t nelements, char *const output);
-extern int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, const int32_t nelements,
- char *const output);
-
#ifdef TD_TSZ
extern bool lossyFloat;
extern bool lossyDouble;
int32_t tsCompressInit();
void tsCompressExit();
-#endif
-static FORCE_INLINE int32_t tsCompressTinyint(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm,
- char *const buffer, int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsDecompressTinyint(const char *const input, int32_t compressedSize,
- const int32_t nelements, char *const output, int32_t outputSize,
- char algorithm, char *const buffer, int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsCompressSmallint(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm,
- char *const buffer, int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsDecompressSmallint(const char *const input, int32_t compressedSize,
- const int32_t nelements, char *const output, int32_t outputSize,
- char algorithm, char *const buffer, int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsCompressInt(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsDecompressInt(const char *const input, int32_t compressedSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsCompressBigint(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsDecompressBigint(const char *const input, int32_t compressedSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm,
- char *const buffer, int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsCompressBool(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressBoolImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressBoolImp(input, nelements, buffer);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsDecompressBool(const char *const input, int32_t compressedSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressBoolImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressBoolImp(buffer, nelements, output);
- } else {
- assert(0);
- return -1;
- }
-}
-
-static FORCE_INLINE int32_t tsCompressString(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
- return tsCompressStringImp(input, inputSize, output, outputSize);
-}
-
-static FORCE_INLINE int32_t tsDecompressString(const char *const input, int32_t compressedSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm,
- char *const buffer, int32_t bufferSize) {
- return tsDecompressStringImp(input, compressedSize, output, outputSize);
-}
-
-static FORCE_INLINE int32_t tsCompressFloat(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
-#ifdef TD_TSZ
- // lossy mode
- if (lossyFloat) {
- return tsCompressFloatLossyImp(input, nelements, output);
- // lossless mode
- } else {
-#endif
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressFloatImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressFloatImp(input, nelements, buffer);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-#ifdef TD_TSZ
- }
-#endif
-}
-
-static FORCE_INLINE int32_t tsDecompressFloat(const char *const input, int32_t compressedSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm,
- char *const buffer, int32_t bufferSize) {
-#ifdef TD_TSZ
- if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
- // decompress lossy
- return tsDecompressFloatLossyImp(input, compressedSize, nelements, output);
- } else {
-#endif
- // decompress lossless
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressFloatImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressFloatImp(buffer, nelements, output);
- } else {
- assert(0);
- return -1;
- }
-#ifdef TD_TSZ
- }
-#endif
-}
-
-static FORCE_INLINE int32_t tsCompressDouble(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm, char *const buffer,
- int32_t bufferSize) {
-#ifdef TD_TSZ
- if (lossyDouble) {
- // lossy mode
- return tsCompressDoubleLossyImp(input, nelements, output);
- } else {
-#endif
- // lossless mode
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressDoubleImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressDoubleImp(input, nelements, buffer);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-#ifdef TD_TSZ
- }
-#endif
-}
-
-static FORCE_INLINE int32_t tsDecompressDouble(const char *const input, int32_t compressedSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm,
- char *const buffer, int32_t bufferSize) {
-#ifdef TD_TSZ
- if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
- // decompress lossy
- return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output);
- } else {
-#endif
- // decompress lossless
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressDoubleImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressDoubleImp(buffer, nelements, output);
- } else {
- assert(0);
- return -1;
- }
-#ifdef TD_TSZ
- }
-#endif
-}
-
-#ifdef TD_TSZ
-//
-// lossy float double
-//
static FORCE_INLINE int32_t tsCompressFloatLossy(const char *const input, int32_t inputSize, const int32_t nelements,
char *const output, int32_t outputSize, char algorithm,
char *const buffer, int32_t bufferSize) {
@@ -358,33 +83,56 @@ static FORCE_INLINE int32_t tsDecompressDoubleLossy(const char *const input, int
#endif
-static FORCE_INLINE int32_t tsCompressTimestamp(const char *const input, int32_t inputSize, const int32_t nelements,
- char *const output, int32_t outputSize, char algorithm,
- char *const buffer, int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsCompressTimestampImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- int32_t len = tsCompressTimestampImp(input, nelements, buffer);
- return tsCompressStringImp(buffer, len, output, outputSize);
- } else {
- assert(0);
- return -1;
- }
-}
+/*************************************************************************
+ * REGULAR COMPRESSION
+ *************************************************************************/
+int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
+ void *pBuf, int32_t nBuf);
+int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
+ void *pBuf, int32_t nBuf);
+int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
+int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf);
-static FORCE_INLINE int32_t tsDecompressTimestamp(const char *const input, int32_t compressedSize,
- const int32_t nelements, char *const output, int32_t outputSize,
- char algorithm, char *const buffer, int32_t bufferSize) {
- if (algorithm == ONE_STAGE_COMP) {
- return tsDecompressTimestampImp(input, nelements, output);
- } else if (algorithm == TWO_STAGE_COMP) {
- if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1;
- return tsDecompressTimestampImp(buffer, nelements, output);
- } else {
- assert(0);
- return -1;
- }
-}
+/*************************************************************************
+ * STREAM COMPRESSION
+ *************************************************************************/
+typedef struct SCompressor SCompressor;
+
+int32_t tCompressorCreate(SCompressor **ppCmprsor);
+int32_t tCompressorDestroy(SCompressor *pCmprsor);
+int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
+int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin);
+int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData);
#ifdef __cplusplus
}
diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c
index c0ae99806e..e461e06158 100644
--- a/source/common/src/tdataformat.c
+++ b/source/common/src/tdataformat.c
@@ -33,105 +33,21 @@ typedef struct {
// SValue
int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type) {
- int32_t n = 0;
-
if (IS_VAR_DATA_TYPE(type)) {
- n += tPutBinary(p ? p + n : p, pValue->pData, pValue->nData);
+ return tPutBinary(p, pValue->pData, pValue->nData);
} else {
- switch (type) {
- case TSDB_DATA_TYPE_BOOL:
- n += tPutI8(p ? p + n : p, pValue->i8 ? 1 : 0);
- break;
- case TSDB_DATA_TYPE_TINYINT:
- n += tPutI8(p ? p + n : p, pValue->i8);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- n += tPutI16(p ? p + n : p, pValue->i16);
- break;
- case TSDB_DATA_TYPE_INT:
- n += tPutI32(p ? p + n : p, pValue->i32);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- n += tPutI64(p ? p + n : p, pValue->i64);
- break;
- case TSDB_DATA_TYPE_FLOAT:
- n += tPutFloat(p ? p + n : p, pValue->f);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- n += tPutDouble(p ? p + n : p, pValue->d);
- break;
- case TSDB_DATA_TYPE_TIMESTAMP:
- n += tPutI64(p ? p + n : p, pValue->ts);
- break;
- case TSDB_DATA_TYPE_UTINYINT:
- n += tPutU8(p ? p + n : p, pValue->u8);
- break;
- case TSDB_DATA_TYPE_USMALLINT:
- n += tPutU16(p ? p + n : p, pValue->u16);
- break;
- case TSDB_DATA_TYPE_UINT:
- n += tPutU32(p ? p + n : p, pValue->u32);
- break;
- case TSDB_DATA_TYPE_UBIGINT:
- n += tPutU64(p ? p + n : p, pValue->u64);
- break;
- default:
- ASSERT(0);
- }
+ if (p) memcpy(p, &pValue->val, tDataTypes[type].bytes);
+ return tDataTypes[type].bytes;
}
-
- return n;
}
int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type) {
- int32_t n = 0;
-
if (IS_VAR_DATA_TYPE(type)) {
- n += tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL);
+ return tGetBinary(p, &pValue->pData, pValue ? &pValue->nData : NULL);
} else {
- switch (type) {
- case TSDB_DATA_TYPE_BOOL:
- n += tGetI8(p, &pValue->i8);
- break;
- case TSDB_DATA_TYPE_TINYINT:
- n += tGetI8(p, &pValue->i8);
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- n += tGetI16(p, &pValue->i16);
- break;
- case TSDB_DATA_TYPE_INT:
- n += tGetI32(p, &pValue->i32);
- break;
- case TSDB_DATA_TYPE_BIGINT:
- n += tGetI64(p, &pValue->i64);
- break;
- case TSDB_DATA_TYPE_FLOAT:
- n += tGetFloat(p, &pValue->f);
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- n += tGetDouble(p, &pValue->d);
- break;
- case TSDB_DATA_TYPE_TIMESTAMP:
- n += tGetI64(p, &pValue->ts);
- break;
- case TSDB_DATA_TYPE_UTINYINT:
- n += tGetU8(p, &pValue->u8);
- break;
- case TSDB_DATA_TYPE_USMALLINT:
- n += tGetU16(p, &pValue->u16);
- break;
- case TSDB_DATA_TYPE_UINT:
- n += tGetU32(p, &pValue->u32);
- break;
- case TSDB_DATA_TYPE_UBIGINT:
- n += tGetU64(p, &pValue->u64);
- break;
- default:
- ASSERT(0);
- }
+ memcpy(&pValue->val, p, tDataTypes[type].bytes);
+ return tDataTypes[type].bytes;
}
-
- return n;
}
int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type) {
@@ -1219,288 +1135,397 @@ static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVa
_exit:
return code;
}
-static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0
+static FORCE_INLINE int32_t tColDataAppendValue00(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
-
- if (COL_VAL_IS_NONE(pColVal)) {
- pColData->flag = HAS_NONE;
- } else if (COL_VAL_IS_NULL(pColVal)) {
- pColData->flag = HAS_NULL;
- } else {
- pColData->flag = HAS_VALUE;
- code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
- }
- pColData->nVal++;
-
-_exit:
- return code;
-}
-static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE
- int32_t code = 0;
-
- if (!COL_VAL_IS_NONE(pColVal)) {
- int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
-
- code = tRealloc(&pColData->pBitMap, nBit);
- if (code) goto _exit;
-
- memset(pColData->pBitMap, 0, nBit);
- SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
-
- if (COL_VAL_IS_NULL(pColVal)) {
- pColData->flag |= HAS_NULL;
- } else {
- pColData->flag |= HAS_VALUE;
-
- if (pColData->nVal) {
- if (IS_VAR_DATA_TYPE(pColData->type)) {
- int32_t nOffset = sizeof(int32_t) * pColData->nVal;
- code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
- if (code) goto _exit;
- memset(pColData->aOffset, 0, nOffset);
- } else {
- pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
- code = tRealloc(&pColData->pData, pColData->nData);
- if (code) goto _exit;
- memset(pColData->pData, 0, pColData->nData);
- }
- }
-
- code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
- }
- }
- pColData->nVal++;
-
-_exit:
- return code;
-}
-static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL
- int32_t code = 0;
-
- if (!COL_VAL_IS_NULL(pColVal)) {
- int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
- code = tRealloc(&pColData->pBitMap, nBit);
- if (code) goto _exit;
-
- if (COL_VAL_IS_NONE(pColVal)) {
- pColData->flag |= HAS_NONE;
-
- memset(pColData->pBitMap, 255, nBit);
- SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
- } else {
- pColData->flag |= HAS_VALUE;
-
- memset(pColData->pBitMap, 0, nBit);
- SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
-
- if (pColData->nVal) {
- if (IS_VAR_DATA_TYPE(pColData->type)) {
- int32_t nOffset = sizeof(int32_t) * pColData->nVal;
- code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
- if (code) goto _exit;
- memset(pColData->aOffset, 0, nOffset);
- } else {
- pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
- code = tRealloc(&pColData->pData, pColData->nData);
- if (code) goto _exit;
- memset(pColData->pData, 0, pColData->nData);
- }
- }
-
- code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
- }
- }
- pColData->nVal++;
-
-_exit:
- return code;
-}
-static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE
- int32_t code = 0;
-
- if (COL_VAL_IS_NONE(pColVal)) {
- code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
-
- SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
- } else if (COL_VAL_IS_NULL(pColVal)) {
- code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
-
- SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
- } else {
- pColData->flag |= HAS_VALUE;
-
- uint8_t *pBitMap = NULL;
- code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
-
- for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
- SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
- }
- SET_BIT2(pBitMap, pColData->nVal, 2);
-
- tFree(pColData->pBitMap);
- pColData->pBitMap = pBitMap;
-
- if (pColData->nVal) {
- if (IS_VAR_DATA_TYPE(pColData->type)) {
- int32_t nOffset = sizeof(int32_t) * pColData->nVal;
- code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
- if (code) goto _exit;
- memset(pColData->aOffset, 0, nOffset);
- } else {
- pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
- code = tRealloc(&pColData->pData, pColData->nData);
- if (code) goto _exit;
- memset(pColData->pData, 0, pColData->nData);
- }
- }
-
- code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
- }
- pColData->nVal++;
-
-_exit:
- return code;
-}
-static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE
- int32_t code = 0;
-
- if (!COL_VAL_IS_VALUE(pColVal)) {
- if (COL_VAL_IS_NONE(pColVal)) {
- pColData->flag |= HAS_NONE;
- } else {
- pColData->flag |= HAS_NULL;
- }
-
- int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
- code = tRealloc(&pColData->pBitMap, nBit);
- if (code) goto _exit;
-
- memset(pColData->pBitMap, 255, nBit);
- SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
-
- code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
- } else {
- code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
- }
- pColData->nVal++;
-
-_exit:
- return code;
-}
-static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE
- int32_t code = 0;
-
- if (COL_VAL_IS_NULL(pColVal)) {
- pColData->flag |= HAS_NULL;
-
- uint8_t *pBitMap = NULL;
- code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
-
- for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
- SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
- }
- SET_BIT2(pBitMap, pColData->nVal, 1);
-
- tFree(pColData->pBitMap);
- pColData->pBitMap = pBitMap;
- } else {
- code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
-
- if (COL_VAL_IS_NONE(pColVal)) {
- SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
- } else {
- SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
- }
- }
+ pColData->flag = HAS_VALUE;
code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
-
+ if (code) return code;
pColData->nVal++;
-
-_exit:
return code;
}
-static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL
+static FORCE_INLINE int32_t tColDataAppendValue01(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+ pColData->flag = HAS_NONE;
+ pColData->nVal++;
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue02(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+ pColData->flag = HAS_NULL;
+ pColData->nVal++;
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue10(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
- if (COL_VAL_IS_NONE(pColVal)) {
- pColData->flag |= HAS_NONE;
+ int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+ code = tRealloc(&pColData->pBitMap, nBit);
+ if (code) return code;
- uint8_t *pBitMap = NULL;
- code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
+ memset(pColData->pBitMap, 0, nBit);
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
- for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
- SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
- }
- SET_BIT2(pBitMap, pColData->nVal, 0);
+ pColData->flag |= HAS_VALUE;
- tFree(pColData->pBitMap);
- pColData->pBitMap = pBitMap;
- } else {
- code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
-
- if (COL_VAL_IS_NULL(pColVal)) {
- SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+ if (pColData->nVal) {
+ if (IS_VAR_DATA_TYPE(pColData->type)) {
+ int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+ code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+ if (code) return code;
+ memset(pColData->aOffset, 0, nOffset);
} else {
- SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+ pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+ code = tRealloc(&pColData->pData, pColData->nData);
+ if (code) return code;
+ memset(pColData->pData, 0, pColData->nData);
}
}
+
code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
+ if (code) return code;
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue11(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+ pColData->nVal++;
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue12(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+ code = tRealloc(&pColData->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pColData->pBitMap, 0, nBit);
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+ pColData->flag |= HAS_NULL;
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue20(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+ code = tRealloc(&pColData->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pColData->pBitMap, 0, nBit);
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+ pColData->flag |= HAS_VALUE;
+
+ if (pColData->nVal) {
+ if (IS_VAR_DATA_TYPE(pColData->type)) {
+ int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+ code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+ if (code) return code;
+ memset(pColData->aOffset, 0, nOffset);
+ } else {
+ pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+ code = tRealloc(&pColData->pData, pColData->nData);
+ if (code) return code;
+ memset(pColData->pData, 0, pColData->nData);
+ }
+ }
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue21(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+ code = tRealloc(&pColData->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pColData->pBitMap, 255, nBit);
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+ pColData->flag |= HAS_NONE;
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue22(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+ pColData->nVal++;
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue30(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pColData->flag |= HAS_VALUE;
+
+ uint8_t *pBitMap = NULL;
+ code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+ if (code) return code;
+
+ for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+ SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
+ }
+ SET_BIT2(pBitMap, pColData->nVal, 2);
+
+ tFree(pColData->pBitMap);
+ pColData->pBitMap = pBitMap;
+
+ if (pColData->nVal) {
+ if (IS_VAR_DATA_TYPE(pColData->type)) {
+ int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+ code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+ if (code) return code;
+ memset(pColData->aOffset, 0, nOffset);
+ } else {
+ pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+ code = tRealloc(&pColData->pData, pColData->nData);
+ if (code) return code;
+ memset(pColData->pData, 0, pColData->nData);
+ }
+ }
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue31(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+ if (code) return code;
+
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue32(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+ if (code) return code;
+
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue40(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue41(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pColData->flag |= HAS_NONE;
+
+ int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+ code = tRealloc(&pColData->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pColData->pBitMap, 255, nBit);
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
pColData->nVal++;
-_exit:
return code;
}
-static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData,
- SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE
+static FORCE_INLINE int32_t tColDataAppendValue42(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pColData->flag |= HAS_NULL;
+
+ int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+ code = tRealloc(&pColData->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pColData->pBitMap, 255, nBit);
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue50(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+ if (code) return code;
+
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue51(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+ if (code) return code;
+
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue52(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pColData->flag |= HAS_NULL;
+
+ uint8_t *pBitMap = NULL;
+ code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+ if (code) return code;
+
+ for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+ SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
+ }
+ SET_BIT2(pBitMap, pColData->nVal, 1);
+
+ tFree(pColData->pBitMap);
+ pColData->pBitMap = pBitMap;
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue60(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue61(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pColData->flag |= HAS_NONE;
+
+ uint8_t *pBitMap = NULL;
+ code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+ if (code) return code;
+
+ for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+ SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
+ }
+ SET_BIT2(pBitMap, pColData->nVal, 0);
+
+ tFree(pColData->pBitMap);
+ pColData->pBitMap = pBitMap;
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue62(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue70(SColData *pColData, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
- if (code) goto _exit;
+ if (code) return code;
+ SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
- if (COL_VAL_IS_NONE(pColVal)) {
- SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
- } else if (COL_VAL_IS_NULL(pColVal)) {
- SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
- } else {
- SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
- }
code = tColDataPutValue(pColData, pColVal);
- if (code) goto _exit;
+ if (code) return code;
pColData->nVal++;
-_exit:
return code;
}
-static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = {
- tColDataAppendValue0, // 0
- tColDataAppendValue1, // HAS_NONE
- tColDataAppendValue2, // HAS_NULL
- tColDataAppendValue3, // HAS_NULL|HAS_NONE
- tColDataAppendValue4, // HAS_VALUE
- tColDataAppendValue5, // HAS_VALUE|HAS_NONE
- tColDataAppendValue6, // HAS_VALUE|HAS_NULL
- tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE
+static FORCE_INLINE int32_t tColDataAppendValue71(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
+ if (code) return code;
+ SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue72(SColData *pColData, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
+ if (code) return code;
+ SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
+
+ code = tColDataPutValue(pColData, pColVal);
+ if (code) return code;
+
+ pColData->nVal++;
+
+ return code;
+}
+static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, SColVal *pColVal) = {
+ {tColDataAppendValue00, tColDataAppendValue01, tColDataAppendValue02}, // 0
+ {tColDataAppendValue10, tColDataAppendValue11, tColDataAppendValue12}, // HAS_NONE
+ {tColDataAppendValue20, tColDataAppendValue21, tColDataAppendValue22}, // HAS_NULL
+ {tColDataAppendValue30, tColDataAppendValue31, tColDataAppendValue32}, // HAS_NULL|HAS_NONE
+ {tColDataAppendValue40, tColDataAppendValue41, tColDataAppendValue42}, // HAS_VALUE
+ {tColDataAppendValue50, tColDataAppendValue51, tColDataAppendValue52}, // HAS_VALUE|HAS_NONE
+ {tColDataAppendValue60, tColDataAppendValue61, tColDataAppendValue62}, // HAS_VALUE|HAS_NULL
+ {tColDataAppendValue70, tColDataAppendValue71, tColDataAppendValue72}, // HAS_VALUE|HAS_NULL|HAS_NONE
};
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
- return tColDataAppendValueImpl[pColData->flag](pColData, pColVal);
+ return tColDataAppendValueImpl[pColData->flag][pColVal->flag](pColData, pColVal);
}
static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE
diff --git a/source/common/src/trow.c b/source/common/src/trow.c
index f23a2b386e..e1085ce5d8 100644
--- a/source/common/src/trow.c
+++ b/source/common/src/trow.c
@@ -689,7 +689,7 @@ int32_t tdSTSRowNew(SArray *pArray, STSchema *pTSchema, STSRow **ppRow) {
memcpy(varDataVal(varBuf), pColVal->value.pData, pColVal->value.nData);
val = varBuf;
} else {
- val = (const void *)&pColVal->value.i64;
+ val = (const void *)&pColVal->value.val;
}
} else {
pColVal = NULL;
diff --git a/source/common/test/dataformatTest.cpp b/source/common/test/dataformatTest.cpp
index 6d77180610..b05ae602f8 100644
--- a/source/common/test/dataformatTest.cpp
+++ b/source/common/test/dataformatTest.cpp
@@ -13,6 +13,7 @@
* along with this program. If not, see .
*/
+#if 0
#include
#include
@@ -476,4 +477,5 @@ TEST(testCase, NoneTest) {
taosArrayDestroy(pArray);
taosMemoryFree(pTSchema);
}
+#endif
#endif
\ No newline at end of file
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index 75b2f74096..bf110f1ae3 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -38,39 +38,43 @@ extern "C" {
goto LABEL; \
}
-typedef struct TSDBROW TSDBROW;
-typedef struct TABLEID TABLEID;
-typedef struct TSDBKEY TSDBKEY;
-typedef struct SDelData SDelData;
-typedef struct SDelIdx SDelIdx;
-typedef struct STbData STbData;
-typedef struct SMemTable SMemTable;
-typedef struct STbDataIter STbDataIter;
-typedef struct SMapData SMapData;
-typedef struct SBlockIdx SBlockIdx;
-typedef struct SDataBlk SDataBlk;
-typedef struct SSttBlk SSttBlk;
-typedef struct SDiskDataHdr SDiskDataHdr;
-typedef struct SBlockData SBlockData;
-typedef struct SDelFile SDelFile;
-typedef struct SHeadFile SHeadFile;
-typedef struct SDataFile SDataFile;
-typedef struct SSttFile SSttFile;
-typedef struct SSmaFile SSmaFile;
-typedef struct SDFileSet SDFileSet;
-typedef struct SDataFWriter SDataFWriter;
-typedef struct SDataFReader SDataFReader;
-typedef struct SDelFWriter SDelFWriter;
-typedef struct SDelFReader SDelFReader;
-typedef struct SRowIter SRowIter;
-typedef struct STsdbFS STsdbFS;
-typedef struct SRowMerger SRowMerger;
-typedef struct STsdbReadSnap STsdbReadSnap;
-typedef struct SBlockInfo SBlockInfo;
-typedef struct SSmaInfo SSmaInfo;
-typedef struct SBlockCol SBlockCol;
-typedef struct SVersionRange SVersionRange;
-typedef struct SLDataIter SLDataIter;
+typedef struct TSDBROW TSDBROW;
+typedef struct TABLEID TABLEID;
+typedef struct TSDBKEY TSDBKEY;
+typedef struct SDelData SDelData;
+typedef struct SDelIdx SDelIdx;
+typedef struct STbData STbData;
+typedef struct SMemTable SMemTable;
+typedef struct STbDataIter STbDataIter;
+typedef struct SMapData SMapData;
+typedef struct SBlockIdx SBlockIdx;
+typedef struct SDataBlk SDataBlk;
+typedef struct SSttBlk SSttBlk;
+typedef struct SDiskDataHdr SDiskDataHdr;
+typedef struct SBlockData SBlockData;
+typedef struct SDelFile SDelFile;
+typedef struct SHeadFile SHeadFile;
+typedef struct SDataFile SDataFile;
+typedef struct SSttFile SSttFile;
+typedef struct SSmaFile SSmaFile;
+typedef struct SDFileSet SDFileSet;
+typedef struct SDataFWriter SDataFWriter;
+typedef struct SDataFReader SDataFReader;
+typedef struct SDelFWriter SDelFWriter;
+typedef struct SDelFReader SDelFReader;
+typedef struct SRowIter SRowIter;
+typedef struct STsdbFS STsdbFS;
+typedef struct SRowMerger SRowMerger;
+typedef struct STsdbReadSnap STsdbReadSnap;
+typedef struct SBlockInfo SBlockInfo;
+typedef struct SSmaInfo SSmaInfo;
+typedef struct SBlockCol SBlockCol;
+typedef struct SVersionRange SVersionRange;
+typedef struct SLDataIter SLDataIter;
+typedef struct SDiskCol SDiskCol;
+typedef struct SDiskData SDiskData;
+typedef struct SDiskDataBuilder SDiskDataBuilder;
+typedef struct SBlkInfo SBlkInfo;
#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F)
#define TSDB_MAX_SUBBLOCKS 8
@@ -170,7 +174,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut
int32_t aBufN[]);
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
// SDiskDataHdr
-int32_t tPutDiskDataHdr(uint8_t *p, void *ph);
+int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr);
int32_t tGetDiskDataHdr(uint8_t *p, void *ph);
// SDelIdx
int32_t tPutDelIdx(uint8_t *p, void *ph);
@@ -267,6 +271,7 @@ int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *p
int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
int8_t cmprAlg, int8_t toLast);
+int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo);
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo);
// SDataFReader
@@ -300,7 +305,7 @@ int32_t tsdbMerge(STsdb *pTsdb);
#define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0)
#define TSDB_CACHE_LAST(c) (((c).cacheLast & 2) > 0)
-// tsdbCache
+// tsdbCache ==============================================================================================
int32_t tsdbOpenCache(STsdb *pTsdb);
void tsdbCloseCache(STsdb *pTsdb);
int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb *pTsdb);
@@ -318,6 +323,15 @@ size_t tsdbCacheGetCapacity(SVnode *pVnode);
int32_t tsdbCacheLastArray2Row(SArray *pLastArray, STSRow **ppRow, STSchema *pSchema);
+// tsdbDiskData ==============================================================================================
+int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
+void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder);
+int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
+ uint8_t calcSma);
+int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder);
+int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId);
+int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo);
+
// structs =======================
struct STsdbFS {
SDelFile *pDelFile;
@@ -438,6 +452,17 @@ struct SSmaInfo {
int32_t size;
};
+struct SBlkInfo {
+ int64_t minUid;
+ int64_t maxUid;
+ TSKEY minKey;
+ TSKEY maxKey;
+ int64_t minVer;
+ int64_t maxVer;
+ TSDBKEY minTKey;
+ TSDBKEY maxTKey;
+};
+
struct SDataBlk {
TSDBKEY minKey;
TSDBKEY maxKey;
@@ -661,6 +686,38 @@ typedef struct {
STSchema *pTSchema;
} SSkmInfo;
+struct SDiskCol {
+ SBlockCol bCol;
+ const uint8_t *pBit;
+ const uint8_t *pOff;
+ const uint8_t *pVal;
+ SColumnDataAgg agg;
+};
+
+struct SDiskData {
+ SDiskDataHdr hdr;
+ const uint8_t *pUid;
+ const uint8_t *pVer;
+ const uint8_t *pKey;
+ SArray *aDiskCol; // SArray
+};
+
+struct SDiskDataBuilder {
+ int64_t suid;
+ int64_t uid;
+ int32_t nRow;
+ uint8_t cmprAlg;
+ uint8_t calcSma;
+ SCompressor *pUidC;
+ SCompressor *pVerC;
+ SCompressor *pKeyC;
+ int32_t nBuilder;
+ SArray *aBuilder; // SArray
+ uint8_t *aBuf[2];
+ SDiskData dd;
+ SBlkInfo bi;
+};
+
int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
STimeWindow *pTimeWindow, SVersionRange *pVerRange, SSttBlockLoadInfo *pBlockLoadInfo,
bool destroyLoadInfo, const char *idStr);
diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c
index 3a921349e6..2668f8d4f6 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCache.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCache.c
@@ -260,7 +260,7 @@ int32_t tsdbCacheInsertLast(SLRUCache *pCache, tb_uid_t uid, STSRow *row, STsdb
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
if (keyTs > tTsVal->ts) {
STColumn *pTColumn = &pTSchema->columns[0];
- SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = keyTs});
+ SColVal tColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = keyTs});
taosArraySet(pLast, iCol, &(SLastCol){.ts = keyTs, .colVal = tColVal});
}
@@ -1052,7 +1052,7 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, STSRow **ppRo
lastRowTs = TSDBROW_TS(pRow);
STColumn *pTColumn = &pTSchema->columns[0];
- *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs});
+ *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
if (taosArrayPush(pColArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
@@ -1151,7 +1151,7 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray) {
lastRowTs = rowTs;
STColumn *pTColumn = &pTSchema->columns[0];
- *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = lastRowTs});
+ *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = lastRowTs});
if (taosArrayPush(pColArray, &(SLastCol){.ts = lastRowTs, .colVal = *pColVal}) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 5403395623..995d7a3155 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -17,6 +17,8 @@
typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT;
+#define USE_STREAM_COMPRESSION 0
+
typedef struct {
SRBTreeNode n;
SRowInfo r;
@@ -75,7 +77,11 @@ typedef struct {
SArray *aSttBlk; // SArray
SMapData mBlock; // SMapData
SBlockData bData;
- SBlockData bDatal;
+#if USE_STREAM_COMPRESSION
+ SDiskDataBuilder *pBuilder;
+#else
+ SBlockData bDatal;
+#endif
} dWriter;
SSkmInfo skmTable;
SSkmInfo skmRow;
@@ -115,33 +121,32 @@ int32_t tRowInfoCmprFn(const void *p1, const void *p2) {
int32_t tsdbBegin(STsdb *pTsdb) {
int32_t code = 0;
+ int32_t lino = 0;
if (!pTsdb) return code;
SMemTable *pMemTable;
code = tsdbMemTableCreate(pTsdb, &pMemTable);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// lock
- code = taosThreadRwlockWrlock(&pTsdb->rwLock);
- if (code) {
+ if ((code = taosThreadRwlockWrlock(&pTsdb->rwLock))) {
code = TAOS_SYSTEM_ERROR(code);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pTsdb->mem = pMemTable;
// unlock
- code = taosThreadRwlockUnlock(&pTsdb->rwLock);
- if (code) {
+ if ((code = taosThreadRwlockUnlock(&pTsdb->rwLock))) {
code = TAOS_SYSTEM_ERROR(code);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb begin failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
@@ -149,6 +154,7 @@ int32_t tsdbCommit(STsdb *pTsdb) {
if (!pTsdb) return 0;
int32_t code = 0;
+ int32_t lino = 0;
SCommitter commith;
SMemTable *pMemTable = pTsdb->mem;
@@ -164,76 +170,74 @@ int32_t tsdbCommit(STsdb *pTsdb) {
// start commit
code = tsdbStartCommit(pTsdb, &commith);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// commit impl
code = tsdbCommitData(&commith);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbCommitDel(&commith);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// end commit
code = tsdbEndCommit(&commith, 0);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
_exit:
- return code;
-
-_err:
- tsdbEndCommit(&commith, code);
- tsdbError("vgId:%d, failed to commit since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbEndCommit(&commith, code);
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitDelStart(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
- pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
- if (pCommitter->aDelIdx == NULL) {
+ if ((pCommitter->aDelIdx = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData));
- if (pCommitter->aDelData == NULL) {
+ if ((pCommitter->aDelData = taosArrayInit(0, sizeof(SDelData))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx));
- if (pCommitter->aDelIdxN == NULL) {
+ if ((pCommitter->aDelIdxN = taosArrayInit(0, sizeof(SDelIdx))) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
SDelFile *pDelFileR = pCommitter->fs.pDelFile;
if (pDelFileR) {
code = tsdbDelFReaderOpen(&pCommitter->pDelFReader, pDelFileR, pTsdb);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadDelIdx(pCommitter->pDelFReader, pCommitter->aDelIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
// prepare new
SDelFile wDelFile = {.commitID = pCommitter->commitID, .size = 0, .offset = 0};
code = tsdbDelFWriterOpen(&pCommitter->pDelFWriter, &wDelFile, pTsdb);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
_exit:
- tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode));
- return code;
-
-_err:
- tsdbError("vgId:%d, commit del start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ } else {
+ tsdbDebug("vgId:%d, commit del start", TD_VID(pTsdb->pVnode));
+ }
return code;
}
static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDelIdx *pDelIdx) {
int32_t code = 0;
+ int32_t lino = 0;
SDelData *pDelData;
tb_uid_t suid;
tb_uid_t uid;
@@ -252,7 +256,7 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
uid = pDelIdx->uid;
code = tsdbReadDelData(pCommitter->pDelFReader, pDelIdx, pCommitter->aDelData);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
} else {
taosArrayClear(pCommitter->aDelData);
}
@@ -266,62 +270,65 @@ static int32_t tsdbCommitTableDel(SCommitter *pCommitter, STbData *pTbData, SDel
for (; pDelData; pDelData = pDelData->pNext) {
if (taosArrayPush(pCommitter->aDelData, pDelData) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
// write
code = tsdbWriteDelData(pCommitter->pDelFWriter, pCommitter->aDelData, &delIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// put delIdx
if (taosArrayPush(pCommitter->aDelIdxN, &delIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d, commit table del failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitDelEnd(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb;
code = tsdbWriteDelIdx(pCommitter->pDelFWriter, pCommitter->aDelIdxN);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbUpdateDelFileHdr(pCommitter->pDelFWriter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbFSUpsertDelFile(&pCommitter->fs, &pCommitter->pDelFWriter->fDel);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDelFWriterClose(&pCommitter->pDelFWriter, 1);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
if (pCommitter->pDelFReader) {
code = tsdbDelFReaderClose(&pCommitter->pDelFReader);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
taosArrayDestroy(pCommitter->aDelIdx);
taosArrayDestroy(pCommitter->aDelData);
taosArrayDestroy(pCommitter->aDelIdxN);
- return code;
-
-_err:
- tsdbError("vgId:%d, commit del end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d, %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) {
int32_t code = 0;
+ int32_t lino = 0;
if (suid) {
if (pSkmInfo->suid == suid) {
@@ -336,7 +343,7 @@ int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo
pSkmInfo->uid = uid;
tTSchemaDestroy(pSkmInfo->pTSchema);
code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema);
- if (code) goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
_exit:
return code;
@@ -344,6 +351,7 @@ _exit:
static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid, int64_t uid, int32_t sver) {
int32_t code = 0;
+ int32_t lino = 0;
if (pCommitter->skmRow.pTSchema) {
if (pCommitter->skmRow.suid == suid) {
@@ -359,9 +367,7 @@ static int32_t tsdbCommitterUpdateRowSchema(SCommitter *pCommitter, int64_t suid
pCommitter->skmRow.uid = uid;
tTSchemaDestroy(pCommitter->skmRow.pTSchema);
code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, sver, &pCommitter->skmRow.pTSchema);
- if (code) {
- goto _exit;
- }
+ TSDB_CHECK_CODE(code, lino, _exit);
_exit:
return code;
@@ -369,6 +375,7 @@ _exit:
static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
ASSERT(pCommitter->dReader.pBlockIdx);
@@ -378,7 +385,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) {
(SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx);
code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
- if (code) goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
ASSERT(pCommitter->dReader.mBlock.nItem > 0);
} else {
@@ -391,6 +398,7 @@ _exit:
static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
pCommitter->pIter = NULL;
tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn);
@@ -431,14 +439,14 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
pIter->iStt = iStt;
code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayGetSize(pIter->aSttBlk) == 0) continue;
pIter->iSttBlk = 0;
SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0);
code = tsdbReadSttBlockEx(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
pIter->iRow = 0;
pIter->r.suid = pIter->bData.suid;
@@ -460,16 +468,19 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
}
code = tsdbNextCommitRow(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
- return code;
-
-_err:
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SDFileSet *pRSet = NULL;
@@ -484,17 +495,17 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ);
if (pRSet) {
code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// data
code = tsdbReadBlockIdx(pCommitter->dReader.pReader, pCommitter->dReader.aBlockIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
pCommitter->dReader.iBlockIdx = 0;
if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) {
pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0);
code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
} else {
pCommitter->dReader.pBlockIdx = NULL;
}
@@ -531,28 +542,32 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
}
wSet.aSttF[wSet.nSttF - 1] = &fStt;
code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
taosArrayClear(pCommitter->dWriter.aBlockIdx);
taosArrayClear(pCommitter->dWriter.aSttBlk);
tMapDataReset(&pCommitter->dWriter.mBlock);
tBlockDataReset(&pCommitter->dWriter.bData);
+#if USE_STREAM_COMPRESSION
+ tDiskDataBuilderClear(pCommitter->dWriter.pBuilder);
+#else
tBlockDataReset(&pCommitter->dWriter.bDatal);
+#endif
// open iter
code = tsdbOpenCommitIter(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d, commit file data start failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) {
int32_t code = 0;
+ int32_t lino = 0;
if (pBlockData->nRow == 0) return code;
@@ -586,24 +601,25 @@ int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapDa
dataBlk.nSubBlock++;
code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1],
((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// put SDataBlk
code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// clear
tBlockDataClear(pBlockData);
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) {
int32_t code = 0;
+ int32_t lino = 0;
SSttBlk sstBlk;
if (pBlockData->nRow == 0) return code;
@@ -626,114 +642,159 @@ int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray
// write
code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// push SSttBlk
if (taosArrayPush(aSttBlk, &sstBlk) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
// clear
tBlockDataClear(pBlockData);
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
+}
-_err:
- tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilder, SArray *aSttBlk) {
+ int32_t code = 0;
+ int32_t lino = 0;
+
+ if (pBuilder->nRow == 0) return code;
+
+ // gnrt
+ const SDiskData *pDiskData;
+ const SBlkInfo *pBlkInfo;
+ code = tGnrtDiskData(pBuilder, &pDiskData, &pBlkInfo);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ SSttBlk sttBlk = {.suid = pBuilder->suid,
+ .minUid = pBlkInfo->minUid,
+ .maxUid = pBlkInfo->maxUid,
+ .minKey = pBlkInfo->minKey,
+ .maxKey = pBlkInfo->maxKey,
+ .minVer = pBlkInfo->minVer,
+ .maxVer = pBlkInfo->maxVer,
+ .nRow = pBuilder->nRow};
+ // write
+ code = tsdbWriteDiskData(pWriter, pDiskData, &sttBlk.bInfo, NULL);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ // push
+ if (taosArrayPush(aSttBlk, &sttBlk) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ TSDB_CHECK_CODE(code, lino, _exit);
+ }
+
+ // clear
+ tDiskDataBuilderClear(pBuilder);
+
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
// write aBlockIdx
code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// write aSttBlk
code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// update file header
code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// upsert SDFileSet
code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// close and sync
code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
if (pCommitter->dReader.pReader) {
code = tsdbDataFReaderClose(&pCommitter->dReader.pReader);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) {
int32_t code = 0;
+ int32_t lino = 0;
while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) {
SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx;
code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbCommitterNextTableData(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter);
static int32_t tsdbCommitFileData(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
// commit file data start
code = tsdbCommitFileDataStart(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// impl
code = tsdbCommitFileDataImpl(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// commit file data end
code = tsdbCommitFileDataEnd(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
- return code;
-
-_err:
- tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
- tsdbDataFReaderClose(&pCommitter->dReader.pReader);
- tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0);
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ tsdbDataFReaderClose(&pCommitter->dReader.pReader);
+ tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0);
+ }
return code;
}
// ----------------------------------------------------------------------------
static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
memset(pCommitter, 0, sizeof(*pCommitter));
ASSERT(pTsdb->mem && pTsdb->imem == NULL);
@@ -754,30 +815,31 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
if (pCommitter->aTbDataP == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tsdbFSCopy(pTsdb, &pCommitter->fs);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
// reader
pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCommitter->dReader.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pCommitter->dReader.bData);
- if (code) goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
// merger
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
@@ -785,33 +847,41 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pIter->bData);
- if (code) goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
// writer
pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
if (pCommitter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pCommitter->dWriter.aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
code = tBlockDataCreate(&pCommitter->dWriter.bData);
- if (code) goto _exit;
+ TSDB_CHECK_CODE(code, lino, _exit);
+#if USE_STREAM_COMPRESSION
+ code = tDiskDataBuilderCreate(&pCommitter->dWriter.pBuilder);
+#else
code = tBlockDataCreate(&pCommitter->dWriter.bDatal);
- if (code) goto _exit;
+#endif
+ TSDB_CHECK_CODE(code, lino, _exit);
_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
@@ -833,13 +903,19 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
taosArrayDestroy(pCommitter->dWriter.aSttBlk);
tMapDataClear(&pCommitter->dWriter.mBlock);
tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
+#if USE_STREAM_COMPRESSION
+ tDiskDataBuilderDestroy(pCommitter->dWriter.pBuilder);
+#else
tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
+#endif
tTSchemaDestroy(pCommitter->skmTable.pTSchema);
tTSchemaDestroy(pCommitter->skmRow.pTSchema);
}
static int32_t tsdbCommitData(SCommitter *pCommitter) {
- int32_t code = 0;
+ int32_t code = 0;
+ int32_t lino = 0;
+
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
@@ -848,30 +924,29 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) {
// start ====================
code = tsdbCommitDataStart(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// impl ====================
pCommitter->nextKey = pMemTable->minKey;
while (pCommitter->nextKey < TSKEY_MAX) {
code = tsdbCommitFileData(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
// end ====================
tsdbCommitDataEnd(pCommitter);
_exit:
- tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow);
- return code;
-
-_err:
- tsdbCommitDataEnd(pCommitter);
- tsdbError("vgId:%d, commit data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitDel(SCommitter *pCommitter) {
- int32_t code = 0;
+ int32_t code = 0;
+ int32_t lino = 0;
+
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
@@ -882,7 +957,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
// start
code = tsdbCommitDelStart(pCommitter);
if (code) {
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
// impl
@@ -918,7 +993,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
_commit_mem_del:
code = tsdbCommitTableDel(pCommitter, pTbData, NULL);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iTbData++;
pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL;
@@ -926,7 +1001,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
_commit_disk_del:
code = tsdbCommitTableDel(pCommitter, NULL, pDelIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iDelIdx++;
pDelIdx = (iDelIdx < nDelIdx) ? (SDelIdx *)taosArrayGet(pCommitter->aDelIdx, iDelIdx) : NULL;
@@ -934,7 +1009,7 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
_commit_mem_and_disk_del:
code = tsdbCommitTableDel(pCommitter, pTbData, pDelIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iTbData++;
pTbData = (iTbData < nTbData) ? (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData) : NULL;
@@ -945,28 +1020,28 @@ static int32_t tsdbCommitDel(SCommitter *pCommitter) {
// end
code = tsdbCommitDelEnd(pCommitter);
- if (code) {
- goto _err;
- }
+ TSDB_CHECK_CODE(code, lino, _exit);
_exit:
- tsdbDebug("vgId:%d, commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel);
- return code;
-
-_err:
- tsdbError("vgId:%d, commit del failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ } else {
+ tsdbDebug("vgId:%d commit del done, nDel:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nDel);
+ }
return code;
}
static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
- int32_t code = 0;
+ int32_t code = 0;
+ int32_t lino = 0;
+
STsdb *pTsdb = pCommitter->pTsdb;
SMemTable *pMemTable = pTsdb->imem;
ASSERT(eno == 0);
code = tsdbFSCommit1(pTsdb, &pCommitter->fs);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// lock
taosThreadRwlockWrlock(&pTsdb->rwLock);
@@ -975,7 +1050,7 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
code = tsdbFSCommit2(pTsdb, &pCommitter->fs);
if (code) {
taosThreadRwlockUnlock(&pTsdb->rwLock);
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
pTsdb->imem = NULL;
@@ -987,16 +1062,12 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) {
tsdbFSDestroy(&pCommitter->fs);
taosArrayDestroy(pCommitter->aTbDataP);
- // if (pCommitter->toMerge) {
- // code = tsdbMerge(pTsdb);
- // if (code) goto _err;
- // }
-
- tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode));
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pTsdb->pVnode), __func__, lino, tstrerror(code));
+ } else {
+ tsdbInfo("vgId:%d tsdb end commit", TD_VID(pTsdb->pVnode));
+ }
return code;
}
@@ -1008,6 +1079,7 @@ static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) {
static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
if (pCommitter->pIter) {
SDataIter *pIter = pCommitter->pIter;
@@ -1085,11 +1157,17 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
}
_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
- int32_t code = 0;
+ int32_t code = 0;
+ int32_t lino = 0;
+
SBlockData *pBlockData = &pCommitter->dWriter.bData;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
@@ -1098,13 +1176,13 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
while (pRowInfo) {
ASSERT(pRowInfo->row.type == 0);
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbNextCommitRow(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo) {
@@ -1119,29 +1197,32 @@ static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
if (pBlockData->nRow >= pCommitter->maxRow) {
code =
tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) {
- int32_t code = 0;
+ int32_t code = 0;
+ int32_t lino = 0;
+
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
SBlockData *pBDataR = &pCommitter->dReader.bData;
SBlockData *pBDataW = &pCommitter->dWriter.bData;
code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
tBlockDataClear(pBDataW);
int32_t iRow = 0;
@@ -1152,7 +1233,7 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row);
if (c < 0) {
code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iRow++;
if (iRow < pBDataR->nRow) {
@@ -1163,13 +1244,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
} else if (c > 0) {
ASSERT(pRowInfo->row.type == 0);
code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbNextCommitRow(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo) {
@@ -1186,13 +1267,13 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
if (pBDataW->nRow >= pCommitter->maxRow) {
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
while (pRow) {
code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iRow++;
if (iRow < pBDataR->nRow) {
@@ -1203,22 +1284,25 @@ static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk)
if (pBDataW->nRow >= pCommitter->maxRow) {
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
- return code;
-
-_err:
- tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
- int32_t code = 0;
+ int32_t code = 0;
+ int32_t lino = 0;
+
SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx;
ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0);
@@ -1237,7 +1321,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
if (c < 0) {
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) {
@@ -1247,13 +1331,13 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
}
} else if (c > 0) {
code = tsdbCommitAheadBlock(pCommitter, pDataBlk);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL;
} else {
code = tsdbCommitMergeBlock(pCommitter, pDataBlk);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) {
@@ -1268,7 +1352,7 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
while (pDataBlk) {
code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
iBlock++;
if (iBlock < pCommitter->dReader.mBlock.nItem) {
@@ -1279,70 +1363,113 @@ static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) {
}
code = tsdbCommitterNextTableData(pCommitter);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
-static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) {
+static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0;
+ int32_t lino = 0;
- SBlockData *pBDatal = &pCommitter->dWriter.bDatal;
- if (pBDatal->suid || pBDatal->uid) {
- if ((pBDatal->suid != id.suid) || (id.suid == 0)) {
- code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
- if (code) goto _exit;
- tBlockDataReset(pBDatal);
+#if USE_STREAM_COMPRESSION
+ SDiskDataBuilder *pBuilder = pCommitter->dWriter.pBuilder;
+ if (pBuilder->suid || pBuilder->uid) {
+ if (!TABLE_SAME_SCHEMA(pBuilder->suid, pBuilder->uid, id.suid, id.uid)) {
+ code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pBuilder, pCommitter->dWriter.aSttBlk);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ tDiskDataBuilderClear(pBuilder);
}
}
- if (!pBDatal->suid && !pBDatal->uid) {
+ if (!pBuilder->suid && !pBuilder->uid) {
+ ASSERT(pCommitter->skmTable.suid == id.suid);
+ ASSERT(pCommitter->skmTable.uid == id.uid);
+ code = tDiskDataBuilderInit(pBuilder, pCommitter->skmTable.pTSchema, &id, pCommitter->cmprAlg, 0);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ }
+#else
+ SBlockData *pBData = &pCommitter->dWriter.bDatal;
+ if (pBData->suid || pBData->uid) {
+ if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) {
+ code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ tBlockDataReset(pBData);
+ }
+ }
+
+ if (!pBData->suid && !pBData->uid) {
ASSERT(pCommitter->skmTable.suid == id.suid);
ASSERT(pCommitter->skmTable.uid == id.uid);
TABLEID tid = {.suid = id.suid, .uid = id.suid ? 0 : id.uid};
- code = tBlockDataInit(pBDatal, &tid, pCommitter->skmTable.pTSchema, NULL, 0);
- if (code) goto _exit;
+ code = tBlockDataInit(pBData, &tid, pCommitter->skmTable.pTSchema, NULL, 0);
+ TSDB_CHECK_CODE(code, lino, _exit);
}
+#endif
_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
SBlockData *pBData = &pCommitter->dWriter.bData;
- SBlockData *pBDatal = &pCommitter->dWriter.bDatal;
+ TABLEID id = {.suid = pBData->suid, .uid = pBData->uid};
- TABLEID id = {.suid = pBData->suid, .uid = pBData->uid};
- code = tsdbInitLastBlockIfNeed(pCommitter, id);
- if (code) goto _err;
+ code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
+ TSDB_CHECK_CODE(code, lino, _exit);
for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) {
TSDBROW row = tsdbRowFromBlockData(pBData, iRow);
- code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid);
- if (code) goto _err;
- if (pBDatal->nRow >= pCommitter->maxRow) {
- code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
- if (code) goto _err;
+#if USE_STREAM_COMPRESSION
+ code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &row, NULL, &id);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) {
+ code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
+ TSDB_CHECK_CODE(code, lino, _exit);
}
+#else
+ code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &row, NULL, id.uid);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ if (pCommitter->dWriter.bDatal.nRow >= pCommitter->maxRow) {
+ code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
+ pCommitter->cmprAlg);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ }
+#endif
}
- return code;
-
-_err:
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
int32_t code = 0;
+ int32_t lino = 0;
SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter);
if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
@@ -1351,67 +1478,102 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
if (pRowInfo == NULL) goto _exit;
- SBlockData *pBData;
if (pCommitter->toLastOnly) {
- pBData = &pCommitter->dWriter.bDatal;
- code = tsdbInitLastBlockIfNeed(pCommitter, id);
- if (code) goto _err;
+ code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ while (pRowInfo) {
+ STSchema *pTSchema = NULL;
+ if (pRowInfo->row.type == 0) {
+ code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
+ TSDB_CHECK_CODE(code, lino, _exit);
+ pTSchema = pCommitter->skmRow.pTSchema;
+ }
+
+#if USE_STREAM_COMPRESSION
+ code = tDiskDataAddRow(pCommitter->dWriter.pBuilder, &pRowInfo->row, pTSchema, &id);
+#else
+ code = tBlockDataAppendRow(&pCommitter->dWriter.bDatal, &pRowInfo->row, pTSchema, id.uid);
+#endif
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ code = tsdbNextCommitRow(pCommitter);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
+ pRowInfo = NULL;
+ }
+
+#if USE_STREAM_COMPRESSION
+ if (pCommitter->dWriter.pBuilder->nRow >= pCommitter->maxRow) {
+ code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ code = tsdbInitSttBlockBuilderIfNeed(pCommitter, id);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ }
+#else
+ if (pCommitter->dWriter.bData.nRow >= pCommitter->maxRow) {
+ code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
+ pCommitter->cmprAlg);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ }
+#endif
+ }
} else {
- pBData = &pCommitter->dWriter.bData;
+ SBlockData *pBData = &pCommitter->dWriter.bData;
ASSERT(pBData->nRow == 0);
- }
- while (pRowInfo) {
- STSchema *pTSchema = NULL;
- if (pRowInfo->row.type == 0) {
- code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
- if (code) goto _err;
- pTSchema = pCommitter->skmRow.pTSchema;
- }
+ while (pRowInfo) {
+ STSchema *pTSchema = NULL;
+ if (pRowInfo->row.type == 0) {
+ code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row));
+ TSDB_CHECK_CODE(code, lino, _exit);
+ pTSchema = pCommitter->skmRow.pTSchema;
+ }
- code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid);
- if (code) goto _err;
+ code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid);
+ TSDB_CHECK_CODE(code, lino, _exit);
- code = tsdbNextCommitRow(pCommitter);
- if (code) goto _err;
+ code = tsdbNextCommitRow(pCommitter);
+ TSDB_CHECK_CODE(code, lino, _exit);
- pRowInfo = tsdbGetCommitRow(pCommitter);
- if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
- pRowInfo = NULL;
- }
+ pRowInfo = tsdbGetCommitRow(pCommitter);
+ if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) {
+ pRowInfo = NULL;
+ }
- if (pBData->nRow >= pCommitter->maxRow) {
- if (pCommitter->toLastOnly) {
- code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg);
- if (code) goto _err;
- } else {
+ if (pBData->nRow >= pCommitter->maxRow) {
code =
tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
+ }
+ }
+
+ if (pBData->nRow) {
+ if (pBData->nRow > pCommitter->minRow) {
+ code =
+ tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ } else {
+ code = tsdbAppendLastBlock(pCommitter);
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
}
- if (!pCommitter->toLastOnly && pBData->nRow) {
- if (pBData->nRow > pCommitter->minRow) {
- code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg);
- if (code) goto _err;
- } else {
- code = tsdbAppendLastBlock(pCommitter);
- if (code) goto _err;
- }
- }
-
_exit:
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
int32_t code = 0;
+ int32_t lino = 0;
SRowInfo *pRowInfo;
TABLEID id = {0};
@@ -1421,36 +1583,36 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
id.uid = pRowInfo->uid;
code = tsdbMoveCommitData(pCommitter, id);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// start
tMapDataReset(&pCommitter->dWriter.mBlock);
// impl
code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataInit(&pCommitter->dReader.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
code = tBlockDataInit(&pCommitter->dWriter.bData, &id, pCommitter->skmTable.pTSchema, NULL, 0);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
/* merge with data in .data file */
code = tsdbMergeTableData(pCommitter, id);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
/* handle remain table data */
code = tsdbCommitTableData(pCommitter, id);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
// end
if (pCommitter->dWriter.mBlock.nItem > 0) {
SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid};
code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
}
}
}
@@ -1458,15 +1620,20 @@ static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) {
id.suid = INT64_MAX;
id.uid = INT64_MAX;
code = tsdbMoveCommitData(pCommitter, id);
- if (code) goto _err;
+ TSDB_CHECK_CODE(code, lino, _exit);
+#if USE_STREAM_COMPRESSION
+ code = tsdbCommitSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.pBuilder, pCommitter->dWriter.aSttBlk);
+#else
code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk,
pCommitter->cmprAlg);
- if (code) goto _err;
+#endif
+ TSDB_CHECK_CODE(code, lino, _exit);
- return code;
-
-_err:
- tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code));
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at line %d since %s", TD_VID(pCommitter->pTsdb->pVnode), __func__, lino,
+ tstrerror(code));
+ }
return code;
}
diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c
index 3bd71f0ea6..43be51a694 100644
--- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c
+++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c
@@ -16,69 +16,682 @@
#include "tsdb.h"
typedef struct SDiskColBuilder SDiskColBuilder;
+
struct SDiskColBuilder {
- uint8_t flags;
- uint8_t *pBitMap;
- int32_t *aOffset;
- int32_t nData;
- uint8_t *pData;
+ int16_t cid;
+ int8_t type;
+ uint8_t cmprAlg;
+ uint8_t calcSma;
+ int8_t flag;
+ int32_t nVal;
+ uint8_t *pBitMap;
+ int32_t offset;
+ SCompressor *pOffC;
+ SCompressor *pValC;
+ SColumnDataAgg sma;
+ uint8_t minSet;
+ uint8_t maxSet;
+ uint8_t *aBuf[2];
};
-int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+// SDiskData ================================================
+static int32_t tDiskDataDestroy(SDiskData *pDiskData) {
int32_t code = 0;
- // TODO
+ pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol);
return code;
}
-// ================================================================
-typedef struct SDiskDataBuilder SDiskDataBuilder;
-struct SDiskDataBuilder {
- SDiskDataHdr hdr;
- SArray *aBlockCol; // SArray
-};
+// SDiskColBuilder ================================================
+#define tDiskColBuilderCreate() \
+ (SDiskColBuilder) { 0 }
-int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) {
- int32_t code = 0;
- // TODO
- return code;
-}
-
-void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
- // TODO
-}
-
-void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) {
- pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, //
- .fmtVer = 0,
- .suid = suid,
- .uid = uid,
- .cmprAlg = cmprAlg};
-}
-
-void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) {
- // TODO
-}
-
-int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) {
+static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) {
int32_t code = 0;
- // uid (todo)
-
- // version (todo)
-
- // TSKEY (todo)
-
- SRowIter iter = {0};
- tRowIterInit(&iter, pRow, pTSchema);
-
- for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) {
+ tFree(pBuilder->pBitMap);
+ if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC);
+ if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC);
+ for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
+ tFree(pBuilder->aBuf[iBuf]);
}
return code;
}
-int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) {
+static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg,
+ uint8_t calcSma) {
int32_t code = 0;
- // TODO
+
+ pBuilder->cid = cid;
+ pBuilder->type = type;
+ pBuilder->cmprAlg = cmprAlg;
+ pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma;
+ pBuilder->flag = 0;
+ pBuilder->nVal = 0;
+ pBuilder->offset = 0;
+
+ if (IS_VAR_DATA_TYPE(type)) {
+ if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code;
+ code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg);
+ if (code) return code;
+ }
+
+ if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code;
+ code = tCompressStart(pBuilder->pValC, type, cmprAlg);
+ if (code) return code;
+
+ if (pBuilder->calcSma) {
+ pBuilder->sma = (SColumnDataAgg){.colId = cid};
+ pBuilder->minSet = 0;
+ pBuilder->maxSet = 0;
+ }
+
return code;
-}
\ No newline at end of file
+}
+
+static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) {
+ int32_t code = 0;
+
+ ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE);
+
+ *pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid,
+ .type = pBuilder->type,
+ .smaOn = pBuilder->calcSma,
+ .flag = pBuilder->flag,
+ .szOrigin = 0,
+ .szBitmap = 0,
+ .szOffset = 0,
+ .szValue = 0,
+ .offset = 0},
+ .pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma};
+
+ if (pBuilder->flag == HAS_NULL) return code;
+
+ // BITMAP
+ if (pBuilder->flag != HAS_VALUE) {
+ int32_t nBit;
+ if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+ nBit = BIT2_SIZE(pBuilder->nVal);
+ } else {
+ nBit = BIT1_SIZE(pBuilder->nVal);
+ }
+
+ code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES);
+ if (code) return code;
+
+ code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
+ if (code) return code;
+
+ pDiskCol->bCol.szBitmap =
+ tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES,
+ pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
+ pDiskCol->pBit = pBuilder->aBuf[0];
+ }
+
+ // OFFSET
+ if (IS_VAR_DATA_TYPE(pBuilder->type)) {
+ code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL);
+ if (code) return code;
+ }
+
+ // VALUE
+ if (pBuilder->flag != (HAS_NULL | HAS_NONE)) {
+ code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin);
+ if (code) return code;
+ }
+
+ return code;
+}
+
+static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ if (IS_VAR_DATA_TYPE(pColVal->type)) {
+ code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t));
+ if (code) return code;
+ pBuilder->offset += pColVal->value.nData;
+
+ code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData);
+ if (code) return code;
+ } else {
+ code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes);
+ if (code) return code;
+ }
+
+ return code;
+}
+static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ pBuilder->flag = HAS_VALUE;
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ pBuilder->flag = HAS_NONE;
+ return 0;
+}
+static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ pBuilder->flag = HAS_NULL;
+ return 0;
+}
+static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ // bit map
+ int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
+ code = tRealloc(&pBuilder->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pBuilder->pBitMap, 0, nBit);
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
+
+ // value
+ pBuilder->flag |= HAS_VALUE;
+
+ SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
+ for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
+ code = tDiskColPutValue(pBuilder, &cv);
+ if (code) return code;
+ }
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
+ code = tRealloc(&pBuilder->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pBuilder->pBitMap, 0, nBit);
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
+
+ pBuilder->flag |= HAS_NULL;
+
+ return code;
+}
+static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
+ code = tRealloc(&pBuilder->pBitMap, nBit);
+ if (code) return code;
+
+ pBuilder->flag |= HAS_VALUE;
+
+ memset(pBuilder->pBitMap, 0, nBit);
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
+
+ SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
+ for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
+ code = tDiskColPutValue(pBuilder, &cv);
+ if (code) return code;
+ }
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
+ code = tRealloc(&pBuilder->pBitMap, nBit);
+ if (code) return code;
+
+ pBuilder->flag |= HAS_NONE;
+
+ memset(pBuilder->pBitMap, 255, nBit);
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
+
+ return code;
+}
+static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pBuilder->flag |= HAS_VALUE;
+
+ uint8_t *pBitMap = NULL;
+ code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+
+ for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
+ SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal));
+ }
+ SET_BIT2(pBitMap, pBuilder->nVal, 2);
+
+ tFree(pBuilder->pBitMap);
+ pBuilder->pBitMap = pBitMap;
+
+ SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
+ for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
+ code = tDiskColPutValue(pBuilder, &cv);
+ if (code) return code;
+ }
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
+
+ return code;
+}
+static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
+
+ return code;
+}
+static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pBuilder->flag |= HAS_NONE;
+
+ int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
+ code = tRealloc(&pBuilder->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pBuilder->pBitMap, 255, nBit);
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pBuilder->flag |= HAS_NULL;
+
+ int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
+ code = tRealloc(&pBuilder->pBitMap, nBit);
+ if (code) return code;
+
+ memset(pBuilder->pBitMap, 255, nBit);
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pBuilder->flag |= HAS_NULL;
+
+ uint8_t *pBitMap = NULL;
+ code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+
+ for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
+ SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0);
+ }
+ SET_BIT2(pBitMap, pBuilder->nVal, 1);
+
+ tFree(pBuilder->pBitMap);
+ pBuilder->pBitMap = pBitMap;
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ pBuilder->flag |= HAS_NONE;
+
+ uint8_t *pBitMap = NULL;
+ code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+
+ for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
+ SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1);
+ }
+ SET_BIT2(pBitMap, pBuilder->nVal, 0);
+
+ tFree(pBuilder->pBitMap);
+ pBuilder->pBitMap = pBitMap;
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
+ if (code) return code;
+ SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1);
+
+ return tDiskColPutValue(pBuilder, pColVal);
+}
+static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = {
+ {tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0
+ {tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE
+ {tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL
+ {tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE
+ {tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE
+ {tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE
+ {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL
+ {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE
+};
+extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet);
+static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
+ int32_t code = 0;
+
+ if (pBuilder->calcSma) {
+ if (COL_VAL_IS_VALUE(pColVal)) {
+ tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet);
+ } else {
+ pBuilder->sma.numOfNull++;
+ }
+ }
+
+ if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) {
+ code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal);
+ if (code) return code;
+ }
+
+ pBuilder->nVal++;
+
+ return code;
+}
+
+// SDiskDataBuilder ================================================
+int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) {
+ int32_t code = 0;
+
+ *ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder));
+ if (*ppBuilder == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
+
+ return code;
+}
+
+void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
+ if (pBuilder == NULL) return NULL;
+
+ if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC);
+ if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC);
+ if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC);
+
+ if (pBuilder->aBuilder) {
+ for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) {
+ SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
+ tDiskColBuilderDestroy(pDCBuilder);
+ }
+ taosArrayDestroy(pBuilder->aBuilder);
+ }
+ for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
+ tFree(pBuilder->aBuf[iBuf]);
+ }
+ tDiskDataDestroy(&pBuilder->dd);
+ taosMemoryFree(pBuilder);
+
+ return NULL;
+}
+
+int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
+ uint8_t calcSma) {
+ int32_t code = 0;
+
+ ASSERT(pId->suid || pId->uid);
+
+ pBuilder->suid = pId->suid;
+ pBuilder->uid = pId->uid;
+ pBuilder->nRow = 0;
+ pBuilder->cmprAlg = cmprAlg;
+ pBuilder->calcSma = calcSma;
+ pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX,
+ .maxUid = INT64_MIN,
+ .minKey = TSKEY_MAX,
+ .maxKey = TSKEY_MIN,
+ .minVer = VERSION_MAX,
+ .maxVer = VERSION_MIN,
+ .minTKey = TSDBKEY_MAX,
+ .maxTKey = TSDBKEY_MIN};
+
+ if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code;
+ code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
+ if (code) return code;
+
+ if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code;
+ code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
+ if (code) return code;
+
+ if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code;
+ code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg);
+ if (code) return code;
+
+ if (pBuilder->aBuilder == NULL) {
+ pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder));
+ if (pBuilder->aBuilder == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
+ }
+
+ pBuilder->nBuilder = 0;
+ for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
+ STColumn *pTColumn = &pTSchema->columns[iCol];
+
+ if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) {
+ SDiskColBuilder dc = tDiskColBuilderCreate();
+ if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
+ }
+
+ SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder);
+
+ code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg,
+ (calcSma && (pTColumn->flags & COL_SMA_ON)));
+ if (code) return code;
+
+ pBuilder->nBuilder++;
+ }
+
+ return code;
+}
+
+int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) {
+ int32_t code = 0;
+ pBuilder->suid = 0;
+ pBuilder->uid = 0;
+ pBuilder->nRow = 0;
+ return code;
+}
+
+int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) {
+ int32_t code = 0;
+
+ ASSERT(pBuilder->suid || pBuilder->uid);
+ ASSERT(pId->suid == pBuilder->suid);
+
+ TSDBKEY kRow = TSDBROW_KEY(pRow);
+ if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow;
+ if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow;
+
+ // uid
+ if (pBuilder->uid && pBuilder->uid != pId->uid) {
+ ASSERT(pBuilder->suid);
+ for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) {
+ code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t));
+ if (code) return code;
+ }
+ pBuilder->uid = 0;
+ }
+ if (pBuilder->uid == 0) {
+ code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t));
+ if (code) return code;
+ }
+ if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid;
+ if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid;
+
+ // version
+ code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t));
+ if (code) return code;
+ if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version;
+ if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version;
+
+ // TSKEY
+ code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t));
+ if (code) return code;
+ if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts;
+ if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts;
+
+ SRowIter iter = {0};
+ tRowIterInit(&iter, pRow, pTSchema);
+
+ SColVal *pColVal = tRowIterNext(&iter);
+ for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
+ SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
+
+ while (pColVal && pColVal->cid < pDCBuilder->cid) {
+ pColVal = tRowIterNext(&iter);
+ }
+
+ if (pColVal && pColVal->cid == pDCBuilder->cid) {
+ code = tDiskColAddVal(pDCBuilder, pColVal);
+ if (code) return code;
+ pColVal = tRowIterNext(&iter);
+ } else {
+ code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type));
+ if (code) return code;
+ }
+ }
+ pBuilder->nRow++;
+
+ return code;
+}
+
+int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) {
+ int32_t code = 0;
+
+ ASSERT(pBuilder->nRow);
+
+ *ppDiskData = NULL;
+ *ppBlkInfo = NULL;
+
+ SDiskData *pDiskData = &pBuilder->dd;
+ // reset SDiskData
+ pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT,
+ .fmtVer = 0,
+ .suid = pBuilder->suid,
+ .uid = pBuilder->uid,
+ .szUid = 0,
+ .szVer = 0,
+ .szKey = 0,
+ .szBlkCol = 0,
+ .nRow = pBuilder->nRow,
+ .cmprAlg = pBuilder->cmprAlg};
+ pDiskData->pUid = NULL;
+ pDiskData->pVer = NULL;
+ pDiskData->pKey = NULL;
+
+ // UID
+ if (pBuilder->uid == 0) {
+ code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL);
+ if (code) return code;
+ }
+
+ // VERSION
+ code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL);
+ if (code) return code;
+
+ // TSKEY
+ code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL);
+ if (code) return code;
+
+ // aDiskCol
+ if (pDiskData->aDiskCol) {
+ taosArrayClear(pDiskData->aDiskCol);
+ } else {
+ pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol));
+ if (pDiskData->aDiskCol == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
+ }
+
+ int32_t offset = 0;
+ for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
+ SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
+
+ if (pDCBuilder->flag == HAS_NONE) continue;
+
+ SDiskCol dCol;
+
+ code = tGnrtDiskCol(pDCBuilder, &dCol);
+ if (code) return code;
+
+ dCol.bCol.offset = offset;
+ offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue;
+
+ if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ return code;
+ }
+
+ pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol);
+ }
+
+ *ppDiskData = pDiskData;
+ *ppBlkInfo = &pBuilder->bi;
+ return code;
+}
diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
index fc577e3962..167b4a104c 100644
--- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
@@ -128,7 +128,7 @@ _exit:
return code;
}
-static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
+static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, const uint8_t *pBuf, int64_t size) {
int32_t code = 0;
int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
@@ -522,9 +522,6 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData,
// write
if (pSmaInfo->size) {
- code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size);
- if (code) goto _err;
-
code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size);
if (code) goto _err;
@@ -607,6 +604,132 @@ _err:
return code;
}
+int32_t tsdbWriteDiskData(SDataFWriter *pWriter, const SDiskData *pDiskData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo) {
+ int32_t code = 0;
+ int32_t lino = 0;
+
+ STsdbFD *pFD = NULL;
+ if (pSmaInfo) {
+ pFD = pWriter->pDataFD;
+ pBlkInfo->offset = pWriter->fData.size;
+ } else {
+ pFD = pWriter->pSttFD;
+ pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size;
+ }
+ pBlkInfo->szBlock = 0;
+ pBlkInfo->szKey = 0;
+
+ // hdr
+ int32_t n = tPutDiskDataHdr(NULL, &pDiskData->hdr);
+ code = tRealloc(&pWriter->aBuf[0], n);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ tPutDiskDataHdr(pWriter->aBuf[0], &pDiskData->hdr);
+
+ code = tsdbWriteFile(pFD, pBlkInfo->offset, pWriter->aBuf[0], n);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ pBlkInfo->szKey += n;
+ pBlkInfo->szBlock += n;
+
+ // uid + ver + key
+ if (pDiskData->pUid) {
+ code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pUid, pDiskData->hdr.szUid);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ pBlkInfo->szKey += pDiskData->hdr.szUid;
+ pBlkInfo->szBlock += pDiskData->hdr.szUid;
+ }
+
+ code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pVer, pDiskData->hdr.szVer);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ pBlkInfo->szKey += pDiskData->hdr.szVer;
+ pBlkInfo->szBlock += pDiskData->hdr.szVer;
+
+ code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskData->pKey, pDiskData->hdr.szKey);
+ TSDB_CHECK_CODE(code, lino, _exit);
+ pBlkInfo->szKey += pDiskData->hdr.szKey;
+ pBlkInfo->szBlock += pDiskData->hdr.szKey;
+
+ // aBlockCol
+ if (pDiskData->hdr.szBlkCol) {
+ code = tRealloc(&pWriter->aBuf[0], pDiskData->hdr.szBlkCol);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ n = 0;
+ for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
+ SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
+ n += tPutBlockCol(pWriter->aBuf[0] + n, pDiskCol);
+ }
+ ASSERT(n == pDiskData->hdr.szBlkCol);
+
+ code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pWriter->aBuf[0], pDiskData->hdr.szBlkCol);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ pBlkInfo->szBlock += pDiskData->hdr.szBlkCol;
+ }
+
+ // aDiskCol
+ for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
+ SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
+
+ if (pDiskCol->pBit) {
+ code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pBit, pDiskCol->bCol.szBitmap);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ pBlkInfo->szBlock += pDiskCol->bCol.szBitmap;
+ }
+
+ if (pDiskCol->pOff) {
+ code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pOff, pDiskCol->bCol.szOffset);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ pBlkInfo->szBlock += pDiskCol->bCol.szOffset;
+ }
+
+ if (pDiskCol->pVal) {
+ code = tsdbWriteFile(pFD, pBlkInfo->offset + pBlkInfo->szBlock, pDiskCol->pVal, pDiskCol->bCol.szValue);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ pBlkInfo->szBlock += pDiskCol->bCol.szValue;
+ }
+ }
+
+ if (pSmaInfo) {
+ pWriter->fData.size += pBlkInfo->szBlock;
+ } else {
+ pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock;
+ goto _exit;
+ }
+
+ pSmaInfo->offset = 0;
+ pSmaInfo->size = 0;
+ for (int32_t iDiskCol = 0; iDiskCol < taosArrayGetSize(pDiskData->aDiskCol); iDiskCol++) {
+ SDiskCol *pDiskCol = (SDiskCol *)taosArrayGet(pDiskData->aDiskCol, iDiskCol);
+
+ if (IS_VAR_DATA_TYPE(pDiskCol->bCol.type)) continue;
+ if (pDiskCol->bCol.flag == HAS_NULL || pDiskCol->bCol.flag == (HAS_NULL | HAS_NONE)) continue;
+ if (!pDiskCol->bCol.smaOn) continue;
+
+ code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &pDiskCol->agg));
+ TSDB_CHECK_CODE(code, lino, _exit);
+ pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &pDiskCol->agg);
+ }
+
+ if (pSmaInfo->size) {
+ pSmaInfo->offset = pWriter->fSma.size;
+
+ code = tsdbWriteFile(pWriter->pSmaFD, pSmaInfo->offset, pWriter->aBuf[0], pSmaInfo->size);
+ TSDB_CHECK_CODE(code, lino, _exit);
+
+ pWriter->fSma.size += pSmaInfo->size;
+ }
+
+_exit:
+ if (code) {
+ tsdbError("vgId:%d %s failed at %d since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, lino, tstrerror(code));
+ }
+ return code;
+}
+
int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
int32_t code = 0;
int64_t n;
@@ -946,8 +1069,8 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
ASSERT(pBlockData->suid == hdr.suid);
- ASSERT(pBlockData->uid == hdr.uid);
+ pBlockData->uid = hdr.uid;
pBlockData->nRow = hdr.nRow;
// uid
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index 4999e7a49a..47c836d9c1 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -649,7 +649,7 @@ int32_t tRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRo
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
- *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts});
+ *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
@@ -690,7 +690,7 @@ int32_t tRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
STColumn *pTColumn;
int32_t iCol, jCol = 1;
- ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts);
+ ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) {
pTColumn = &pMerger->pTSchema->columns[iCol];
@@ -744,7 +744,7 @@ int32_t tRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) {
ASSERT(pTColumn->type == TSDB_DATA_TYPE_TIMESTAMP);
- *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.ts = key.ts});
+ *pColVal = COL_VAL_VALUE(pTColumn->colId, pTColumn->type, (SValue){.val = key.ts});
if (taosArrayPush(pMerger->pArray, pColVal) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit;
@@ -770,7 +770,7 @@ int32_t tRowMerge(SRowMerger *pMerger, TSDBROW *pRow) {
TSDBKEY key = TSDBROW_KEY(pRow);
SColVal *pColVal = &(SColVal){0};
- ASSERT(((SColVal *)pMerger->pArray->pData)->value.ts == key.ts);
+ ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts);
for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) {
tsdbRowGetColVal(pRow, pMerger->pTSchema, iCol, pColVal);
@@ -1105,9 +1105,8 @@ int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
pColVal = tRowIterNext(&rIter);
}
}
-
-_exit:
pBlockData->nRow++;
+
return code;
_err:
@@ -1455,9 +1454,8 @@ _exit:
}
// SDiskDataHdr ==============================
-int32_t tPutDiskDataHdr(uint8_t *p, void *ph) {
- int32_t n = 0;
- SDiskDataHdr *pHdr = (SDiskDataHdr *)ph;
+int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) {
+ int32_t n = 0;
n += tPutU32(p ? p + n : p, pHdr->delimiter);
n += tPutU32v(p ? p + n : p, pHdr->fmtVer);
@@ -1516,174 +1514,107 @@ int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
return n;
}
+#define SMA_UPDATE(SUM_V, MIN_V, MAX_V, VAL, MINSET, MAXSET) \
+ do { \
+ (SUM_V) += (VAL); \
+ if (!(MINSET)) { \
+ (MIN_V) = (VAL); \
+ (MINSET) = 1; \
+ } else if ((MIN_V) > (VAL)) { \
+ (MIN_V) = (VAL); \
+ } \
+ if (!(MAXSET)) { \
+ (MAX_V) = (VAL); \
+ (MAXSET) = 1; \
+ } else if ((MAX_V) < (VAL)) { \
+ (MAX_V) = (VAL); \
+ } \
+ } while (0)
+
+static FORCE_INLINE void tSmaUpdateBool(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
+ int8_t val = *(int8_t *)&pColVal->value.val ? 1 : 0;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
+ uint8_t *maxSet) {
+ int8_t val = *(int8_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
+ uint8_t *maxSet) {
+ int16_t val = *(int16_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
+ int32_t val = *(int32_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
+ int64_t val = *(int64_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateFloat(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
+ float val = *(float *)&pColVal->value.val;
+ SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateDouble(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
+ double val = *(double *)&pColVal->value.val;
+ SMA_UPDATE(*(double *)&pColAgg->sum, *(double *)&pColAgg->min, *(double *)&pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateUTinyint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
+ uint8_t *maxSet) {
+ uint8_t val = *(uint8_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateUSmallint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
+ uint8_t *maxSet) {
+ uint16_t val = *(uint16_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateUInt(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) {
+ uint32_t val = *(uint32_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+static FORCE_INLINE void tSmaUpdateUBigint(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet,
+ uint8_t *maxSet) {
+ uint64_t val = *(uint64_t *)&pColVal->value.val;
+ SMA_UPDATE(pColAgg->sum, pColAgg->min, pColAgg->max, val, *minSet, *maxSet);
+}
+void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet) = {
+ NULL,
+ tSmaUpdateBool, // TSDB_DATA_TYPE_BOOL
+ tSmaUpdateTinyint, // TSDB_DATA_TYPE_TINYINT
+ tSmaUpdateSmallint, // TSDB_DATA_TYPE_SMALLINT
+ tSmaUpdateInt, // TSDB_DATA_TYPE_INT
+ tSmaUpdateBigint, // TSDB_DATA_TYPE_BIGINT
+ tSmaUpdateFloat, // TSDB_DATA_TYPE_FLOAT
+ tSmaUpdateDouble, // TSDB_DATA_TYPE_DOUBLE
+ NULL, // TSDB_DATA_TYPE_VARCHAR
+ tSmaUpdateBigint, // TSDB_DATA_TYPE_TIMESTAMP
+ NULL, // TSDB_DATA_TYPE_NCHAR
+ tSmaUpdateUTinyint, // TSDB_DATA_TYPE_UTINYINT
+ tSmaUpdateUSmallint, // TSDB_DATA_TYPE_USMALLINT
+ tSmaUpdateUInt, // TSDB_DATA_TYPE_UINT
+ tSmaUpdateUBigint, // TSDB_DATA_TYPE_UBIGINT
+ NULL, // TSDB_DATA_TYPE_JSON
+ NULL, // TSDB_DATA_TYPE_VARBINARY
+ NULL, // TSDB_DATA_TYPE_DECIMAL
+ NULL, // TSDB_DATA_TYPE_BLOB
+ NULL, // TSDB_DATA_TYPE_MEDIUMBLOB
+};
void tsdbCalcColDataSMA(SColData *pColData, SColumnDataAgg *pColAgg) {
- SColVal colVal;
- SColVal *pColVal = &colVal;
-
- memset(pColAgg, 0, sizeof(*pColAgg));
- bool minAssigned = false;
- bool maxAssigned = false;
-
*pColAgg = (SColumnDataAgg){.colId = pColData->cid};
- for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
- tColDataGetValue(pColData, iVal, pColVal);
+ uint8_t minSet = 0;
+ uint8_t maxSet = 0;
- if (!COL_VAL_IS_VALUE(pColVal)) {
- pColAgg->numOfNull++;
+ SColVal cv;
+ for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+ tColDataGetValue(pColData, iVal, &cv);
+
+ if (COL_VAL_IS_VALUE(&cv)) {
+ tSmaUpdateImpl[pColData->type](pColAgg, &cv, &minSet, &maxSet);
} else {
- switch (pColData->type) {
- case TSDB_DATA_TYPE_NULL:
- break;
- case TSDB_DATA_TYPE_BOOL:
- break;
- case TSDB_DATA_TYPE_TINYINT: {
- pColAgg->sum += colVal.value.i8;
- if (!minAssigned || pColAgg->min > colVal.value.i8) {
- pColAgg->min = colVal.value.i8;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.i8) {
- pColAgg->max = colVal.value.i8;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_SMALLINT: {
- pColAgg->sum += colVal.value.i16;
- if (!minAssigned || pColAgg->min > colVal.value.i16) {
- pColAgg->min = colVal.value.i16;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.i16) {
- pColAgg->max = colVal.value.i16;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_INT: {
- pColAgg->sum += colVal.value.i32;
- if (!minAssigned || pColAgg->min > colVal.value.i32) {
- pColAgg->min = colVal.value.i32;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.i32) {
- pColAgg->max = colVal.value.i32;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_BIGINT: {
- pColAgg->sum += colVal.value.i64;
- if (!minAssigned || pColAgg->min > colVal.value.i64) {
- pColAgg->min = colVal.value.i64;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.i64) {
- pColAgg->max = colVal.value.i64;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_FLOAT: {
- *(double *)(&pColAgg->sum) += colVal.value.f;
- if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.f) {
- *(double *)(&pColAgg->min) = colVal.value.f;
- minAssigned = true;
- }
- if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.f) {
- *(double *)(&pColAgg->max) = colVal.value.f;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_DOUBLE: {
- *(double *)(&pColAgg->sum) += colVal.value.d;
- if (!minAssigned || *(double *)(&pColAgg->min) > colVal.value.d) {
- *(double *)(&pColAgg->min) = colVal.value.d;
- minAssigned = true;
- }
- if (!maxAssigned || *(double *)(&pColAgg->max) < colVal.value.d) {
- *(double *)(&pColAgg->max) = colVal.value.d;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_VARCHAR:
- break;
- case TSDB_DATA_TYPE_TIMESTAMP: {
- if (!minAssigned || pColAgg->min > colVal.value.i64) {
- pColAgg->min = colVal.value.i64;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.i64) {
- pColAgg->max = colVal.value.i64;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_NCHAR:
- break;
- case TSDB_DATA_TYPE_UTINYINT: {
- pColAgg->sum += colVal.value.u8;
- if (!minAssigned || pColAgg->min > colVal.value.u8) {
- pColAgg->min = colVal.value.u8;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.u8) {
- pColAgg->max = colVal.value.u8;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_USMALLINT: {
- pColAgg->sum += colVal.value.u16;
- if (!minAssigned || pColAgg->min > colVal.value.u16) {
- pColAgg->min = colVal.value.u16;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.u16) {
- pColAgg->max = colVal.value.u16;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_UINT: {
- pColAgg->sum += colVal.value.u32;
- if (!minAssigned || pColAgg->min > colVal.value.u32) {
- pColAgg->min = colVal.value.u32;
- minAssigned = true;
- }
- if (!minAssigned || pColAgg->max < colVal.value.u32) {
- pColAgg->max = colVal.value.u32;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_UBIGINT: {
- pColAgg->sum += colVal.value.u64;
- if (!minAssigned || pColAgg->min > colVal.value.u64) {
- pColAgg->min = colVal.value.u64;
- minAssigned = true;
- }
- if (!maxAssigned || pColAgg->max < colVal.value.u64) {
- pColAgg->max = colVal.value.u64;
- maxAssigned = true;
- }
- break;
- }
- case TSDB_DATA_TYPE_JSON:
- break;
- case TSDB_DATA_TYPE_VARBINARY:
- break;
- case TSDB_DATA_TYPE_DECIMAL:
- break;
- case TSDB_DATA_TYPE_BLOB:
- break;
- case TSDB_DATA_TYPE_MEDIUMBLOB:
- break;
- default:
- ASSERT(0);
- }
+ pColAgg->numOfNull++;
}
}
}
diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c
index 62b9d87628..be220c2585 100644
--- a/source/util/src/tcompression.c
+++ b/source/util/src/tcompression.c
@@ -62,8 +62,8 @@ static const int32_t TEST_NUMBER = 1;
#define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL)
#define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a)))
-#define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode
-#define ZIGZAG_DECODE(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode
+#define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode
+#define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode
#ifdef TD_TSZ
bool lossyFloat = false;
@@ -999,41 +999,212 @@ int32_t tsDecompressDoubleLossyImp(const char *input, int32_t compressedSize, co
/*************************************************************************
* STREAM COMPRESSION
*************************************************************************/
-#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
-typedef struct SCompressor SCompressor;
+#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (a)) || ((a) < 0 && (b) >= INT64_MIN - (a)))
+static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
+
+static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
+
+static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
+
+static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
+
+static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
+
+static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg);
static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData);
+static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData);
+
+static FORCE_INLINE int64_t tGetI64OfI8(const void *pData) { return *(int8_t *)pData; }
+static FORCE_INLINE int64_t tGetI64OfI16(const void *pData) { return *(int16_t *)pData; }
+static FORCE_INLINE int64_t tGetI64OfI32(const void *pData) { return *(int32_t *)pData; }
+static FORCE_INLINE int64_t tGetI64OfI64(const void *pData) { return *(int64_t *)pData; }
+
+static FORCE_INLINE void tPutI64OfI8(int64_t v, void *pData) { *(int8_t *)pData = v; }
+static FORCE_INLINE void tPutI64OfI16(int64_t v, void *pData) { *(int16_t *)pData = v; }
+static FORCE_INLINE void tPutI64OfI32(int64_t v, void *pData) { *(int32_t *)pData = v; }
+static FORCE_INLINE void tPutI64OfI64(int64_t v, void *pData) { *(int64_t *)pData = v; }
+
static struct {
int8_t type;
int32_t bytes;
int8_t isVarLen;
+ int32_t (*startFn)(SCompressor *, int8_t type, int8_t cmprAlg);
int32_t (*cmprFn)(SCompressor *, const void *, int32_t nData);
+ int32_t (*endFn)(SCompressor *, const uint8_t **, int32_t *);
+ int64_t (*getI64)(const void *pData);
+ void (*putI64)(int64_t v, void *pData);
} DATA_TYPE_INFO[] = {
- {TSDB_DATA_TYPE_NULL, 0, 0, NULL}, // TSDB_DATA_TYPE_NULL
- {TSDB_DATA_TYPE_BOOL, 1, 0, tCompBool}, // TSDB_DATA_TYPE_BOOL
- {TSDB_DATA_TYPE_TINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_TINYINT
- {TSDB_DATA_TYPE_SMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_SMALLINT
- {TSDB_DATA_TYPE_INT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_INT
- {TSDB_DATA_TYPE_BIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_BIGINT
- {TSDB_DATA_TYPE_FLOAT, 4, 0, tCompFloat}, // TSDB_DATA_TYPE_FLOAT
- {TSDB_DATA_TYPE_DOUBLE, 8, 0, tCompDouble}, // TSDB_DATA_TYPE_DOUBLE
- {TSDB_DATA_TYPE_VARCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARCHAR
- {TSDB_DATA_TYPE_TIMESTAMP, 8, 0, tCompTimestamp}, // pTSDB_DATA_TYPE_TIMESTAMP
- {TSDB_DATA_TYPE_NCHAR, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_NCHAR
- {TSDB_DATA_TYPE_UTINYINT, 1, 0, tCompInt}, // TSDB_DATA_TYPE_UTINYINT
- {TSDB_DATA_TYPE_USMALLINT, 2, 0, tCompInt}, // TSDB_DATA_TYPE_USMALLINT
- {TSDB_DATA_TYPE_UINT, 4, 0, tCompInt}, // TSDB_DATA_TYPE_UINT
- {TSDB_DATA_TYPE_UBIGINT, 8, 0, tCompInt}, // TSDB_DATA_TYPE_UBIGINT
- {TSDB_DATA_TYPE_JSON, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_JSON
- {TSDB_DATA_TYPE_VARBINARY, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_VARBINARY
- {TSDB_DATA_TYPE_DECIMAL, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_DECIMAL
- {TSDB_DATA_TYPE_BLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_BLOB
- {TSDB_DATA_TYPE_MEDIUMBLOB, 1, 1, tCompBinary}, // TSDB_DATA_TYPE_MEDIUMBLOB
+ {.type = TSDB_DATA_TYPE_NULL,
+ .bytes = 0,
+ .isVarLen = 0,
+ .startFn = NULL,
+ .cmprFn = NULL,
+ .endFn = NULL,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_BOOL,
+ .bytes = 1,
+ .isVarLen = 0,
+ .startFn = tCompBoolStart,
+ .cmprFn = tCompBool,
+ .endFn = tCompBoolEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_TINYINT,
+ .bytes = 1,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI8,
+ .putI64 = tPutI64OfI8},
+ {.type = TSDB_DATA_TYPE_SMALLINT,
+ .bytes = 2,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI16,
+ .putI64 = tPutI64OfI16},
+ {.type = TSDB_DATA_TYPE_INT,
+ .bytes = 4,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI32,
+ .putI64 = tPutI64OfI32},
+ {.type = TSDB_DATA_TYPE_BIGINT,
+ .bytes = 8,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI64,
+ .putI64 = tPutI64OfI64},
+ {.type = TSDB_DATA_TYPE_FLOAT,
+ .bytes = 4,
+ .isVarLen = 0,
+ .startFn = tCompFloatStart,
+ .cmprFn = tCompFloat,
+ .endFn = tCompFloatEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_DOUBLE,
+ .bytes = 8,
+ .isVarLen = 0,
+ .startFn = tCompDoubleStart,
+ .cmprFn = tCompDouble,
+ .endFn = tCompDoubleEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_VARCHAR,
+ .bytes = 1,
+ .isVarLen = 1,
+ .startFn = tCompBinaryStart,
+ .cmprFn = tCompBinary,
+ .endFn = tCompBinaryEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_TIMESTAMP,
+ .bytes = 8,
+ .isVarLen = 0,
+ .startFn = tCompTimestampStart,
+ .cmprFn = tCompTimestamp,
+ .endFn = tCompTimestampEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_NCHAR,
+ .bytes = 1,
+ .isVarLen = 1,
+ .startFn = tCompBinaryStart,
+ .cmprFn = tCompBinary,
+ .endFn = tCompBinaryEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_UTINYINT,
+ .bytes = 1,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI8,
+ .putI64 = tPutI64OfI8},
+ {.type = TSDB_DATA_TYPE_USMALLINT,
+ .bytes = 2,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI16,
+ .putI64 = tPutI64OfI16},
+ {.type = TSDB_DATA_TYPE_UINT,
+ .bytes = 4,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI32,
+ .putI64 = tPutI64OfI32},
+ {.type = TSDB_DATA_TYPE_UBIGINT,
+ .bytes = 8,
+ .isVarLen = 0,
+ .startFn = tCompIntStart,
+ .cmprFn = tCompInt,
+ .endFn = tCompIntEnd,
+ .getI64 = tGetI64OfI64,
+ .putI64 = tPutI64OfI64},
+ {.type = TSDB_DATA_TYPE_JSON,
+ .bytes = 1,
+ .isVarLen = 1,
+ .startFn = tCompBinaryStart,
+ .cmprFn = tCompBinary,
+ .endFn = tCompBinaryEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_VARBINARY,
+ .bytes = 1,
+ .isVarLen = 1,
+ .startFn = tCompBinaryStart,
+ .cmprFn = tCompBinary,
+ .endFn = tCompBinaryEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_DECIMAL,
+ .bytes = 1,
+ .isVarLen = 1,
+ .startFn = tCompBinaryStart,
+ .cmprFn = tCompBinary,
+ .endFn = tCompBinaryEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_BLOB,
+ .bytes = 1,
+ .isVarLen = 1,
+ .startFn = tCompBinaryStart,
+ .cmprFn = tCompBinary,
+ .endFn = tCompBinaryEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
+ {.type = TSDB_DATA_TYPE_MEDIUMBLOB,
+ .bytes = 1,
+ .isVarLen = 1,
+ .startFn = tCompBinaryStart,
+ .cmprFn = tCompBinary,
+ .endFn = tCompBinaryEnd,
+ .getI64 = NULL,
+ .putI64 = NULL},
};
struct SCompressor {
@@ -1041,8 +1212,9 @@ struct SCompressor {
int8_t cmprAlg;
int8_t autoAlloc;
int32_t nVal;
- uint8_t *aBuf[2];
- int64_t nBuf[2];
+ uint8_t *pBuf;
+ int32_t nBuf;
+ uint8_t *aBuf[1];
union {
// Timestamp ----
struct {
@@ -1056,6 +1228,7 @@ struct SCompressor {
int32_t i_selector;
int32_t i_start;
int32_t i_end;
+ int32_t i_nEle;
uint64_t i_aZigzag[241];
int8_t i_aBitN[241];
};
@@ -1072,86 +1245,114 @@ struct SCompressor {
};
};
-// Timestamp =====================================================
-static int32_t tCompSetCopyMode(SCompressor *pCmprsor) {
+static int32_t tTwoStageComp(SCompressor *pCmprsor, int32_t *szComp) {
int32_t code = 0;
- if (pCmprsor->nVal) {
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[1], sizeof(int64_t) * pCmprsor->nVal);
- if (code) return code;
- }
- pCmprsor->nBuf[1] = 0;
-
- int64_t n = 1;
- int64_t value;
- int64_t delta;
- uint64_t vZigzag;
- while (n < pCmprsor->nBuf[0]) {
- uint8_t aN[2];
- aN[0] = pCmprsor->aBuf[0][n] & 0xf;
- aN[1] = pCmprsor->aBuf[0][n] >> 4;
-
- n++;
-
- for (int32_t i = 0; i < 2; i++) {
- vZigzag = 0;
- for (uint8_t j = 0; j < aN[i]; j++) {
- vZigzag |= (((uint64_t)pCmprsor->aBuf[0][n]) << (8 * j));
- n++;
- }
-
- int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
- if (pCmprsor->nBuf[1] == 0) {
- delta = 0;
- value = delta_of_delta;
- } else {
- delta = delta_of_delta + delta;
- value = delta + value;
- }
-
- memcpy(pCmprsor->aBuf[1] + pCmprsor->nBuf[1], &value, sizeof(int64_t));
- pCmprsor->nBuf[1] += sizeof(int64_t);
-
- if (n >= pCmprsor->nBuf[0]) break;
- }
- }
-
- ASSERT(n == pCmprsor->nBuf[0]);
-
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[1] + 1);
- if (code) return code;
- }
- memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->aBuf[1], pCmprsor->nBuf[1]);
- pCmprsor->nBuf[0] = 1 + pCmprsor->nBuf[1];
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf + 1))) {
+ return code;
}
- pCmprsor->aBuf[0][0] = 0;
+ *szComp = LZ4_compress_default(pCmprsor->pBuf, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf, pCmprsor->nBuf);
+ if (*szComp && *szComp < pCmprsor->nBuf) {
+ pCmprsor->aBuf[0][0] = 1;
+ *szComp += 1;
+ } else {
+ pCmprsor->aBuf[0][0] = 0;
+ memcpy(pCmprsor->aBuf[0] + 1, pCmprsor->pBuf, pCmprsor->nBuf);
+ *szComp = pCmprsor->nBuf + 1;
+ }
+
+ return code;
+}
+
+// Timestamp =====================================================
+static int32_t tCompTimestampStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+ int32_t code = 0;
+
+ pCmprsor->nBuf = 1;
+
+ code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
+ if (code) return code;
+
+ pCmprsor->pBuf[0] = 1;
+
+ return code;
+}
+
+static int32_t tCompTSSwitchToCopy(SCompressor *pCmprsor) {
+ int32_t code = 0;
+
+ if (pCmprsor->nVal == 0) goto _exit;
+
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(int64_t) * pCmprsor->nVal + 1))) {
+ return code;
+ }
+
+ int32_t n = 1;
+ int32_t nBuf = 1;
+ int64_t value;
+ int64_t delta;
+ for (int32_t iVal = 0; iVal < pCmprsor->nVal;) {
+ uint8_t aN[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)};
+
+ n++;
+
+ for (int32_t i = 0; i < 2; i++) {
+ uint64_t vZigzag = 0;
+ for (uint8_t j = 0; j < aN[i]; j++) {
+ vZigzag |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j));
+ n++;
+ }
+
+ int64_t delta_of_delta = ZIGZAG_DECODE(int64_t, vZigzag);
+ if (iVal) {
+ delta = delta_of_delta + delta;
+ value = delta + value;
+ } else {
+ delta = 0;
+ value = delta_of_delta;
+ }
+
+ memcpy(pCmprsor->aBuf[0] + nBuf, &value, sizeof(value));
+ nBuf += sizeof(int64_t);
+
+ iVal++;
+ if (iVal >= pCmprsor->nVal) break;
+ }
+ }
+
+ ASSERT(n == pCmprsor->nBuf && nBuf == sizeof(int64_t) * pCmprsor->nVal + 1);
+
+ uint8_t *pBuf = pCmprsor->pBuf;
+ pCmprsor->pBuf = pCmprsor->aBuf[0];
+ pCmprsor->aBuf[0] = pBuf;
+ pCmprsor->nBuf = nBuf;
+
+_exit:
+ pCmprsor->pBuf[0] = 0;
return code;
}
static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0;
int64_t ts = *(int64_t *)pData;
- ASSERT(pCmprsor->type == TSDB_DATA_TYPE_TIMESTAMP);
ASSERT(nData == 8);
- if (pCmprsor->aBuf[0][0] == 1) {
+ if (pCmprsor->pBuf[0] == 1) {
if (pCmprsor->nVal == 0) {
pCmprsor->ts_prev_val = ts;
pCmprsor->ts_prev_delta = -ts;
}
if (!I64_SAFE_ADD(ts, -pCmprsor->ts_prev_val)) {
- code = tCompSetCopyMode(pCmprsor);
+ code = tCompTSSwitchToCopy(pCmprsor);
if (code) return code;
goto _copy_cmpr;
}
int64_t delta = ts - pCmprsor->ts_prev_val;
if (!I64_SAFE_ADD(delta, -pCmprsor->ts_prev_delta)) {
- code = tCompSetCopyMode(pCmprsor);
+ code = tCompTSSwitchToCopy(pCmprsor);
if (code) return code;
goto _copy_cmpr;
}
@@ -1162,43 +1363,63 @@ static int32_t tCompTimestamp(SCompressor *pCmprsor, const void *pData, int32_t
pCmprsor->ts_prev_delta = delta;
if ((pCmprsor->nVal & 0x1) == 0) {
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17);
- if (code) return code;
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) {
+ return code;
}
- pCmprsor->ts_flag_p = pCmprsor->aBuf[0] + pCmprsor->nBuf[0];
- pCmprsor->nBuf[0]++;
+ pCmprsor->ts_flag_p = pCmprsor->pBuf + pCmprsor->nBuf;
+ pCmprsor->nBuf++;
pCmprsor->ts_flag_p[0] = 0;
while (vZigzag) {
- pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff);
- pCmprsor->nBuf[0]++;
+ pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff);
+ pCmprsor->nBuf++;
pCmprsor->ts_flag_p[0]++;
vZigzag >>= 8;
}
} else {
while (vZigzag) {
- pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (vZigzag & 0xff);
- pCmprsor->nBuf[0]++;
+ pCmprsor->pBuf[pCmprsor->nBuf] = (vZigzag & 0xff);
+ pCmprsor->nBuf++;
pCmprsor->ts_flag_p[0] += 0x10;
vZigzag >>= 8;
}
}
} else {
_copy_cmpr:
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(ts));
- if (code) return code;
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(ts)))) {
+ return code;
}
- memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], &ts, sizeof(ts));
- pCmprsor->nBuf[0] += sizeof(ts);
+ memcpy(pCmprsor->pBuf + pCmprsor->nBuf, &ts, sizeof(ts));
+ pCmprsor->nBuf += sizeof(ts);
}
pCmprsor->nVal++;
return code;
}
+static int32_t tCompTimestampEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
+ int32_t code = 0;
+
+ if (pCmprsor->nBuf >= sizeof(int64_t) * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 1) {
+ code = tCompTSSwitchToCopy(pCmprsor);
+ if (code) return code;
+ }
+
+ if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
+ code = tTwoStageComp(pCmprsor, nData);
+ if (code) return code;
+ *ppData = pCmprsor->aBuf[0];
+ } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
+ *ppData = pCmprsor->pBuf;
+ *nData = pCmprsor->nBuf;
+ } else {
+ ASSERT(0);
+ }
+
+ return code;
+}
+
// Integer =====================================================
#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL)
static const uint8_t BIT_PER_INTEGER[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60};
@@ -1207,94 +1428,163 @@ static const uint8_t BIT_TO_SELECTOR[] = {0, 2, 3, 4, 5, 6, 7, 8, 9, 10
13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15};
+static const int32_t NEXT_IDX[] = {
+ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
+ 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44,
+ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66,
+ 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88,
+ 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110,
+ 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132,
+ 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154,
+ 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176,
+ 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198,
+ 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220,
+ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 0};
+
+static int32_t tCompIntStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+ int32_t code = 0;
+
+ pCmprsor->i_prev = 0;
+ pCmprsor->i_selector = 0;
+ pCmprsor->i_start = 0;
+ pCmprsor->i_end = 0;
+ pCmprsor->i_nEle = 0;
+ pCmprsor->nBuf = 1;
+
+ code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
+ if (code) return code;
+
+ pCmprsor->pBuf[0] = 0;
+
+ return code;
+}
+
+static int32_t tCompIntSwitchToCopy(SCompressor *pCmprsor) {
+ int32_t code = 0;
+
+ if (pCmprsor->nVal == 0) goto _exit;
+
+ int32_t size = DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1;
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], size))) {
+ return code;
+ }
+
+ int32_t n = 1;
+ int32_t nBuf = 1;
+ int64_t vPrev = 0;
+ while (n < pCmprsor->nBuf) {
+ uint64_t b;
+ memcpy(&b, pCmprsor->pBuf + n, sizeof(b));
+ n += sizeof(b);
+
+ int32_t i_selector = (b & 0xf);
+ int32_t nEle = SELECTOR_TO_ELEMS[i_selector];
+ uint8_t bits = BIT_PER_INTEGER[i_selector];
+ uint64_t mask = (((uint64_t)1) << bits) - 1;
+ for (int32_t iEle = 0; iEle < nEle; iEle++) {
+ uint64_t vZigzag = (b >> (bits * iEle + 4)) & mask;
+ vPrev = ZIGZAG_DECODE(int64_t, vZigzag) + vPrev;
+
+ DATA_TYPE_INFO[pCmprsor->type].putI64(vPrev, pCmprsor->aBuf[0] + nBuf);
+ nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes;
+ }
+ }
+
+ while (pCmprsor->i_nEle) {
+ vPrev = ZIGZAG_DECODE(int64_t, pCmprsor->i_aZigzag[pCmprsor->i_start]) + vPrev;
+
+ memcpy(pCmprsor->aBuf[0] + nBuf, &vPrev, DATA_TYPE_INFO[pCmprsor->type].bytes);
+ nBuf += DATA_TYPE_INFO[pCmprsor->type].bytes;
+
+ pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start];
+ pCmprsor->i_nEle--;
+ }
+
+ ASSERT(n == pCmprsor->nBuf && nBuf == size);
+
+ uint8_t *pBuf = pCmprsor->pBuf;
+ pCmprsor->pBuf = pCmprsor->aBuf[0];
+ pCmprsor->aBuf[0] = pBuf;
+ pCmprsor->nBuf = size;
+
+_exit:
+ pCmprsor->pBuf[0] = 1;
+ return code;
+}
static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0;
ASSERT(nData == DATA_TYPE_INFO[pCmprsor->type].bytes);
- if (pCmprsor->aBuf[0][0] == 0) {
- int64_t val;
-
- switch (pCmprsor->type) {
- case TSDB_DATA_TYPE_TINYINT:
- val = *(int8_t *)pData;
- break;
- case TSDB_DATA_TYPE_SMALLINT:
- val = *(int16_t *)pData;
- break;
- case TSDB_DATA_TYPE_INT:
- val = *(int32_t *)pData;
- break;
- case TSDB_DATA_TYPE_BIGINT:
- val = *(int64_t *)pData;
- break;
- case TSDB_DATA_TYPE_UTINYINT:
- val = *(uint8_t *)pData;
- break;
- case TSDB_DATA_TYPE_USMALLINT:
- val = *(uint16_t *)pData;
- break;
- case TSDB_DATA_TYPE_UINT:
- val = *(uint32_t *)pData;
- break;
- case TSDB_DATA_TYPE_UBIGINT:
- val = *(int64_t *)pData;
- break;
- default:
- ASSERT(0);
- break;
- }
+ if (pCmprsor->pBuf[0] == 0) {
+ int64_t val = DATA_TYPE_INFO[pCmprsor->type].getI64(pData);
if (!I64_SAFE_ADD(val, -pCmprsor->i_prev)) {
- // TODO
+ code = tCompIntSwitchToCopy(pCmprsor);
+ if (code) return code;
goto _copy_cmpr;
}
int64_t diff = val - pCmprsor->i_prev;
uint64_t vZigzag = ZIGZAG_ENCODE(int64_t, diff);
if (vZigzag >= SIMPLE8B_MAX) {
- // TODO
+ code = tCompIntSwitchToCopy(pCmprsor);
+ if (code) return code;
goto _copy_cmpr;
}
int8_t nBit = (vZigzag) ? (64 - BUILDIN_CLZL(vZigzag)) : 0;
pCmprsor->i_prev = val;
- while (1) {
- int32_t nEle = (pCmprsor->i_end + 241 - pCmprsor->i_start) % 241;
-
- if (nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] && nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) {
+ for (;;) {
+ if (pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[pCmprsor->i_selector] &&
+ pCmprsor->i_nEle + 1 <= SELECTOR_TO_ELEMS[BIT_TO_SELECTOR[nBit]]) {
if (pCmprsor->i_selector < BIT_TO_SELECTOR[nBit]) {
pCmprsor->i_selector = BIT_TO_SELECTOR[nBit];
}
- pCmprsor->i_end = (pCmprsor->i_end + 1) % 241;
pCmprsor->i_aZigzag[pCmprsor->i_end] = vZigzag;
pCmprsor->i_aBitN[pCmprsor->i_end] = nBit;
+ pCmprsor->i_end = NEXT_IDX[pCmprsor->i_end];
+ pCmprsor->i_nEle++;
break;
} else {
- while (nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
- pCmprsor->i_selector++;
- }
- nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
+ if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
+ int32_t lidx = pCmprsor->i_selector + 1;
+ int32_t ridx = 15;
+ while (lidx <= ridx) {
+ pCmprsor->i_selector = (lidx + ridx) >> 1;
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + sizeof(uint64_t));
- if (code) return code;
+ if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
+ lidx = pCmprsor->i_selector + 1;
+ } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
+ ridx = pCmprsor->i_selector - 1;
+ } else {
+ break;
+ }
+ }
+
+ if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++;
+ }
+ int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
+
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) {
+ return code;
}
- uint64_t *bp = (uint64_t *)(pCmprsor->aBuf[0] + pCmprsor->nBuf[0]);
- pCmprsor->nBuf[0] += sizeof(uint64_t);
+ uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf);
+ pCmprsor->nBuf += sizeof(uint64_t);
bp[0] = pCmprsor->i_selector;
uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector];
for (int32_t iVal = 0; iVal < nEle; iVal++) {
- bp[0] |= ((pCmprsor->i_aZigzag[pCmprsor->i_start] & ((((uint64_t)1) << bits) - 1)) << (bits * iVal + 4));
- pCmprsor->i_start = (pCmprsor->i_start + 1) % 241;
+ bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4));
+ pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start];
+ pCmprsor->i_nEle--;
}
// reset and continue
pCmprsor->i_selector = 0;
- for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = (iVal + 1) % 241) {
+ for (int32_t iVal = pCmprsor->i_start; iVal < pCmprsor->i_end; iVal = NEXT_IDX[iVal]) {
if (pCmprsor->i_selector < BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]]) {
pCmprsor->i_selector = BIT_TO_SELECTOR[pCmprsor->i_aBitN[iVal]];
}
@@ -1303,18 +1593,146 @@ static int32_t tCompInt(SCompressor *pCmprsor, const void *pData, int32_t nData)
}
} else {
_copy_cmpr:
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
+ code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData);
if (code) return code;
- memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
- pCmprsor->nBuf[0] += nData;
+ memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData);
+ pCmprsor->nBuf += nData;
}
pCmprsor->nVal++;
return code;
}
+static int32_t tCompIntEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
+ int32_t code = 0;
+
+ for (; pCmprsor->i_nEle;) {
+ if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
+ int32_t lidx = pCmprsor->i_selector + 1;
+ int32_t ridx = 15;
+ while (lidx <= ridx) {
+ pCmprsor->i_selector = (lidx + ridx) >> 1;
+
+ if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
+ lidx = pCmprsor->i_selector + 1;
+ } else if (pCmprsor->i_nEle > SELECTOR_TO_ELEMS[pCmprsor->i_selector]) {
+ ridx = pCmprsor->i_selector - 1;
+ } else {
+ break;
+ }
+ }
+
+ if (pCmprsor->i_nEle < SELECTOR_TO_ELEMS[pCmprsor->i_selector]) pCmprsor->i_selector++;
+ }
+ int32_t nEle = SELECTOR_TO_ELEMS[pCmprsor->i_selector];
+
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + sizeof(uint64_t)))) {
+ return code;
+ }
+
+ uint64_t *bp = (uint64_t *)(pCmprsor->pBuf + pCmprsor->nBuf);
+ pCmprsor->nBuf += sizeof(uint64_t);
+ bp[0] = pCmprsor->i_selector;
+ uint8_t bits = BIT_PER_INTEGER[pCmprsor->i_selector];
+ for (int32_t iVal = 0; iVal < nEle; iVal++) {
+ bp[0] |= (pCmprsor->i_aZigzag[pCmprsor->i_start] << (bits * iVal + 4));
+ pCmprsor->i_start = NEXT_IDX[pCmprsor->i_start];
+ pCmprsor->i_nEle--;
+ }
+
+ pCmprsor->i_selector = 0;
+ }
+
+ if (pCmprsor->nBuf >= DATA_TYPE_INFO[pCmprsor->type].bytes * pCmprsor->nVal + 1 && pCmprsor->pBuf[0] == 0) {
+ code = tCompIntSwitchToCopy(pCmprsor);
+ if (code) return code;
+ }
+
+ if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
+ code = tTwoStageComp(pCmprsor, nData);
+ if (code) return code;
+ *ppData = pCmprsor->aBuf[0];
+ } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
+ *ppData = pCmprsor->pBuf;
+ *nData = pCmprsor->nBuf;
+ } else {
+ ASSERT(0);
+ }
+
+ return code;
+}
+
// Float =====================================================
+static int32_t tCompFloatStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+ int32_t code = 0;
+
+ pCmprsor->f_prev = 0;
+ pCmprsor->f_flag_p = NULL;
+
+ pCmprsor->nBuf = 1;
+
+ code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
+ if (code) return code;
+
+ pCmprsor->pBuf[0] = 0;
+
+ return code;
+}
+
+static int32_t tCompFloatSwitchToCopy(SCompressor *pCmprsor) {
+ int32_t code = 0;
+
+ if (pCmprsor->nVal == 0) goto _exit;
+
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(float) * pCmprsor->nVal + 1))) {
+ return code;
+ }
+
+ int32_t n = 1;
+ int32_t nBuf = 1;
+ union {
+ float f;
+ uint32_t u;
+ } val = {.u = 0};
+
+ for (int32_t iVal = 0; iVal < pCmprsor->nVal;) {
+ uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)};
+
+ n++;
+
+ for (int8_t i = 0; i < 2; i++) {
+ uint8_t flag = flags[i];
+
+ uint32_t diff = 0;
+ int8_t nBytes = (flag & 0x7) + 1;
+ for (int j = 0; j < nBytes; j++) {
+ diff |= (((uint32_t)pCmprsor->pBuf[n]) << (8 * j));
+ n++;
+ }
+
+ if (flag & 0x8) {
+ diff <<= (32 - nBytes * 8);
+ }
+
+ val.u ^= diff;
+
+ memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val));
+ nBuf += sizeof(val);
+
+ iVal++;
+ if (iVal >= pCmprsor->nVal) break;
+ }
+ }
+ uint8_t *pBuf = pCmprsor->pBuf;
+ pCmprsor->pBuf = pCmprsor->aBuf[0];
+ pCmprsor->aBuf[0] = pBuf;
+ pCmprsor->nBuf = nBuf;
+
+_exit:
+ pCmprsor->pBuf[0] = 1;
+ return code;
+}
static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0;
@@ -1347,13 +1765,12 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat
if (nBytes == 0) nBytes++;
if ((pCmprsor->nVal & 0x1) == 0) {
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 9);
- if (code) return code;
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 9))) {
+ return code;
}
- pCmprsor->f_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
- pCmprsor->nBuf[0]++;
+ pCmprsor->f_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf];
+ pCmprsor->nBuf++;
if (clz < ctz) {
pCmprsor->f_flag_p[0] = (0x08 | (nBytes - 1));
@@ -1368,8 +1785,8 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat
}
}
for (; nBytes; nBytes--) {
- pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
- pCmprsor->nBuf[0]++;
+ pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff);
+ pCmprsor->nBuf++;
diff >>= BITS_PER_BYTE;
}
pCmprsor->nVal++;
@@ -1377,7 +1794,98 @@ static int32_t tCompFloat(SCompressor *pCmprsor, const void *pData, int32_t nDat
return code;
}
+static int32_t tCompFloatEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
+ int32_t code = 0;
+
+ if (pCmprsor->nBuf >= sizeof(float) * pCmprsor->nVal + 1) {
+ code = tCompFloatSwitchToCopy(pCmprsor);
+ if (code) return code;
+ }
+
+ if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
+ code = tTwoStageComp(pCmprsor, nData);
+ if (code) return code;
+ *ppData = pCmprsor->aBuf[0];
+ } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
+ *ppData = pCmprsor->pBuf;
+ *nData = pCmprsor->nBuf;
+ } else {
+ ASSERT(0);
+ }
+
+ return code;
+}
+
// Double =====================================================
+static int32_t tCompDoubleStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+ int32_t code = 0;
+
+ pCmprsor->d_prev = 0;
+ pCmprsor->d_flag_p = NULL;
+
+ pCmprsor->nBuf = 1;
+
+ code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf);
+ if (code) return code;
+
+ pCmprsor->pBuf[0] = 0;
+
+ return code;
+}
+
+static int32_t tCompDoubleSwitchToCopy(SCompressor *pCmprsor) {
+ int32_t code = 0;
+
+ if (pCmprsor->nVal == 0) goto _exit;
+
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], sizeof(double) * pCmprsor->nVal + 1))) {
+ return code;
+ }
+
+ int32_t n = 1;
+ int32_t nBuf = 1;
+ union {
+ double f;
+ uint64_t u;
+ } val = {.u = 0};
+
+ for (int32_t iVal = 0; iVal < pCmprsor->nVal;) {
+ uint8_t flags[2] = {(pCmprsor->pBuf[n] & 0xf), (pCmprsor->pBuf[n] >> 4)};
+
+ n++;
+
+ for (int8_t i = 0; i < 2; i++) {
+ uint8_t flag = flags[i];
+
+ uint64_t diff = 0;
+ int8_t nBytes = (flag & 0x7) + 1;
+ for (int j = 0; j < nBytes; j++) {
+ diff |= (((uint64_t)pCmprsor->pBuf[n]) << (8 * j));
+ n++;
+ }
+
+ if (flag & 0x8) {
+ diff <<= (64 - nBytes * 8);
+ }
+
+ val.u ^= diff;
+
+ memcpy(pCmprsor->aBuf[0] + nBuf, &val.f, sizeof(val));
+ nBuf += sizeof(val);
+
+ iVal++;
+ if (iVal >= pCmprsor->nVal) break;
+ }
+ }
+ uint8_t *pBuf = pCmprsor->pBuf;
+ pCmprsor->pBuf = pCmprsor->aBuf[0];
+ pCmprsor->aBuf[0] = pBuf;
+ pCmprsor->nBuf = nBuf;
+
+_exit:
+ pCmprsor->pBuf[0] = 1;
+ return code;
+}
static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0;
@@ -1410,13 +1918,12 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa
if (nBytes == 0) nBytes++;
if ((pCmprsor->nVal & 0x1) == 0) {
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + 17);
- if (code) return code;
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + 17))) {
+ return code;
}
- pCmprsor->d_flag_p = &pCmprsor->aBuf[0][pCmprsor->nBuf[0]];
- pCmprsor->nBuf[0]++;
+ pCmprsor->d_flag_p = &pCmprsor->pBuf[pCmprsor->nBuf];
+ pCmprsor->nBuf++;
if (clz < ctz) {
pCmprsor->d_flag_p[0] = (0x08 | (nBytes - 1));
@@ -1431,8 +1938,8 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa
}
}
for (; nBytes; nBytes--) {
- pCmprsor->aBuf[0][pCmprsor->nBuf[0]] = (diff & 0xff);
- pCmprsor->nBuf[0]++;
+ pCmprsor->pBuf[pCmprsor->nBuf] = (diff & 0xff);
+ pCmprsor->nBuf++;
diff >>= BITS_PER_BYTE;
}
pCmprsor->nVal++;
@@ -1440,51 +1947,122 @@ static int32_t tCompDouble(SCompressor *pCmprsor, const void *pData, int32_t nDa
return code;
}
+static int32_t tCompDoubleEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
+ int32_t code = 0;
+
+ if (pCmprsor->nBuf >= sizeof(double) * pCmprsor->nVal + 1) {
+ code = tCompDoubleSwitchToCopy(pCmprsor);
+ if (code) return code;
+ }
+
+ if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
+ code = tTwoStageComp(pCmprsor, nData);
+ if (code) return code;
+ *ppData = pCmprsor->aBuf[0];
+ } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
+ *ppData = pCmprsor->pBuf;
+ *nData = pCmprsor->nBuf;
+ } else {
+ ASSERT(0);
+ }
+
+ return code;
+}
+
// Binary =====================================================
+static int32_t tCompBinaryStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+ pCmprsor->nBuf = 1;
+ return 0;
+}
+
static int32_t tCompBinary(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0;
if (nData) {
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0] + nData);
- if (code) return code;
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf + nData))) {
+ return code;
}
- memcpy(pCmprsor->aBuf[0] + pCmprsor->nBuf[0], pData, nData);
- pCmprsor->nBuf[0] += nData;
+ memcpy(pCmprsor->pBuf + pCmprsor->nBuf, pData, nData);
+ pCmprsor->nBuf += nData;
}
pCmprsor->nVal++;
return code;
}
+static int32_t tCompBinaryEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
+ int32_t code = 0;
+
+ if (pCmprsor->nBuf == 1) return code;
+
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf))) {
+ return code;
+ }
+
+ int32_t szComp =
+ LZ4_compress_default(pCmprsor->pBuf + 1, pCmprsor->aBuf[0] + 1, pCmprsor->nBuf - 1, pCmprsor->nBuf - 1);
+ if (szComp && szComp < pCmprsor->nBuf - 1) {
+ pCmprsor->aBuf[0][0] = 1;
+ *ppData = pCmprsor->aBuf[0];
+ *nData = szComp + 1;
+ } else {
+ pCmprsor->pBuf[0] = 0;
+ *ppData = pCmprsor->pBuf;
+ *nData = pCmprsor->nBuf;
+ }
+
+ return code;
+}
+
// Bool =====================================================
static const uint8_t BOOL_CMPR_TABLE[] = {0b01, 0b0100, 0b010000, 0b01000000};
+static int32_t tCompBoolStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
+ pCmprsor->nBuf = 0;
+ return 0;
+}
+
static int32_t tCompBool(SCompressor *pCmprsor, const void *pData, int32_t nData) {
int32_t code = 0;
bool vBool = *(int8_t *)pData;
- int32_t mod4 = pCmprsor->nVal & 3;
+ int32_t mod4 = (pCmprsor->nVal & 3);
if (mod4 == 0) {
- pCmprsor->nBuf[0]++;
+ pCmprsor->nBuf++;
- if (pCmprsor->autoAlloc) {
- code = tRealloc(&pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
- if (code) return code;
+ if (pCmprsor->autoAlloc && (code = tRealloc(&pCmprsor->pBuf, pCmprsor->nBuf))) {
+ return code;
}
- pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] = 0;
+ pCmprsor->pBuf[pCmprsor->nBuf - 1] = 0;
}
if (vBool) {
- pCmprsor->aBuf[0][pCmprsor->nBuf[0] - 1] |= BOOL_CMPR_TABLE[mod4];
+ pCmprsor->pBuf[pCmprsor->nBuf - 1] |= BOOL_CMPR_TABLE[mod4];
}
pCmprsor->nVal++;
return code;
}
+static int32_t tCompBoolEnd(SCompressor *pCmprsor, const uint8_t **ppData, int32_t *nData) {
+ int32_t code = 0;
+
+ if (pCmprsor->cmprAlg == TWO_STAGE_COMP) {
+ code = tTwoStageComp(pCmprsor, nData);
+ if (code) return code;
+ *ppData = pCmprsor->aBuf[0];
+ } else if (pCmprsor->cmprAlg == ONE_STAGE_COMP) {
+ *ppData = pCmprsor->pBuf;
+ *nData = pCmprsor->nBuf;
+ } else {
+ ASSERT(0);
+ }
+
+ return code;
+}
+
// SCompressor =====================================================
int32_t tCompressorCreate(SCompressor **ppCmprsor) {
int32_t code = 0;
@@ -1492,119 +2070,59 @@ int32_t tCompressorCreate(SCompressor **ppCmprsor) {
*ppCmprsor = (SCompressor *)taosMemoryCalloc(1, sizeof(SCompressor));
if ((*ppCmprsor) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
+ return code;
}
- code = tRealloc(&(*ppCmprsor)->aBuf[0], 1024);
- if (code) {
- taosMemoryFree(*ppCmprsor);
- *ppCmprsor = NULL;
- goto _exit;
- }
-
-_exit:
return code;
}
int32_t tCompressorDestroy(SCompressor *pCmprsor) {
int32_t code = 0;
- if (pCmprsor) {
- int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]);
- for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) {
- tFree(pCmprsor->aBuf[iBuf]);
- }
+ tFree(pCmprsor->pBuf);
- taosMemoryFree(pCmprsor);
+ int32_t nBuf = sizeof(pCmprsor->aBuf) / sizeof(pCmprsor->aBuf[0]);
+ for (int32_t iBuf = 0; iBuf < nBuf; iBuf++) {
+ tFree(pCmprsor->aBuf[iBuf]);
}
+ taosMemoryFree(pCmprsor);
+
return code;
}
-int32_t tCompressorReset(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg, int8_t autoAlloc) {
+int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
int32_t code = 0;
pCmprsor->type = type;
pCmprsor->cmprAlg = cmprAlg;
- pCmprsor->autoAlloc = autoAlloc;
+ pCmprsor->autoAlloc = 1;
pCmprsor->nVal = 0;
- switch (type) {
- case TSDB_DATA_TYPE_TIMESTAMP:
- pCmprsor->ts_prev_val = 0;
- pCmprsor->ts_prev_delta = 0;
- pCmprsor->ts_flag_p = NULL;
- pCmprsor->aBuf[0][0] = 1; // For timestamp, 1 means compressed, 0 otherwise
- pCmprsor->nBuf[0] = 1;
- break;
- case TSDB_DATA_TYPE_BOOL:
- pCmprsor->nBuf[0] = 0;
- break;
- case TSDB_DATA_TYPE_BINARY:
- pCmprsor->nBuf[0] = 0;
- break;
- case TSDB_DATA_TYPE_FLOAT:
- pCmprsor->f_prev = 0;
- pCmprsor->f_flag_p = NULL;
- pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
- pCmprsor->nBuf[0] = 1;
- break;
- case TSDB_DATA_TYPE_DOUBLE:
- pCmprsor->d_prev = 0;
- pCmprsor->d_flag_p = NULL;
- pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
- pCmprsor->nBuf[0] = 1;
- break;
- case TSDB_DATA_TYPE_TINYINT:
- case TSDB_DATA_TYPE_SMALLINT:
- case TSDB_DATA_TYPE_INT:
- case TSDB_DATA_TYPE_BIGINT:
- case TSDB_DATA_TYPE_UTINYINT:
- case TSDB_DATA_TYPE_USMALLINT:
- case TSDB_DATA_TYPE_UINT:
- case TSDB_DATA_TYPE_UBIGINT:
- pCmprsor->i_prev = 0;
- pCmprsor->i_selector = 0;
- pCmprsor->i_start = 0;
- pCmprsor->i_end = 0;
- pCmprsor->aBuf[0][0] = 0; // 0 means compressed, 1 otherwise (for backward compatibility)
- pCmprsor->nBuf[0] = 1;
- break;
- default:
- break;
+ if (DATA_TYPE_INFO[type].startFn) {
+ DATA_TYPE_INFO[type].startFn(pCmprsor, type, cmprAlg);
}
return code;
}
-int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData) {
+int32_t tCompressEnd(SCompressor *pCmprsor, const uint8_t **ppOut, int32_t *nOut, int32_t *nOrigin) {
int32_t code = 0;
- if (pCmprsor->nVal == 0) {
- *ppData = NULL;
- *nData = 0;
- return code;
+ *ppOut = NULL;
+ *nOut = 0;
+ if (nOrigin) {
+ if (DATA_TYPE_INFO[pCmprsor->type].isVarLen) {
+ *nOrigin = pCmprsor->nBuf - 1;
+ } else {
+ *nOrigin = pCmprsor->nVal * DATA_TYPE_INFO[pCmprsor->type].bytes;
+ }
}
- if (pCmprsor->cmprAlg == TWO_STAGE_COMP /*|| IS_VAR_DATA_TYPE(pCmprsor->type)*/) {
- code = tRealloc(&pCmprsor->aBuf[1], pCmprsor->nBuf[0] + 1);
- if (code) return code;
+ if (pCmprsor->nVal == 0) return code;
- int64_t ret = LZ4_compress_default(pCmprsor->aBuf[0], pCmprsor->aBuf[1] + 1, pCmprsor->nBuf[0], pCmprsor->nBuf[0]);
- if (ret) {
- pCmprsor->aBuf[1][0] = 0;
- pCmprsor->nBuf[1] = ret + 1;
- } else {
- pCmprsor->aBuf[1][0] = 1;
- memcpy(pCmprsor->aBuf[1] + 1, pCmprsor->aBuf[0], pCmprsor->nBuf[0]);
- pCmprsor->nBuf[1] = pCmprsor->nBuf[0] + 1;
- }
-
- *ppData = pCmprsor->aBuf[1];
- *nData = pCmprsor->nBuf[1];
- } else {
- *ppData = pCmprsor->aBuf[0];
- *nData = pCmprsor->nBuf[0];
+ if (DATA_TYPE_INFO[pCmprsor->type].endFn) {
+ return DATA_TYPE_INFO[pCmprsor->type].endFn(pCmprsor, ppOut, nOut);
}
return code;
@@ -1612,4 +2130,274 @@ int32_t tCompGen(SCompressor *pCmprsor, const uint8_t **ppData, int64_t *nData)
int32_t tCompress(SCompressor *pCmprsor, const void *pData, int64_t nData) {
return DATA_TYPE_INFO[pCmprsor->type].cmprFn(pCmprsor, pData, nData);
-}
\ No newline at end of file
+}
+
+/*************************************************************************
+ * REGULAR COMPRESSION
+ *************************************************************************/
+// Timestamp =====================================================
+int32_t tsCompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressTimestampImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressTimestampImp(pIn, nEle, pBuf);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+int32_t tsDecompressTimestamp(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
+ void *pBuf, int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressTimestampImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressTimestampImp(pBuf, nEle, pOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+// Float =====================================================
+int32_t tsCompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+#ifdef TD_TSZ
+ // lossy mode
+ if (lossyFloat) {
+ return tsCompressFloatLossyImp(pIn, nEle, pOut);
+ // lossless mode
+ } else {
+#endif
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressFloatImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressFloatImp(pIn, nEle, pBuf);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+#ifdef TD_TSZ
+ }
+#endif
+}
+
+int32_t tsDecompressFloat(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+#ifdef TD_TSZ
+ if (HEAD_ALGO(pIn[0]) == ALGO_SZ_LOSSY) {
+ // decompress lossy
+ return tsDecompressFloatLossyImp(pIn, nIn, nEle, pOut);
+ } else {
+#endif
+ // decompress lossless
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressFloatImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressFloatImp(pBuf, nEle, pOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+#ifdef TD_TSZ
+ }
+#endif
+}
+
+// Double =====================================================
+int32_t tsCompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+#ifdef TD_TSZ
+ if (lossyDouble) {
+ // lossy mode
+ return tsCompressDoubleLossyImp(pIn, nEle, pOut);
+ } else {
+#endif
+ // lossless mode
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressDoubleImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressDoubleImp(pIn, nEle, pBuf);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+#ifdef TD_TSZ
+ }
+#endif
+}
+
+int32_t tsDecompressDouble(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+#ifdef TD_TSZ
+ if (HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY) {
+ // decompress lossy
+ return tsDecompressDoubleLossyImp(pIn, nIn, nEle, pOut);
+ } else {
+#endif
+ // decompress lossless
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressDoubleImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressDoubleImp(pBuf, nEle, pOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+#ifdef TD_TSZ
+ }
+#endif
+}
+
+// Binary =====================================================
+int32_t tsCompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ return tsCompressStringImp(pIn, nIn, pOut, nOut);
+}
+
+int32_t tsDecompressString(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ return tsDecompressStringImp(pIn, nIn, pOut, nOut);
+}
+
+// Bool =====================================================
+int32_t tsCompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressBoolImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressBoolImp(pIn, nEle, pBuf);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+int32_t tsDecompressBool(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressBoolImp(pIn, nEle, pOut);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressBoolImp(pBuf, nEle, pOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+// Tinyint =====================================================
+int32_t tsCompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_TINYINT);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+int32_t tsDecompressTinyint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_TINYINT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_TINYINT);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+// Smallint =====================================================
+int32_t tsCompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_SMALLINT);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+int32_t tsDecompressSmallint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg,
+ void *pBuf, int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_SMALLINT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_SMALLINT);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+// Int =====================================================
+int32_t tsCompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_INT);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+int32_t tsDecompressInt(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_INT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_INT);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+// Bigint =====================================================
+int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsCompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ int32_t len = tsCompressINTImp(pIn, nEle, pBuf, TSDB_DATA_TYPE_BIGINT);
+ return tsCompressStringImp(pBuf, len, pOut, nOut);
+ } else {
+ assert(0);
+ return -1;
+ }
+}
+
+int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf,
+ int32_t nBuf) {
+ if (cmprAlg == ONE_STAGE_COMP) {
+ return tsDecompressINTImp(pIn, nEle, pOut, TSDB_DATA_TYPE_BIGINT);
+ } else if (cmprAlg == TWO_STAGE_COMP) {
+ if (tsDecompressStringImp(pIn, nIn, pBuf, nBuf) < 0) return -1;
+ return tsDecompressINTImp(pBuf, nEle, pOut, TSDB_DATA_TYPE_BIGINT);
+ } else {
+ assert(0);
+ return -1;
+ }
+}