From 5569c846245454d196c843656bcf91e79c7fffc7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Mon, 18 Apr 2022 15:06:13 +0800 Subject: [PATCH] bsma integrate --- include/common/trow.h | 21 +++-- include/common/ttypes.h | 4 +- source/common/src/trow.c | 55 ++++++++++-- source/common/src/ttypes.c | 102 +++++++++++------------ source/dnode/vnode/src/tsdb/tsdbCommit.c | 31 +++++-- 5 files changed, 140 insertions(+), 73 deletions(-) diff --git a/include/common/trow.h b/include/common/trow.h index 9a09bd3c8f..dbbdef84a0 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -48,9 +48,12 @@ extern "C" { #define TD_VTYPE_NONE 0x02U // none or unknown/undefined #define TD_VTYPE_MAX 0x03U // -#define TD_VTYPE_NORM_BYTE 0x0U -#define TD_VTYPE_NULL_BYTE 0x55U -#define TD_VTYPE_NONE_BYTE 0xAAU +#define TD_VTYPE_NORM_BYTE_I 0x0U +#define TD_VTYPE_NULL_BYTE_I 0xFFU + +#define TD_VTYPE_NORM_BYTE_II 0x0U +#define TD_VTYPE_NULL_BYTE_II 0x55U +#define TD_VTYPE_NONE_BYTE_II 0xAAU #define TD_ROWS_ALL_NORM 0x00U #define TD_ROWS_NULL_NORM 0x01U @@ -224,8 +227,10 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); -static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); -static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx); +static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, + int8_t bitmapMode); +static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode); +bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, int8_t bitmapMode); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, @@ -233,7 +238,7 @@ static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowVa static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, col_id_t colId); -int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); +int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); /** * @brief @@ -327,9 +332,9 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, return TSDB_CODE_SUCCESS; } -static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx) { +static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) { TDRowValT valType = 0; - tdGetBitmapValTypeII(pBitmap, idx, &valType); + tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode); if (tdValTypeIsNorm(valType)) { return true; } diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 1a09368bd3..7f2e59ea85 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -248,8 +248,8 @@ typedef struct tDataTypeDescriptor { int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - void (*statisFunc)(const void* pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, - int16_t *maxindex, int16_t *numofnull); + void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; extern tDataTypeDescriptor tDataTypes[15]; diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 7392b0bf8b..a1a2d236f9 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -16,10 +16,19 @@ #define _DEFAULT_SOURCE #include "trow.h" -const uint8_t tdVTypeByte[3] = { - TD_VTYPE_NORM_BYTE, // TD_VTYPE_NORM - TD_VTYPE_NONE_BYTE, // TD_VTYPE_NONE - TD_VTYPE_NULL_BYTE, // TD_VTYPE_NULL +const uint8_t tdVTypeByte[2][3] = {{ + // 2 bits + TD_VTYPE_NORM_BYTE_II, + TD_VTYPE_NONE_BYTE_II, + TD_VTYPE_NULL_BYTE_II, + }, + { + // 1 bit + TD_VTYPE_NORM_BYTE_I, + TD_VTYPE_NULL_BYTE_I, + TD_VTYPE_NULL_BYTE_I, // padding + } + }; // declaration @@ -266,21 +275,53 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit // } // } +/** + * @brief Set bitmap area by byte preferentially and then by bit. + * + * @param pBitmap + * @param nEle + * @param valType + * @param bitmapMode 0 for 2 bits, 1 for 1 bit + * @return int32_t + */ int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode) { TASSERT(valType < TD_VTYPE_MAX); - int16_t nBytes = nEle / TD_VTYPE_PARTS; + int32_t nBytes = (bitmapMode == 0 ? nEle / TD_VTYPE_PARTS : nEle / TD_VTYPE_PARTS_I); + uint8_t vTypeByte = tdVTypeByte[bitmapMode][valType]; for (int i = 0; i < nBytes; ++i) { - *(uint8_t *)pBitmap = tdVTypeByte[valType]; + *(uint8_t *)pBitmap = vTypeByte; pBitmap = POINTER_SHIFT(pBitmap, 1); } - int16_t nLeft = nEle - nBytes * TD_VTYPE_BITS; + int32_t nLeft = nEle - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); for (int j = 0; j < nLeft; ++j) { tdSetBitmapValType(pBitmap, j, valType, bitmapMode); } return TSDB_CODE_SUCCESS; } +bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) { + int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I); + uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM]; + for (int i = 0; i < nBytes; ++i) { + if (*((uint8_t *)pBitmap) != vTypeByte) { + return false; + } + pBitmap = POINTER_SHIFT(pBitmap, 1); + } + + int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I); + + for (int j = 0; j < nLeft; ++j) { + uint8_t vType; + tdGetBitmapValType(pBitmap, j, &vType, bitmapMode); + if (vType != TD_VTYPE_NORM) { + return false; + } + } + return true; +} + static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { if (IS_VAR_DATA_TYPE(pCol->type)) { pCol->dataOff[index] = pCol->len; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index d4371e11f7..cb01a17273 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -50,8 +50,8 @@ const int32_t TYPE_BYTES[15] = { } \ } while (0) -static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_bool(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -62,7 +62,7 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO for (int32_t i = 0; i < numOfRow; ++i) { // if (data[i] == TSDB_DATA_BOOL_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -71,8 +71,8 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO } } -static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -83,7 +83,7 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -92,8 +92,8 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR } } -static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint8_t *data = (uint8_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -106,7 +106,7 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -119,8 +119,8 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR *sum = _sum; } -static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int16_t *data = (int16_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -131,7 +131,7 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -140,8 +140,8 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf } } -static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint16_t *data = (uint16_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -154,7 +154,7 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -167,8 +167,8 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf *sum = _sum; } -static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int32_t *data = (int32_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -179,7 +179,7 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -188,8 +188,8 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf } } -static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint32_t *data = (uint32_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -202,7 +202,7 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -215,8 +215,8 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf *sum = _sum; } -static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_i64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *data = (int64_t *)pData; *min = INT64_MAX; *max = INT64_MIN; @@ -227,7 +227,7 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -236,8 +236,8 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf } } -static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_u64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint64_t *data = (uint64_t *)pData; uint64_t _min = UINT64_MAX; uint64_t _max = 0; @@ -250,7 +250,7 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf for (int32_t i = 0; i < numOfRow; ++i) { // if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -263,8 +263,8 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf *sum = _sum; } -static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_f(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { float *data = (float *)pData; float fmin = FLT_MAX; float fmax = -FLT_MAX; @@ -276,7 +276,7 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo for (int32_t i = 0; i < numOfRow; ++i) { // if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -300,8 +300,8 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo SET_DOUBLE_VAL(min, fmin); } -static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_d(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { double *data = (double *)pData; double dmin = DBL_MAX; double dmax = -DBL_MAX; @@ -313,7 +313,7 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo for (int32_t i = 0; i < numOfRow; ++i) { // if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; continue; } @@ -337,14 +337,14 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo SET_DOUBLE_PTR(min, &dmin); } -static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_bin(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char *data = pData; assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { // if (isNull(data, TSDB_DATA_TYPE_BINARY)) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; } @@ -358,14 +358,14 @@ static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOf *maxIndex = 0; } -static void getStatics_nchr(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, - int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { +static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, + int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char *data = pData; assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { // if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { - if (!tdIsBitmapValTypeNormII(pBitmap, i)) { + if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) { (*numOfNull) += 1; } @@ -405,19 +405,19 @@ tDataTypeDescriptor tDataTypes[15] = { }; char tTokenTypeSwitcher[13] = { - TSDB_DATA_TYPE_NULL, // no type - TSDB_DATA_TYPE_BINARY, // TK_ID - TSDB_DATA_TYPE_BOOL, // TK_BOOL - TSDB_DATA_TYPE_BIGINT, // TK_TINYINT - TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT - TSDB_DATA_TYPE_BIGINT, // TK_INTEGER - TSDB_DATA_TYPE_BIGINT, // TK_BIGINT - TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT - TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE - TSDB_DATA_TYPE_BINARY, // TK_STRING - TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP + TSDB_DATA_TYPE_NULL, // no type + TSDB_DATA_TYPE_BINARY, // TK_ID + TSDB_DATA_TYPE_BOOL, // TK_BOOL + TSDB_DATA_TYPE_BIGINT, // TK_TINYINT + TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT + TSDB_DATA_TYPE_BIGINT, // TK_INTEGER + TSDB_DATA_TYPE_BIGINT, // TK_BIGINT + TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT + TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE + TSDB_DATA_TYPE_BINARY, // TK_STRING + TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP TSDB_DATA_TYPE_VARCHAR, // TK_BINARY - TSDB_DATA_TYPE_NCHAR, // TK_NCHAR + TSDB_DATA_TYPE_NCHAR, // TK_NCHAR }; float floatMin = -FLT_MAX, floatMax = FLT_MAX; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index cb4937d758..09616a8969 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1201,6 +1201,23 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { } } +/** + * @brief Write SDataCols to data file. + * + * @param pRepo + * @param pTable + * @param pDFile + * @param pDFileAggr + * @param pDataCols The pDataCols would be generated from mem/imem directly with 2 bits bitmap or from tsdbRead + * interface with 1 bit bitmap. + * @param pBlock + * @param isLast + * @param isSuper + * @param ppBuf + * @param ppCBuf + * @param ppExBuf + * @return int + */ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { STsdbCfg *pCfg = REPO_CFG(pRepo); @@ -1244,14 +1261,15 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF pBlockCol->type = pDataCol->type; pAggrBlkCol->colId = pDataCol->colId; - if (tDataTypes[pDataCol->type].statisFunc) { + if (isSuper && pColumn->sma && tDataTypes[pDataCol->type].statisFunc) { #if 0 (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), &(pBlockCol->numOfNull)); #endif - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pBitmap, pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), - &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), + (*tDataTypes[pDataCol->type].statisFunc)(pDataCols->bitmapMode, pDataCol->pBitmap, pDataCol->pData, rowsToWrite, + &(pAggrBlkCol->min), &(pAggrBlkCol->max), &(pAggrBlkCol->sum), + &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), &(pAggrBlkCol->numOfNull)); if (pAggrBlkCol->numOfNull == 0) { @@ -1259,13 +1277,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF } else { TD_SET_COL_ROWS_MISC(pBlockCol); } + } else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) { + // check if all rows normal + TD_SET_COL_ROWS_NORM(pBlockCol); } else { TD_SET_COL_ROWS_MISC(pBlockCol); } ++nColsNotAllNull; - if (pColumn->sma) { + if (isSuper && pColumn->sma) { ++nColsOfBlockSma; } } @@ -1277,7 +1298,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF uint32_t toffset = 0; int32_t tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest); int32_t lsize = tsize; - uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); + uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsOfBlockSma, SBlockVerLatest); int32_t keyLen = 0; int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); int32_t sBitmaps = isSuper ? (int32_t)TD_BITMAP_BYTES_I(rowsToWrite) : nBitmaps;