xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #11592 from taosdata/feature/TD-11463-3.0 

feat: block-wise sma integrate
This commit is contained in:
Cary Xu 2022-04-18 15:36:41 +08:00 committed by GitHub
parent df8354affb ca93af74a1
commit 2aeb1c3b25
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 140 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
include/common/trow.h
View File

@ -48,9 +48,12 @@ extern "C" {
#define TD_VTYPE_NONE 0x02U // none or unknown/undefined #define TD_VTYPE_NONE 0x02U // none or unknown/undefined
#define TD_VTYPE_MAX 0x03U // #define TD_VTYPE_MAX 0x03U //
#define TD_VTYPE_NORM_BYTE 0x0U #define TD_VTYPE_NORM_BYTE_I 0x0U
#define TD_VTYPE_NULL_BYTE 0x55U #define TD_VTYPE_NULL_BYTE_I 0xFFU
#define TD_VTYPE_NONE_BYTE 0xAAU
#define TD_VTYPE_NORM_BYTE_II 0x0U
#define TD_VTYPE_NULL_BYTE_II 0x55U
#define TD_VTYPE_NONE_BYTE_II 0xAAU
#define TD_ROWS_ALL_NORM 0x00U #define TD_ROWS_ALL_NORM 0x00U
#define TD_ROWS_NULL_NORM 0x01U #define TD_ROWS_NULL_NORM 0x01U
@ -224,8 +227,10 @@ static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TD
int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode);
static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType,
static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx); int8_t bitmapMode);
static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode);
bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode);
int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
int8_t bitmapMode); int8_t bitmapMode);
static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
@ -233,7 +238,7 @@ static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowVa
static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, static FORCE_INLINE int32_t tdAppendColValToKvRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset, bool isCopyVarData, int8_t colType, int16_t colIdx, int32_t offset,
col_id_t colId); col_id_t colId);
int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols); int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols);
/** /**
* @brief * @brief
@ -327,9 +332,9 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx) { static FORCE_INLINE bool tdIsBitmapValTypeNorm(const void *pBitmap, int16_t idx, int8_t bitmapMode) {
TDRowValT valType = 0; TDRowValT valType = 0;
tdGetBitmapValTypeII(pBitmap, idx, &valType); tdGetBitmapValType(pBitmap, idx, &valType, bitmapMode);
if (tdValTypeIsNorm(valType)) { if (tdValTypeIsNorm(valType)) {
return true; return true;
} }

4
include/common/ttypes.h
View File

@ -248,8 +248,8 @@ typedef struct tDataTypeDescriptor {
int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output,
int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
void (*statisFunc)(const void* pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, void (*statisFunc)(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numofrow, int64_t *min,
int16_t *maxindex, int16_t *numofnull); int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull);
} tDataTypeDescriptor; } tDataTypeDescriptor;
extern tDataTypeDescriptor tDataTypes[15]; extern tDataTypeDescriptor tDataTypes[15];

55
source/common/src/trow.c
View File

@ -16,10 +16,19 @@
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "trow.h" #include "trow.h"
const uint8_t tdVTypeByte[3] = { const uint8_t tdVTypeByte[2][3] = {{
TD_VTYPE_NORM_BYTE, // TD_VTYPE_NORM // 2 bits
TD_VTYPE_NONE_BYTE, // TD_VTYPE_NONE TD_VTYPE_NORM_BYTE_II,
TD_VTYPE_NULL_BYTE, // TD_VTYPE_NULL TD_VTYPE_NONE_BYTE_II,
TD_VTYPE_NULL_BYTE_II,
},
{
// 1 bit
TD_VTYPE_NORM_BYTE_I,
TD_VTYPE_NULL_BYTE_I,
TD_VTYPE_NULL_BYTE_I, // padding
}
}; };
// declaration // declaration
@ -266,21 +275,53 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBit
// } // }
// } // }
/**
* @brief Set bitmap area by byte preferentially and then by bit.
*
* @param pBitmap
* @param nEle
* @param valType
* @param bitmapMode 0 for 2 bits, 1 for 1 bit
* @return int32_t
*/
int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode) { int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode) {
TASSERT(valType < TD_VTYPE_MAX); TASSERT(valType < TD_VTYPE_MAX);
int16_t nBytes = nEle / TD_VTYPE_PARTS; int32_t nBytes = (bitmapMode == 0 ? nEle / TD_VTYPE_PARTS : nEle / TD_VTYPE_PARTS_I);
uint8_t vTypeByte = tdVTypeByte[bitmapMode][valType];
for (int i = 0; i < nBytes; ++i) { for (int i = 0; i < nBytes; ++i) {
*(uint8_t *)pBitmap = tdVTypeByte[valType]; *(uint8_t *)pBitmap = vTypeByte;
pBitmap = POINTER_SHIFT(pBitmap, 1); pBitmap = POINTER_SHIFT(pBitmap, 1);
} }
int16_t nLeft = nEle - nBytes * TD_VTYPE_BITS;
int32_t nLeft = nEle - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
for (int j = 0; j < nLeft; ++j) { for (int j = 0; j < nLeft; ++j) {
tdSetBitmapValType(pBitmap, j, valType, bitmapMode); tdSetBitmapValType(pBitmap, j, valType, bitmapMode);
} }
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
bool tdIsBitmapBlkNorm(const void *pBitmap, int32_t numOfBits, int8_t bitmapMode) {
int32_t nBytes = (bitmapMode == 0 ? numOfBits / TD_VTYPE_PARTS : numOfBits / TD_VTYPE_PARTS_I);
uint8_t vTypeByte = tdVTypeByte[bitmapMode][TD_VTYPE_NORM];
for (int i = 0; i < nBytes; ++i) {
if (*((uint8_t *)pBitmap) != vTypeByte) {
return false;
}
pBitmap = POINTER_SHIFT(pBitmap, 1);
}
int32_t nLeft = numOfBits - nBytes * (bitmapMode == 0 ? TD_VTYPE_BITS : TD_VTYPE_BITS_I);
for (int j = 0; j < nLeft; ++j) {
uint8_t vType;
tdGetBitmapValType(pBitmap, j, &vType, bitmapMode);
if (vType != TD_VTYPE_NORM) {
return false;
}
}
return true;
}
static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) { static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap, int8_t bitmapMode) {
if (IS_VAR_DATA_TYPE(pCol->type)) { if (IS_VAR_DATA_TYPE(pCol->type)) {
pCol->dataOff[index] = pCol->len; pCol->dataOff[index] = pCol->len;

102
source/common/src/ttypes.c
View File

@ -50,8 +50,8 @@ const int32_t TYPE_BYTES[15] = {
} \ } \
} while (0) } while (0)
static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_bool(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
int8_t *data = (int8_t *)pData; int8_t *data = (int8_t *)pData;
*min = INT64_MAX; *min = INT64_MAX;
*max = INT64_MIN; *max = INT64_MIN;
@ -62,7 +62,7 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (data[i] == TSDB_DATA_BOOL_NULL) { // if (data[i] == TSDB_DATA_BOOL_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -71,8 +71,8 @@ static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numO
} }
} }
static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_i8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
int8_t *data = (int8_t *)pData; int8_t *data = (int8_t *)pData;
*min = INT64_MAX; *min = INT64_MAX;
*max = INT64_MIN; *max = INT64_MIN;
@ -83,7 +83,7 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { // if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -92,8 +92,8 @@ static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfR
} }
} }
static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_u8(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
uint8_t *data = (uint8_t *)pData; uint8_t *data = (uint8_t *)pData;
uint64_t _min = UINT64_MAX; uint64_t _min = UINT64_MAX;
uint64_t _max = 0; uint64_t _max = 0;
@ -106,7 +106,7 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { // if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -119,8 +119,8 @@ static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfR
*sum = _sum; *sum = _sum;
} }
static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_i16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
int16_t *data = (int16_t *)pData; int16_t *data = (int16_t *)pData;
*min = INT64_MAX; *min = INT64_MAX;
*max = INT64_MIN; *max = INT64_MIN;
@ -131,7 +131,7 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { // if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -140,8 +140,8 @@ static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOf
} }
} }
static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_u16(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
uint16_t *data = (uint16_t *)pData; uint16_t *data = (uint16_t *)pData;
uint64_t _min = UINT64_MAX; uint64_t _min = UINT64_MAX;
uint64_t _max = 0; uint64_t _max = 0;
@ -154,7 +154,7 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { // if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -167,8 +167,8 @@ static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOf
*sum = _sum; *sum = _sum;
} }
static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_i32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
int32_t *data = (int32_t *)pData; int32_t *data = (int32_t *)pData;
*min = INT64_MAX; *min = INT64_MAX;
*max = INT64_MIN; *max = INT64_MIN;
@ -179,7 +179,7 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { // if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -188,8 +188,8 @@ static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOf
} }
} }
static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_u32(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
uint32_t *data = (uint32_t *)pData; uint32_t *data = (uint32_t *)pData;
uint64_t _min = UINT64_MAX; uint64_t _min = UINT64_MAX;
uint64_t _max = 0; uint64_t _max = 0;
@ -202,7 +202,7 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { // if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -215,8 +215,8 @@ static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOf
*sum = _sum; *sum = _sum;
} }
static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_i64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
int64_t *data = (int64_t *)pData; int64_t *data = (int64_t *)pData;
*min = INT64_MAX; *min = INT64_MAX;
*max = INT64_MIN; *max = INT64_MIN;
@ -227,7 +227,7 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { // if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -236,8 +236,8 @@ static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOf
} }
} }
static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_u64(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
uint64_t *data = (uint64_t *)pData; uint64_t *data = (uint64_t *)pData;
uint64_t _min = UINT64_MAX; uint64_t _min = UINT64_MAX;
uint64_t _max = 0; uint64_t _max = 0;
@ -250,7 +250,7 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { // if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -263,8 +263,8 @@ static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOf
*sum = _sum; *sum = _sum;
} }
static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_f(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
float *data = (float *)pData; float *data = (float *)pData;
float fmin = FLT_MAX; float fmin = FLT_MAX;
float fmax = -FLT_MAX; float fmax = -FLT_MAX;
@ -276,7 +276,7 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) { // if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -300,8 +300,8 @@ static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRo
SET_DOUBLE_VAL(min, fmin); SET_DOUBLE_VAL(min, fmin);
} }
static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_d(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
double *data = (double *)pData; double *data = (double *)pData;
double dmin = DBL_MAX; double dmin = DBL_MAX;
double dmax = -DBL_MAX; double dmax = -DBL_MAX;
@ -313,7 +313,7 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { // if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
continue; continue;
} }
@ -337,14 +337,14 @@ static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRo
SET_DOUBLE_PTR(min, &dmin); SET_DOUBLE_PTR(min, &dmin);
} }
static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_bin(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
const char *data = pData; const char *data = pData;
assert(numOfRow <= INT16_MAX); assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (isNull(data, TSDB_DATA_TYPE_BINARY)) { // if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
} }
@ -358,14 +358,14 @@ static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOf
*maxIndex = 0; *maxIndex = 0;
} }
static void getStatics_nchr(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, static void getStatics_nchr(int8_t bitmapMode, const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min,
int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
const char *data = pData; const char *data = pData;
assert(numOfRow <= INT16_MAX); assert(numOfRow <= INT16_MAX);
for (int32_t i = 0; i < numOfRow; ++i) { for (int32_t i = 0; i < numOfRow; ++i) {
// if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { // if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
if (!tdIsBitmapValTypeNormII(pBitmap, i)) { if (!tdIsBitmapValTypeNorm(pBitmap, i, bitmapMode)) {
(*numOfNull) += 1; (*numOfNull) += 1;
} }
@ -405,19 +405,19 @@ tDataTypeDescriptor tDataTypes[15] = {
}; };
char tTokenTypeSwitcher[13] = { char tTokenTypeSwitcher[13] = {
TSDB_DATA_TYPE_NULL, // no type TSDB_DATA_TYPE_NULL, // no type
TSDB_DATA_TYPE_BINARY, // TK_ID TSDB_DATA_TYPE_BINARY, // TK_ID
TSDB_DATA_TYPE_BOOL, // TK_BOOL TSDB_DATA_TYPE_BOOL, // TK_BOOL
TSDB_DATA_TYPE_BIGINT, // TK_TINYINT TSDB_DATA_TYPE_BIGINT, // TK_TINYINT
TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT
TSDB_DATA_TYPE_BIGINT, // TK_INTEGER TSDB_DATA_TYPE_BIGINT, // TK_INTEGER
TSDB_DATA_TYPE_BIGINT, // TK_BIGINT TSDB_DATA_TYPE_BIGINT, // TK_BIGINT
TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT
TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE
TSDB_DATA_TYPE_BINARY, // TK_STRING TSDB_DATA_TYPE_BINARY, // TK_STRING
TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP
TSDB_DATA_TYPE_VARCHAR, // TK_BINARY TSDB_DATA_TYPE_VARCHAR, // TK_BINARY
TSDB_DATA_TYPE_NCHAR, // TK_NCHAR TSDB_DATA_TYPE_NCHAR, // TK_NCHAR
}; };
float floatMin = -FLT_MAX, floatMax = FLT_MAX; float floatMin = -FLT_MAX, floatMax = FLT_MAX;

31
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -1201,6 +1201,23 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
} }
} }
/**
* @brief Write SDataCols to data file.
*
* @param pRepo
* @param pTable
* @param pDFile
* @param pDFileAggr
* @param pDataCols The pDataCols would be generated from mem/imem directly with 2 bits bitmap or from tsdbRead
* interface with 1 bit bitmap.
* @param pBlock
* @param isLast
* @param isSuper
* @param ppBuf
* @param ppCBuf
* @param ppExBuf
* @return int
*/
int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols, int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDFileAggr, SDataCols *pDataCols,
SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) { SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf, void **ppExBuf) {
STsdbCfg *pCfg = REPO_CFG(pRepo); STsdbCfg *pCfg = REPO_CFG(pRepo);
@ -1244,14 +1261,15 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
pBlockCol->type = pDataCol->type; pBlockCol->type = pDataCol->type;
pAggrBlkCol->colId = pDataCol->colId; pAggrBlkCol->colId = pDataCol->colId;
if (tDataTypes[pDataCol->type].statisFunc) { if (isSuper && pColumn->sma && tDataTypes[pDataCol->type].statisFunc) {
#if 0 #if 0
(*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max), (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max),
&(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex),
&(pBlockCol->numOfNull)); &(pBlockCol->numOfNull));
#endif #endif
(*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pBitmap, pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), (*tDataTypes[pDataCol->type].statisFunc)(pDataCols->bitmapMode, pDataCol->pBitmap, pDataCol->pData, rowsToWrite,
&(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), &(pAggrBlkCol->min), &(pAggrBlkCol->max), &(pAggrBlkCol->sum),
&(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex),
&(pAggrBlkCol->numOfNull)); &(pAggrBlkCol->numOfNull));
if (pAggrBlkCol->numOfNull == 0) { if (pAggrBlkCol->numOfNull == 0) {
@ -1259,13 +1277,16 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
} else { } else {
TD_SET_COL_ROWS_MISC(pBlockCol); TD_SET_COL_ROWS_MISC(pBlockCol);
} }
} else if (tdIsBitmapBlkNorm(pDataCol->pBitmap, rowsToWrite, pDataCols->bitmapMode)) {
// check if all rows normal
TD_SET_COL_ROWS_NORM(pBlockCol);
} else { } else {
TD_SET_COL_ROWS_MISC(pBlockCol); TD_SET_COL_ROWS_MISC(pBlockCol);
} }
++nColsNotAllNull; ++nColsNotAllNull;
if (pColumn->sma) { if (isSuper && pColumn->sma) {
++nColsOfBlockSma; ++nColsOfBlockSma;
} }
} }
@ -1277,7 +1298,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
uint32_t toffset = 0; uint32_t toffset = 0;
int32_t tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest); int32_t tsize = (int32_t)tsdbBlockStatisSize(nColsNotAllNull, SBlockVerLatest);
int32_t lsize = tsize; int32_t lsize = tsize;
uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsNotAllNull, SBlockVerLatest); uint32_t tsizeAggr = (uint32_t)tsdbBlockAggrSize(nColsOfBlockSma, SBlockVerLatest);
int32_t keyLen = 0; int32_t keyLen = 0;
int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite); int32_t nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite);
int32_t sBitmaps = isSuper ? (int32_t)TD_BITMAP_BYTES_I(rowsToWrite) : nBitmaps; int32_t sBitmaps = isSuper ? (int32_t)TD_BITMAP_BYTES_I(rowsToWrite) : nBitmaps;