xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add more compress alg

This commit is contained in:
yihaoDeng 2024-03-20 08:51:40 +00:00
parent bf808a0bf6
commit 672d189d6f
4 changed files with 27 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
include/common/tcommon.h
View File

@ -15,8 +15,7 @@
#ifndef _TD_COMMON_DEF_H_
#define _TD_COMMON_DEF_H_
#include "taosdef.h"
// #include "taosdef.h"
#include "tarray.h"
#include "tmsg.h"
#include "tvariant.h"
@ -241,9 +240,9 @@ typedef struct SColumnInfoData {
char* nullbitmap; // bitmap, one bit for each item in the list
SVarColAttr varmeta;
};
SColumnInfo info; // column info
bool hasNull; // if current column data has null value.
bool reassigned; // if current column data is reassigned.
SColumnInfo info; // column info
bool hasNull; // if current column data has null value.
bool reassigned; // if current column data is reassigned.
} SColumnInfoData;
typedef struct SQueryTableDataCond {
@ -257,13 +256,13 @@ typedef struct SQueryTableDataCond {
STimeWindow twindows;
int64_t startVersion;
int64_t endVersion;
bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock
bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock
} SQueryTableDataCond;
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
void colDataDestroy(SColumnInfoData* pColData);
void colDataDestroy(SColumnInfoData* pColData);
//======================================================================================================================
// the following structure shared by parser and executor
@ -349,8 +348,8 @@ typedef struct {
float f;
};
size_t length;
bool keyEscaped;
bool valueEscaped;
bool keyEscaped;
bool valueEscaped;
} SSmlKv;
#define QUERY_ASC_FORWARD_STEP 1
@ -372,7 +371,6 @@ typedef struct SNonSortExecInfo {
int32_t blkNums;
} SNonSortExecInfo;
typedef struct STUidTagInfo {
char* name;
uint64_t uid;
@ -394,7 +392,7 @@ typedef struct STUidTagInfo {
#define UD_GROUPID_COLUMN_INDEX 1
#define UD_TAG_COLUMN_INDEX 2
int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime);
int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime);
#ifdef __cplusplus
}

22
include/common/tmsg.h
View File

@ -548,22 +548,6 @@ struct SSchemaExt {
uint32_t compress;
};
// compress flag
// |----l1 compAlg----|-----l2 compAlg---|---level--|
// |------8bit--------|------16bit-------|---8bit---|
#define COMPRESS_L1_TYPE_U32(type) (((type) >> 24) & 0xFF)
#define COMPRESS_L2_TYPE_U32(type) (((type) >> 8) & 0xFFFF)
#define COMPRESS_L2_TYPE_LEVEL_U32(type) ((type)&0xFF)
// compress flag
// |----l2lel--|----l2Alg---|---l1Alg--|
// |----2bit---|----3bit----|---3bit---|
#define COMPRESS_L1_TYPE_U8(type) ((type)&0x07)
#define COMPRESS_L2_TYPE_U8(type) (((type) >> 3) & 0x07)
#define COMPRESS_L2_TYPE_LEVEL_U8(type) (((type) >> 6) & 0x03)
//
@ -675,7 +659,7 @@ static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper
pDstWrapper->version = pSrcWrapper->version;
int32_t size = sizeof(SColCmpr) * pDstWrapper->nCols;
pDstWrapper->pColCmpr = taosMemoryCalloc(1, size);
pDstWrapper->pColCmpr = (SColCmpr*)taosMemoryCalloc(1, size);
memcpy(pDstWrapper->pColCmpr, pSrcWrapper->pColCmpr, size);
return pDstWrapper;
@ -683,14 +667,14 @@ static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper
static FORCE_INLINE void tInitDefaultSColCmprWrapperByCols(SColCmprWrapper* pCmpr, int32_t nCols) {
assert(!pCmpr->pColCmpr);
pCmpr->pColCmpr = taosMemoryCalloc(nCols, sizeof(SColCmpr));
pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(nCols, sizeof(SColCmpr));
pCmpr->nCols = nCols;
}
static FORCE_INLINE void tInitDefaultSColCmprWrapper(SColCmprWrapper* pCmpr, SSchemaWrapper* pSchema) {
pCmpr->nCols = pSchema->nCols;
assert(!pCmpr->pColCmpr);
pCmpr->pColCmpr = taosMemoryCalloc(pCmpr->nCols, sizeof(SColCmpr));
pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(pCmpr->nCols, sizeof(SColCmpr));
for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr* pColCmpr = &pCmpr->pColCmpr[i];
SSchema* pColSchema = &pSchema->pSchema[i];

14
include/util/tcompression.h
View File

@ -24,6 +24,20 @@
extern "C" {
#endif
// start compress flag
// |----l1 compAlg----|-----l2 compAlg---|---level--|
// |------8bit--------|------16bit-------|---8bit---|
#define COMPRESS_L1_TYPE_U32(type) (((type) >> 24) & 0xFF)
#define COMPRESS_L2_TYPE_U32(type) (((type) >> 8) & 0xFFFF)
#define COMPRESS_L2_TYPE_LEVEL_U32(type) ((type)&0xFF)
// compress flag
// |----l2lel--|----l2Alg---|---l1Alg--|
// |----2bit---|----3bit----|---3bit---|
#define COMPRESS_L1_TYPE_U8(type) ((type)&0x07)
#define COMPRESS_L2_TYPE_U8(type) (((type) >> 3) & 0x07)
#define COMPRESS_L2_TYPE_LEVEL_U8(type) (((type) >> 6) & 0x03)
// end compress flag
#define COMP_OVERFLOW_BYTES 2
#define BITS_PER_BYTE 8
// Masks

2
source/util/src/tcompression.c
View File

@ -52,7 +52,7 @@
#include "lz4.h"
#include "tcompare.h"
#include "tlog.h"
#include "tmsg.h"
// #include "tmsg.h"
#include "zlib.h"
#include "zstd.h"