xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

add more compress alg

This commit is contained in:
yihaoDeng 2024-03-20 08:51:40 +00:00
parent bf808a0bf6
commit 672d189d6f
4 changed files with 27 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
include/common/tcommon.h
View File

@ -15,8 +15,7 @@
#ifndef _TD_COMMON_DEF_H_ #ifndef _TD_COMMON_DEF_H_
#define _TD_COMMON_DEF_H_ #define _TD_COMMON_DEF_H_
// #include "taosdef.h"
#include "taosdef.h"
#include "tarray.h" #include "tarray.h"
#include "tmsg.h" #include "tmsg.h"
#include "tvariant.h" #include "tvariant.h"
@ -241,9 +240,9 @@ typedef struct SColumnInfoData {
char* nullbitmap; // bitmap, one bit for each item in the list char* nullbitmap; // bitmap, one bit for each item in the list
SVarColAttr varmeta; SVarColAttr varmeta;
}; };
SColumnInfo info; // column info SColumnInfo info; // column info
bool hasNull; // if current column data has null value. bool hasNull; // if current column data has null value.
bool reassigned; // if current column data is reassigned. bool reassigned; // if current column data is reassigned.
} SColumnInfoData; } SColumnInfoData;
typedef struct SQueryTableDataCond { typedef struct SQueryTableDataCond {
@ -257,13 +256,13 @@ typedef struct SQueryTableDataCond {
STimeWindow twindows; STimeWindow twindows;
int64_t startVersion; int64_t startVersion;
int64_t endVersion; int64_t endVersion;
bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock bool notLoadData; // response the actual data, not only the rows in the attribute of info.row of ssdatablock
} SQueryTableDataCond; } SQueryTableDataCond;
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock);
void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock); void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock);
void colDataDestroy(SColumnInfoData* pColData); void colDataDestroy(SColumnInfoData* pColData);
//====================================================================================================================== //======================================================================================================================
// the following structure shared by parser and executor // the following structure shared by parser and executor
@ -349,8 +348,8 @@ typedef struct {
float f; float f;
}; };
size_t length; size_t length;
bool keyEscaped; bool keyEscaped;
bool valueEscaped; bool valueEscaped;
} SSmlKv; } SSmlKv;
#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_ASC_FORWARD_STEP 1
@ -372,7 +371,6 @@ typedef struct SNonSortExecInfo {
int32_t blkNums; int32_t blkNums;
} SNonSortExecInfo; } SNonSortExecInfo;
typedef struct STUidTagInfo { typedef struct STUidTagInfo {
char* name; char* name;
uint64_t uid; uint64_t uid;
@ -394,7 +392,7 @@ typedef struct STUidTagInfo {
#define UD_GROUPID_COLUMN_INDEX 1 #define UD_GROUPID_COLUMN_INDEX 1
#define UD_TAG_COLUMN_INDEX 2 #define UD_TAG_COLUMN_INDEX 2
int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime); int32_t taosGenCrashJsonMsg(int signum, char** pMsg, int64_t clusterId, int64_t startTime);
#ifdef __cplusplus #ifdef __cplusplus
} }

22
include/common/tmsg.h
View File

@ -548,22 +548,6 @@ struct SSchemaExt {
uint32_t compress; uint32_t compress;
}; };
// compress flag
// |----l1 compAlg----|-----l2 compAlg---|---level--|
// |------8bit--------|------16bit-------|---8bit---|
#define COMPRESS_L1_TYPE_U32(type) (((type) >> 24) & 0xFF)
#define COMPRESS_L2_TYPE_U32(type) (((type) >> 8) & 0xFFFF)
#define COMPRESS_L2_TYPE_LEVEL_U32(type) ((type)&0xFF)
// compress flag
// |----l2lel--|----l2Alg---|---l1Alg--|
// |----2bit---|----3bit----|---3bit---|
#define COMPRESS_L1_TYPE_U8(type) ((type)&0x07)
#define COMPRESS_L2_TYPE_U8(type) (((type) >> 3) & 0x07)
#define COMPRESS_L2_TYPE_LEVEL_U8(type) (((type) >> 6) & 0x03)
// //
@ -675,7 +659,7 @@ static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper
pDstWrapper->version = pSrcWrapper->version; pDstWrapper->version = pSrcWrapper->version;
int32_t size = sizeof(SColCmpr) * pDstWrapper->nCols; int32_t size = sizeof(SColCmpr) * pDstWrapper->nCols;
pDstWrapper->pColCmpr = taosMemoryCalloc(1, size); pDstWrapper->pColCmpr = (SColCmpr*)taosMemoryCalloc(1, size);
memcpy(pDstWrapper->pColCmpr, pSrcWrapper->pColCmpr, size); memcpy(pDstWrapper->pColCmpr, pSrcWrapper->pColCmpr, size);
return pDstWrapper; return pDstWrapper;
@ -683,14 +667,14 @@ static FORCE_INLINE SColCmprWrapper* tCloneSColCmprWrapper(const SColCmprWrapper
static FORCE_INLINE void tInitDefaultSColCmprWrapperByCols(SColCmprWrapper* pCmpr, int32_t nCols) { static FORCE_INLINE void tInitDefaultSColCmprWrapperByCols(SColCmprWrapper* pCmpr, int32_t nCols) {
assert(!pCmpr->pColCmpr); assert(!pCmpr->pColCmpr);
pCmpr->pColCmpr = taosMemoryCalloc(nCols, sizeof(SColCmpr)); pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(nCols, sizeof(SColCmpr));
pCmpr->nCols = nCols; pCmpr->nCols = nCols;
} }
static FORCE_INLINE void tInitDefaultSColCmprWrapper(SColCmprWrapper* pCmpr, SSchemaWrapper* pSchema) { static FORCE_INLINE void tInitDefaultSColCmprWrapper(SColCmprWrapper* pCmpr, SSchemaWrapper* pSchema) {
pCmpr->nCols = pSchema->nCols; pCmpr->nCols = pSchema->nCols;
assert(!pCmpr->pColCmpr); assert(!pCmpr->pColCmpr);
pCmpr->pColCmpr = taosMemoryCalloc(pCmpr->nCols, sizeof(SColCmpr)); pCmpr->pColCmpr = (SColCmpr*)taosMemoryCalloc(pCmpr->nCols, sizeof(SColCmpr));
for (int32_t i = 0; i < pCmpr->nCols; i++) { for (int32_t i = 0; i < pCmpr->nCols; i++) {
SColCmpr* pColCmpr = &pCmpr->pColCmpr[i]; SColCmpr* pColCmpr = &pCmpr->pColCmpr[i];
SSchema* pColSchema = &pSchema->pSchema[i]; SSchema* pColSchema = &pSchema->pSchema[i];

14
include/util/tcompression.h
View File

@ -24,6 +24,20 @@
extern "C" { extern "C" {
#endif #endif
// start compress flag
// |----l1 compAlg----|-----l2 compAlg---|---level--|
// |------8bit--------|------16bit-------|---8bit---|
#define COMPRESS_L1_TYPE_U32(type) (((type) >> 24) & 0xFF)
#define COMPRESS_L2_TYPE_U32(type) (((type) >> 8) & 0xFFFF)
#define COMPRESS_L2_TYPE_LEVEL_U32(type) ((type)&0xFF)
// compress flag
// |----l2lel--|----l2Alg---|---l1Alg--|
// |----2bit---|----3bit----|---3bit---|
#define COMPRESS_L1_TYPE_U8(type) ((type)&0x07)
#define COMPRESS_L2_TYPE_U8(type) (((type) >> 3) & 0x07)
#define COMPRESS_L2_TYPE_LEVEL_U8(type) (((type) >> 6) & 0x03)
// end compress flag
#define COMP_OVERFLOW_BYTES 2 #define COMP_OVERFLOW_BYTES 2
#define BITS_PER_BYTE 8 #define BITS_PER_BYTE 8
// Masks // Masks

2
source/util/src/tcompression.c
View File

@ -52,7 +52,7 @@
#include "lz4.h" #include "lz4.h"
#include "tcompare.h" #include "tcompare.h"
#include "tlog.h" #include "tlog.h"
#include "tmsg.h" // #include "tmsg.h"
#include "zlib.h" #include "zlib.h"
#include "zstd.h" #include "zstd.h"