xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

more code

This commit is contained in:
Hongze Cheng 2024-01-26 17:14:00 +08:00
parent cd4c188a8a
commit e7177dae57
13 changed files with 1123 additions and 1349 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
include/common/tdataformat.h
View File

@ -27,7 +27,6 @@
extern "C" { extern "C" {
#endif #endif
typedef struct SBuffer SBuffer;
typedef struct SSchema SSchema; typedef struct SSchema SSchema;
typedef struct SSchema2 SSchema2; typedef struct SSchema2 SSchema2;
typedef struct STColumn STColumn; typedef struct STColumn STColumn;
@ -39,10 +38,12 @@ typedef struct SRowIter SRowIter;
typedef struct STagVal STagVal; typedef struct STagVal STagVal;
typedef struct STag STag; typedef struct STag STag;
typedef struct SColData SColData; typedef struct SColData SColData;
typedef struct SRowKey SRowKey;
#define HAS_NONE ((uint8_t)0x1) #define HAS_NONE ((uint8_t)0x1)
#define HAS_NULL ((uint8_t)0x2) #define HAS_NULL ((uint8_t)0x2)
#define HAS_VALUE ((uint8_t)0x4) #define HAS_VALUE ((uint8_t)0x4)
#define HAS_MULTI_KEY ((uint8_t)0x8)
// bitmap ================================ // bitmap ================================
const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111,
@ -78,19 +79,6 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111
} while (0) } while (0)
#define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) #define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE)
// SBuffer ================================
struct SBuffer {
int64_t nBuf;
uint8_t *pBuf;
};
#define tBufferCreate() \
(SBuffer) { .nBuf = 0, .pBuf = NULL }
void tBufferDestroy(SBuffer *pBuffer);
int32_t tBufferInit(SBuffer *pBuffer, int64_t size);
int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData);
int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData);
// SColVal ================================ // SColVal ================================
#define CV_FLAG_VALUE ((int8_t)0x0) #define CV_FLAG_VALUE ((int8_t)0x0)
#define CV_FLAG_NONE ((int8_t)0x1) #define CV_FLAG_NONE ((int8_t)0x1)
@ -111,6 +99,8 @@ void tRowDestroy(SRow *pRow);
int32_t tRowSort(SArray *aRowP); int32_t tRowSort(SArray *aRowP);
int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag);
int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag);
void tRowGetKey(SRow *pRow, SRowKey *key);
int32_t tRowKeyCmpr(const void *p1, const void *p2);
// SRowIter ================================ // SRowIter ================================
int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter);
@ -134,7 +124,7 @@ int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, voi
// SColData ================================ // SColData ================================
typedef void *(*xMallocFn)(void *, int32_t); typedef void *(*xMallocFn)(void *, int32_t);
void tColDataDestroy(void *ph); void tColDataDestroy(void *ph);
void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t cflag);
void tColDataClear(SColData *pColData); void tColDataClear(SColData *pColData);
void tColDataDeepClear(SColData *pColData); void tColDataDeepClear(SColData *pColData);
int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
@ -172,6 +162,15 @@ struct STSchema {
STColumn columns[]; STColumn columns[];
}; };
/*
* 1. Tuple format:
* SRow + [bit map +] fix-length data + [var-length data +] [(type + offset) * numPrimaryKeyCols + fixedLen +
* numOfCols + numPrimaryKeyCols]
*
* 2. K-V format:
* SRow + numColsNotNone + u8/u16/u32 * numColsNotNone + ([-]cid [+ data]) * numColsNotNone + [(type + index) *
* numPrimaryKeyCols + numPrimaryKeyCols】
*/
struct SRow { struct SRow {
uint8_t flag; uint8_t flag;
uint8_t rsv; uint8_t rsv;
@ -191,6 +190,18 @@ struct SValue {
}; };
}; };
#define TD_MAX_PRIMARY_KEY_COL 8
typedef struct {
int8_t type;
SValue value;
} STypeValue;
struct SRowKey {
TSKEY ts;
uint8_t numOfKeys;
STypeValue keys[TD_MAX_PRIMARY_KEY_COL];
};
struct SColVal { struct SColVal {
int16_t cid; int16_t cid;
int8_t type; int8_t type;
@ -201,7 +212,7 @@ struct SColVal {
struct SColData { struct SColData {
int16_t cid; int16_t cid;
int8_t type; int8_t type;
int8_t smaOn; int8_t cflag;
int32_t numOfNone; // # of none int32_t numOfNone; // # of none
int32_t numOfNull; // # of null int32_t numOfNull; // # of null
int32_t numOfValue; // # of vale int32_t numOfValue; // # of vale

14
include/common/tmsg.h
View File

@ -1393,7 +1393,7 @@ void tFreeSCompactDbReq(SCompactDbReq* pReq);
typedef struct { typedef struct {
int32_t compactId; int32_t compactId;
int8_t bAccepted; int8_t bAccepted;
} SCompactDbRsp; } SCompactDbRsp;
int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp); int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp);
@ -1407,7 +1407,7 @@ typedef struct {
int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq); int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq); int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq);
void tFreeSKillCompactReq(SKillCompactReq *pReq); void tFreeSKillCompactReq(SKillCompactReq* pReq);
typedef struct { typedef struct {
char name[TSDB_FUNC_NAME_LEN]; char name[TSDB_FUNC_NAME_LEN];
@ -1741,9 +1741,9 @@ int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq*
int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq);
typedef struct { typedef struct {
int32_t compactId; int32_t compactId;
int32_t vgId; int32_t vgId;
int32_t dnodeId; int32_t dnodeId;
} SVKillCompactReq; } SVKillCompactReq;
int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq); int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq);
@ -1944,9 +1944,9 @@ typedef struct {
char db[TSDB_DB_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN]; char tb[TSDB_TABLE_NAME_LEN];
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];
char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns
int64_t showId; int64_t showId;
int64_t compactId; // for compact int64_t compactId; // for compact
} SRetrieveTableReq; } SRetrieveTableReq;
typedef struct SSysTableSchema { typedef struct SSysTableSchema {

1
include/util/tdef.h
View File

@ -444,6 +444,7 @@ typedef enum ELogicConditionType {
#define TSDB_MAX_VARBINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 #define TSDB_MAX_VARBINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519
#define PRIMARYKEY_TIMESTAMP_COL_ID 1 #define PRIMARYKEY_TIMESTAMP_COL_ID 1
#define PRIMARYKEY_COL_ID 2 // for another primary key column in addition to timestamp
#define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) #define COL_REACH_END(colId, maxColId) ((colId) > (maxColId))
#ifdef WINDOWS #ifdef WINDOWS

371
include/util/tencode.h
View File

@ -468,272 +468,217 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) {
} }
// =========================================== // ===========================================
#define tPutV(p, v) \ #define TPUT(TYPE, BUF, VAL, FORWARD) \
do { \ do { \
int32_t n = 0; \ if (BUF) { \
for (;;) { \ if (FORWARD) { \
if (v <= 0x7f) { \ *(TYPE*)(BUF) = (VAL); \
if (p) p[n] = v; \ } else { \
n++; \ *(TYPE*)((BUF) - sizeof(TYPE)) = (VAL); \
break; \ } \
} \ } \
if (p) p[n] = (v & 0x7f) | 0x80; \ return sizeof(TYPE); \
n++; \ } while (0)
v >>= 7; \
} \ #define TGET(TYPE, BUF, VAL, FORWARD) \
return n; \ do { \
if (FORWARD) { \
*(TYPE*)(VAL) = *(TYPE*)(BUF); \
} else { \
*(TYPE*)(VAL) = *(TYPE*)((BUF) - sizeof(TYPE)); \
} \
return sizeof(TYPE); \
} while (0)
#define TPUTV(BUF, VAL, FORWARD) \
do { \
int32_t n = 0; \
for (;;) { \
if ((VAL) < 0x80) { \
if (BUF) { \
if (FORWARD) { \
(BUF)[n] = (VAL); \
} else { \
(BUF)[-(n + 1)] = (VAL); \
} \
} \
n++; \
break; \
} else { \
if (BUF) { \
if (FORWARD) { \
(BUF)[n] = ((VAL)&0x7f) | 0x80; \
} else { \
(BUF)[-(n + 1)] = ((VAL)&0x7f) | 0x80; \
} \
} \
n++; \
(VAL) >>= 7; \
} \
} \
return n; \
} while (0)
#define TGETV(BUF, VAL, FORWARD) \
do { \
int32_t n = 0; \
if (VAL) { \
*(VAL) = 0; \
} \
for (;;) { \
if (FORWARD) { \
if (VAL) { \
(*(VAL)) |= ((BUF)[n] & 0x7f) << (7 * n); \
} \
if ((BUF)[n++] < 0x80) break; \
} else { \
if (VAL) { \
(*(VAL)) |= ((BUF)[-(n + 1)] & 0x7f) << (7 * n); \
} \
if ((BUF)[-(n++)] < 0x80) break; \
} \
} \
return n; \
} while (0) } while (0)
// PUT // PUT
static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v, bool forward) { TPUT(uint8_t, p, v, forward); }
if (p) ((uint8_t*)p)[0] = v; static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v, bool forward) { TPUT(int8_t, p, v, forward); }
return sizeof(uint8_t); static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v, bool forward) { TPUT(uint16_t, p, v, forward); }
} static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v, bool forward) { TPUT(int16_t, p, v, forward); }
static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v, bool forward) { TPUT(uint32_t, p, v, forward); }
static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v, bool forward) { TPUT(int32_t, p, v, forward); }
static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v, bool forward) { TPUT(uint64_t, p, v, forward); }
static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v, bool forward) { TPUT(int64_t, p, v, forward); }
static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f, bool forward) {
if (p) ((int8_t*)p)[0] = v;
return sizeof(int8_t);
}
static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) {
if (p) ((uint16_t*)p)[0] = v;
return sizeof(uint16_t);
}
static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) {
if (p) ((int16_t*)p)[0] = v;
return sizeof(int16_t);
}
static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) {
if (p) ((uint32_t*)p)[0] = v;
return sizeof(uint32_t);
}
static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) {
if (p) ((int32_t*)p)[0] = v;
return sizeof(int32_t);
}
static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) {
if (p) ((uint64_t*)p)[0] = v;
return sizeof(uint64_t);
}
static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) {
if (p) ((int64_t*)p)[0] = v;
return sizeof(int64_t);
}
static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) {
union { union {
uint32_t ui; uint32_t ui;
float f; float f;
} v; } v = {.f = f};
v.f = f; return tPutU32(p, v.ui, forward);
return tPutU32(p, v.ui);
} }
static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d, bool forward) {
union { union {
uint64_t ui; uint64_t ui;
double d; double d;
} v; } v = {.d = d};
v.d = d; return tPutU64(p, v.ui, forward);
return tPutU64(p, v.ui);
} }
static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v, bool forward) { TPUTV(p, v, forward); }
static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v, bool forward) {
static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } return tPutU16v(p, ZIGZAGE(int16_t, v), forward);
}
static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v, bool forward) { TPUTV(p, v, forward); }
static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v, bool forward) {
static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } return tPutU32v(p, ZIGZAGE(int32_t, v), forward);
}
static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v, bool forward) { TPUTV(p, v, forward); }
static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v, bool forward) {
static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } return tPutU64v(p, ZIGZAGE(int64_t, v), forward);
}
// GET // GET
static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v, bool forward) { TGET(uint8_t, p, v, forward); }
if (v) *v = ((uint8_t*)p)[0]; static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v, bool forward) { TGET(int8_t, p, v, forward); }
return sizeof(uint8_t); static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v, bool forward) { TGET(uint16_t, p, v, forward); }
} static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v, bool forward) { TGET(int16_t, p, v, forward); }
static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v, bool forward) { TGET(uint32_t, p, v, forward); }
static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v, bool forward) { TGET(int32_t, p, v, forward); }
if (v) *v = ((int8_t*)p)[0]; static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v, bool forward) { TGET(uint64_t, p, v, forward); }
return sizeof(int8_t); static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v, bool forward) { TGET(int64_t, p, v, forward); }
} static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v, bool forward) { TGETV(p, v, forward); }
static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v, bool forward) {
static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) {
if (v) *v = ((uint16_t*)p)[0];
return sizeof(uint16_t);
}
static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) {
if (v) *v = ((int16_t*)p)[0];
return sizeof(int16_t);
}
static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) {
if (v) *v = ((uint32_t*)p)[0];
return sizeof(uint32_t);
}
static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) {
if (v) *v = ((int32_t*)p)[0];
return sizeof(int32_t);
}
static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) {
if (v) *v = ((uint64_t*)p)[0];
return sizeof(uint64_t);
}
static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) {
if (v) *v = ((int64_t*)p)[0];
return sizeof(int64_t);
}
static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) {
int32_t n = 0;
if (v) *v = 0;
for (;;) {
if (p[n] <= 0x7f) {
if (v) (*v) |= (((uint16_t)p[n]) << (7 * n));
n++;
break;
}
if (v) (*v) |= (((uint16_t)(p[n] & 0x7f)) << (7 * n));
n++;
}
return n;
}
static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) {
int32_t n;
uint16_t tv; uint16_t tv;
int32_t n = tGetU16v(p, &tv, forward);
n = tGetU16v(p, &tv); if (v) {
if (v) *v = ZIGZAGD(int16_t, tv); *v = ZIGZAGD(int16_t, tv);
return n;
}
static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) {
int32_t n = 0;
if (v) *v = 0;
for (;;) {
if (p[n] <= 0x7f) {
if (v) (*v) |= (((uint32_t)p[n]) << (7 * n));
n++;
break;
}
if (v) (*v) |= (((uint32_t)(p[n] & 0x7f)) << (7 * n));
n++;
} }
return n; return n;
} }
static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v, bool forward) { TGETV(p, v, forward); }
static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v, bool forward) {
int32_t n;
uint32_t tv; uint32_t tv;
int32_t n = tGetU32v(p, &tv, forward);
n = tGetU32v(p, &tv); if (v) {
if (v) *v = ZIGZAGD(int32_t, tv); *v = ZIGZAGD(int32_t, tv);
return n;
}
static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) {
int32_t n = 0;
if (v) *v = 0;
for (;;) {
if (p[n] <= 0x7f) {
if (v) (*v) |= (((uint64_t)p[n]) << (7 * n));
n++;
break;
}
if (v) (*v) |= (((uint64_t)(p[n] & 0x7f)) << (7 * n));
n++;
} }
return n; return n;
} }
static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v, bool forward) { TGETV(p, v, forward); }
static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v, bool forward) {
int32_t n;
uint64_t tv; uint64_t tv;
int32_t n = tGetU64v(p, &tv, forward);
n = tGetU64v(p, &tv); if (v) {
if (v) *v = ZIGZAGD(int64_t, tv); *v = ZIGZAGD(int64_t, tv);
}
return n; return n;
} }
static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f, bool forward) {
static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) {
int32_t n = 0;
union { union {
uint32_t ui; uint32_t ui;
float f; float f;
} v; } v;
n = tGetU32(p, &v.ui); int32_t n = tGetU32(p, &v.ui, forward);
if (f) {
*f = v.f; *f = v.f;
}
return n; return n;
} }
static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) { static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d, bool forward) {
int32_t n = 0;
union { union {
uint64_t ui; uint64_t ui;
double d; double d;
} v; } v;
n = tGetU64(p, &v.ui); int32_t n = tGetU64(p, &v.ui, forward);
if (d) {
*d = v.d; *d = v.d;
}
return n; return n;
} }
// ===================== // =====================
static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData, bool forward) {
int n = 0; int32_t n = tPutU32v(p, nData, forward);
if (p) {
n += tPutU32v(p ? p + n : p, nData); if (forward) {
if (p) memcpy(p + n, pData, nData); memcpy(p + n, pData, nData);
} else {
memcpy(p - n - nData, pData, nData);
}
}
n += nData; n += nData;
return n; return n;
} }
static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) { static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData, bool forward) {
int32_t n = 0;
uint32_t nt; uint32_t nt;
int32_t n = tGetU32v(p, &nt, forward);
n += tGetU32v(p, &nt); if (nData) {
if (nData) *nData = nt; *nData = nt;
if (ppData) *ppData = p + n; }
n += nt; if (ppData) {
if (forward) {
return n; *ppData = p + n;
} else {
*ppData = p - n - nt;
}
}
return n + nt;
} }
static FORCE_INLINE int32_t tPutCStr(uint8_t* p, char* pData) { static FORCE_INLINE int32_t tPutCStr(uint8_t* p, char* pData, bool forward) {
return tPutBinary(p, (uint8_t*)pData, strlen(pData) + 1); return tPutBinary(p, (uint8_t*)pData, strlen(pData) + 1, forward);
}
static FORCE_INLINE int32_t tGetCStr(uint8_t* p, char** ppData, bool forward) {
return tGetBinary(p, (uint8_t**)ppData, NULL, forward);
} }
static FORCE_INLINE int32_t tGetCStr(uint8_t* p, char** ppData) { return tGetBinary(p, (uint8_t**)ppData, NULL); }
#ifdef __cplusplus #ifdef __cplusplus
} }

376
source/common/src/tdataformat.c
View File

@ -23,46 +23,13 @@
static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData);
static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward); static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward);
// SBuffer ================================
void tBufferDestroy(SBuffer *pBuffer) {
tFree(pBuffer->pBuf);
pBuffer->pBuf = NULL;
}
int32_t tBufferInit(SBuffer *pBuffer, int64_t size) {
pBuffer->nBuf = 0;
return tRealloc(&pBuffer->pBuf, size);
}
int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) {
int32_t code = 0;
code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData);
if (code) return code;
memcpy(pBuffer->pBuf + pBuffer->nBuf, pData, nData);
pBuffer->nBuf += nData;
return code;
}
int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData) {
int32_t code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData);
if (code) return code;
*ppData = pBuffer->pBuf + pBuffer->nBuf;
pBuffer->nBuf += nData;
return code;
}
// ================================ // ================================
static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson);
// SRow ======================================================================== // SRow ========================================================================
#define KV_FLG_LIT ((uint8_t)0x10) #define KV_FLG_LIT ((uint8_t)0x10)
#define KV_FLG_MID ((uint8_t)0x20) #define KV_FLG_MID ((uint8_t)0x20)
#define KV_FLG_BIG ((uint8_t)0x30) #define KV_FLG_BIG ((uint8_t)0x40)
#define BIT_FLG_NONE ((uint8_t)0x0) #define BIT_FLG_NONE ((uint8_t)0x0)
#define BIT_FLG_NULL ((uint8_t)0x1) #define BIT_FLG_NULL ((uint8_t)0x1)
@ -71,7 +38,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson);
#pragma pack(push, 1) #pragma pack(push, 1)
typedef struct { typedef struct {
int16_t nCol; int16_t nCol;
char idx[]; // uint8_t * | uint16_t * | uint32_t * uint8_t idx[]; // uint8_t * | uint16_t * | uint32_t *
} SKVIdx; } SKVIdx;
#pragma pack(pop) #pragma pack(pop)
@ -118,6 +85,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
int32_t nkv = 0; int32_t nkv = 0;
int32_t maxIdx = 0; int32_t maxIdx = 0;
int32_t nIdx = 0; int32_t nIdx = 0;
uint8_t numPrimaryKeyCols = 0;
while (pTColumn) { while (pTColumn) {
if (pColVal) { if (pColVal) {
if (pColVal->cid == pTColumn->colId) { if (pColVal->cid == pTColumn->colId) {
@ -125,18 +93,33 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
flag |= HAS_VALUE; flag |= HAS_VALUE;
maxIdx = nkv; maxIdx = nkv;
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; ntp = ntp + tPutU32v(NULL, pColVal->value.nData, true) + pColVal->value.nData;
nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; nkv = nkv + tPutI16v(NULL, pTColumn->colId, true) + tPutU32v(NULL, pColVal->value.nData, true) +
pColVal->value.nData;
} else { } else {
nkv = nkv + tPutI16v(NULL, pTColumn->colId) + pTColumn->bytes; nkv = nkv + tPutI16v(NULL, pTColumn->colId, true) + pTColumn->bytes;
} }
if (pTColumn->flags & COL_IS_KEY) {
flag |= HAS_MULTI_KEY;
numPrimaryKeyCols++;
ntp += (tPutI8(NULL, pTColumn->type, false) // type
+ tPutI32v(NULL, pTColumn->offset, false) // offset
);
nkv += (tPutI8(NULL, pTColumn->type, false) // type
+ tPutI32v(NULL, nIdx, false) // index
);
}
nIdx++; nIdx++;
} else if (COL_VAL_IS_NONE(pColVal)) { // NONE } else if (COL_VAL_IS_NONE(pColVal)) { // NONE
ASSERT((pTColumn->flags & COL_IS_KEY) == 0);
flag |= HAS_NONE; flag |= HAS_NONE;
} else if (COL_VAL_IS_NULL(pColVal)) { // NULL } else if (COL_VAL_IS_NULL(pColVal)) { // NULL
ASSERT((pTColumn->flags & COL_IS_KEY) == 0);
flag |= HAS_NULL; flag |= HAS_NULL;
maxIdx = nkv; maxIdx = nkv;
nkv += tPutI16v(NULL, -pTColumn->colId); nkv += tPutI16v(NULL, -pTColumn->colId, true);
nIdx++; nIdx++;
} else { } else {
if (ASSERTS(0, "invalid input")) { if (ASSERTS(0, "invalid input")) {
@ -148,19 +131,22 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL;
pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL;
} else if (pColVal->cid > pTColumn->colId) { // NONE } else if (pColVal->cid > pTColumn->colId) { // NONE
ASSERT((pTColumn->flags & COL_IS_KEY) == 0);
flag |= HAS_NONE; flag |= HAS_NONE;
pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL;
} else { } else {
pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL;
} }
} else { // NONE } else { // NONE
ASSERT((pTColumn->flags & COL_IS_KEY) == 0);
flag |= HAS_NONE; flag |= HAS_NONE;
pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL;
} }
} }
// compare --------------- // compare ---------------
switch (flag) { // Tuple Row Format
switch (flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) {
case HAS_NONE: case HAS_NONE:
case HAS_NULL: case HAS_NULL:
ntp = sizeof(SRow); ntp = sizeof(SRow);
@ -184,6 +170,13 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
goto _exit; goto _exit;
} }
} }
if (flag & HAS_MULTI_KEY) {
ntp += (tPutI8(NULL, numPrimaryKeyCols, false) // numPrimaryKeyCols
+ tPutU32v(NULL, pTSchema->numOfCols, false) // numOfCols
+ tPutU32v(NULL, pTSchema->flen, false) // fixedLen
);
}
// Key-Value Row Format
if (maxIdx <= UINT8_MAX) { if (maxIdx <= UINT8_MAX) {
nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv;
flag |= KV_FLG_LIT; flag |= KV_FLG_LIT;
@ -194,16 +187,19 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv; nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv;
flag |= KV_FLG_BIG; flag |= KV_FLG_BIG;
} }
int32_t nRow; if (flag & HAS_MULTI_KEY) {
nkv += tPutI8(NULL, numPrimaryKeyCols, false);
}
int32_t rowLength;
if (nkv < ntp) { if (nkv < ntp) {
nRow = nkv; rowLength = nkv;
} else { } else {
nRow = ntp; rowLength = ntp;
flag &= ((uint8_t)0x0f); flag &= ((uint8_t)0x0f);
} }
// alloc -------------- // alloc --------------
pRow = taosMemoryMalloc(nRow); pRow = taosMemoryMalloc(rowLength);
if (NULL == pRow) { if (NULL == pRow) {
code = TSDB_CODE_OUT_OF_MEMORY; code = TSDB_CODE_OUT_OF_MEMORY;
goto _exit; goto _exit;
@ -215,7 +211,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
pRow->flag = flag; pRow->flag = flag;
pRow->rsv = 0; pRow->rsv = 0;
pRow->sver = pTSchema->version; pRow->sver = pTSchema->version;
pRow->len = nRow; pRow->len = rowLength;
memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY)); memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY));
if (flag == HAS_NONE || flag == HAS_NULL) { if (flag == HAS_NONE || flag == HAS_NULL) {
@ -226,6 +222,13 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL;
iTColumn = 1; iTColumn = 1;
pTColumn = pTSchema->columns + iTColumn; pTColumn = pTSchema->columns + iTColumn;
uint8_t *pEnd = ((uint8_t *)pRow) + rowLength;
if (pRow->flag & HAS_MULTI_KEY) {
pEnd -= tPutI8(pEnd, numPrimaryKeyCols, false); // numPrimaryKeyCols
pEnd -= tPutU32v(pEnd, pTSchema->numOfCols, false); // numOfCols
pEnd -= tPutU32v(NULL, pTSchema->flen, false); // fixedLen
}
if (flag >> 4) { // KV if (flag >> 4) { // KV
SKVIdx *pIdx = (SKVIdx *)pRow->data; SKVIdx *pIdx = (SKVIdx *)pRow->data;
int32_t iIdx = 0; int32_t iIdx = 0;
@ -251,17 +254,23 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
} else { } else {
((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv;
} }
iIdx++;
nv += tPutI16v(pv + nv, pTColumn->colId); nv += tPutI16v(pv + nv, pTColumn->colId, true);
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
nv += tPutU32v(pv + nv, pColVal->value.nData); nv += tPutU32v(pv + nv, pColVal->value.nData, true);
memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData);
nv += pColVal->value.nData; nv += pColVal->value.nData;
} else { } else {
memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes); memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes);
nv += pTColumn->bytes; nv += pTColumn->bytes;
} }
if (pRow->flag & HAS_MULTI_KEY) {
pEnd -= tPutI8(pEnd, pTColumn->type, false);
pEnd -= tPutI32v(pEnd, iIdx, false);
}
iIdx++;
} else if (COL_VAL_IS_NULL(pColVal)) { } else if (COL_VAL_IS_NULL(pColVal)) {
if (flag & KV_FLG_LIT) { if (flag & KV_FLG_LIT) {
((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv;
@ -270,8 +279,8 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
} else { } else {
((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv;
} }
nv += tPutI16v(pv + nv, -pTColumn->colId, true);
iIdx++; iIdx++;
nv += tPutI16v(pv + nv, -pTColumn->colId);
} }
pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL;
@ -291,7 +300,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
uint8_t *pv = NULL; uint8_t *pv = NULL;
int32_t nv = 0; int32_t nv = 0;
switch (flag) { switch (flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) {
case (HAS_NULL | HAS_NONE): case (HAS_NULL | HAS_NONE):
pb = pRow->data; pb = pRow->data;
break; break;
@ -334,7 +343,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
*(int32_t *)(pf + pTColumn->offset) = nv; *(int32_t *)(pf + pTColumn->offset) = nv;
nv += tPutU32v(pv + nv, pColVal->value.nData); nv += tPutU32v(pv + nv, pColVal->value.nData, true);
if (pColVal->value.nData) { if (pColVal->value.nData) {
memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData);
nv += pColVal->value.nData; nv += pColVal->value.nData;
@ -342,6 +351,11 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) {
} else { } else {
memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]);
} }
if (pTColumn->flags & COL_IS_KEY) {
pEnd -= tPutI8(pEnd, pTColumn->type, false);
pEnd -= tPutI32v(pEnd, pTColumn->offset, false);
}
} else if (COL_VAL_IS_NONE(pColVal)) { // NONE } else if (COL_VAL_IS_NONE(pColVal)) { // NONE
ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE);
if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]);
@ -391,12 +405,12 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
return 0; return 0;
} }
if (pRow->flag == HAS_NONE) { if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NONE) {
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
return 0; return 0;
} }
if (pRow->flag == HAS_NULL) { if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NULL) {
*pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type);
return 0; return 0;
} }
@ -426,7 +440,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
} }
int16_t cid; int16_t cid;
pData += tGetI16v(pData, &cid); pData += tGetI16v(pData, &cid, true);
if (TABS(cid) == pTColumn->colId) { if (TABS(cid) == pTColumn->colId) {
if (cid < 0) { if (cid < 0) {
@ -437,7 +451,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->flag = CV_FLAG_VALUE; pColVal->flag = CV_FLAG_VALUE;
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
pData += tGetU32v(pData, &pColVal->value.nData); pData += tGetU32v(pData, &pColVal->value.nData, true);
if (pColVal->value.nData > 0) { if (pColVal->value.nData > 0) {
pColVal->value.pData = pData; pColVal->value.pData = pData;
} else { } else {
@ -457,13 +471,13 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
*pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
} else { // Tuple Row } else { // Tuple Row
if (pRow->flag == HAS_VALUE) { if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_VALUE) {
pColVal->cid = pTColumn->colId; pColVal->cid = pTColumn->colId;
pColVal->type = pTColumn->type; pColVal->type = pTColumn->type;
pColVal->flag = CV_FLAG_VALUE; pColVal->flag = CV_FLAG_VALUE;
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
uint8_t *pData = pRow->data + pTSchema->flen + *(int32_t *)(pRow->data + pTColumn->offset); uint8_t *pData = pRow->data + pTSchema->flen + *(int32_t *)(pRow->data + pTColumn->offset);
pData += tGetU32v(pData, &pColVal->value.nData); pData += tGetU32v(pData, &pColVal->value.nData, true);
if (pColVal->value.nData) { if (pColVal->value.nData) {
pColVal->value.pData = pData; pColVal->value.pData = pData;
} else { } else {
@ -477,7 +491,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
uint8_t *pv; uint8_t *pv;
uint8_t bv = BIT_FLG_VALUE; uint8_t bv = BIT_FLG_VALUE;
switch (pRow->flag) { switch (pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) {
case (HAS_NULL | HAS_NONE): case (HAS_NULL | HAS_NONE):
bv = GET_BIT1(pRow->data, iCol - 1); bv = GET_BIT1(pRow->data, iCol - 1);
break; break;
@ -516,7 +530,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal)
pColVal->flag = CV_FLAG_VALUE; pColVal->flag = CV_FLAG_VALUE;
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset);
pData += tGetU32v(pData, &pColVal->value.nData); pData += tGetU32v(pData, &pColVal->value.nData, true);
if (pColVal->value.nData) { if (pColVal->value.nData) {
pColVal->value.pData = pData; pColVal->value.pData = pData;
} else { } else {
@ -682,7 +696,9 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) {
pIter->pTSchema = pTSchema; pIter->pTSchema = pTSchema;
pIter->iTColumn = 0; pIter->iTColumn = 0;
if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NONE ||
(pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NULL)
goto _exit;
if (pRow->flag >> 4) { if (pRow->flag >> 4) {
pIter->iCol = 0; pIter->iCol = 0;
@ -695,7 +711,7 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) {
pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); // * sizeof(uint32_t) pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); // * sizeof(uint32_t)
} }
} else { } else {
switch (pRow->flag) { switch (pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) {
case (HAS_NULL | HAS_NONE): case (HAS_NULL | HAS_NONE):
pIter->pb = pRow->data; pIter->pb = pRow->data;
break; break;
@ -753,12 +769,13 @@ SColVal *tRowIterNext(SRowIter *pIter) {
goto _exit; goto _exit;
} }
if (pIter->pRow->flag == HAS_NONE) { uint8_t tflag = (pIter->pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE));
if (tflag == HAS_NONE) {
pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type);
goto _exit; goto _exit;
} }
if (pIter->pRow->flag == HAS_NULL) { if (tflag == HAS_NULL) {
pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type);
goto _exit; goto _exit;
} }
@ -776,7 +793,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
} }
int16_t cid; int16_t cid;
pData += tGetI16v(pData, &cid); pData += tGetI16v(pData, &cid, true);
if (TABS(cid) == pTColumn->colId) { if (TABS(cid) == pTColumn->colId) {
if (cid < 0) { if (cid < 0) {
@ -787,7 +804,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.flag = CV_FLAG_VALUE; pIter->cv.flag = CV_FLAG_VALUE;
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
pData += tGetU32v(pData, &pIter->cv.value.nData); pData += tGetU32v(pData, &pIter->cv.value.nData, true);
if (pIter->cv.value.nData > 0) { if (pIter->cv.value.nData > 0) {
pIter->cv.value.pData = pData; pIter->cv.value.pData = pData;
} else { } else {
@ -813,7 +830,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
} else { // Tuple } else { // Tuple
uint8_t bv = BIT_FLG_VALUE; uint8_t bv = BIT_FLG_VALUE;
if (pIter->pb) { if (pIter->pb) {
switch (pIter->pRow->flag) { switch (tflag) {
case (HAS_NULL | HAS_NONE): case (HAS_NULL | HAS_NONE):
bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1);
break; break;
@ -846,7 +863,7 @@ SColVal *tRowIterNext(SRowIter *pIter) {
pIter->cv.flag = CV_FLAG_VALUE; pIter->cv.flag = CV_FLAG_VALUE;
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset); uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset);
pData += tGetU32v(pData, &pIter->cv.value.nData); pData += tGetU32v(pData, &pIter->cv.value.nData, true);
if (pIter->cv.value.nData > 0) { if (pIter->cv.value.nData > 0) {
pIter->cv.value.pData = pData; pIter->cv.value.pData = pData;
} else { } else {
@ -921,7 +938,8 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *
uint8_t *pb = NULL, *pf = NULL, *pv = NULL; uint8_t *pb = NULL, *pf = NULL, *pv = NULL;
switch (pRow->flag) { uint8_t tflag = pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE);
switch (tflag) {
case HAS_VALUE: case HAS_VALUE:
pf = pRow->data; pf = pRow->data;
pv = pf + pTSchema->flen; pv = pf + pTSchema->flen;
@ -951,7 +969,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *
ASSERT(pTColumn->type == pColData->type); ASSERT(pTColumn->type == pColData->type);
if (pb) { if (pb) {
uint8_t bv; uint8_t bv;
switch (pRow->flag) { switch (tflag) {
case (HAS_NULL | HAS_NONE): case (HAS_NULL | HAS_NONE):
bv = GET_BIT1(pb, iTColumn - 1); bv = GET_BIT1(pb, iTColumn - 1);
break; break;
@ -988,7 +1006,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *
if (IS_VAR_DATA_TYPE(pColData->type)) { if (IS_VAR_DATA_TYPE(pColData->type)) {
uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset);
uint32_t nData; uint32_t nData;
pData += tGetU32v(pData, &nData); pData += tGetU32v(pData, &nData, true);
if (flag == 0) { if (flag == 0) {
code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData);
} else { } else {
@ -1062,7 +1080,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo
} }
int16_t cid; int16_t cid;
pData += tGetI16v(pData, &cid); pData += tGetI16v(pData, &cid, true);
if (TABS(cid) == pTColumn->colId) { if (TABS(cid) == pTColumn->colId) {
if (cid < 0) { if (cid < 0) {
@ -1075,7 +1093,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo
} else { } else {
uint32_t nData; uint32_t nData;
if (IS_VAR_DATA_TYPE(pTColumn->type)) { if (IS_VAR_DATA_TYPE(pTColumn->type)) {
pData += tGetU32v(pData, &nData); pData += tGetU32v(pData, &nData, true);
} else { } else {
nData = 0; nData = 0;
} }
@ -1123,9 +1141,10 @@ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, in
ASSERT(pRow->sver == pTSchema->version); ASSERT(pRow->sver == pTSchema->version);
ASSERT(nColData > 0); ASSERT(nColData > 0);
if (pRow->flag == HAS_NONE) { uint8_t tflag = pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE);
if (tflag == HAS_NONE) {
return tRowNoneUpsertColData(aColData, nColData, flag); return tRowNoneUpsertColData(aColData, nColData, flag);
} else if (pRow->flag == HAS_NULL) { } else if (tflag == HAS_NULL) {
return tRowNullUpsertColData(aColData, nColData, pTSchema, flag); return tRowNullUpsertColData(aColData, nColData, pTSchema, flag);
} else if (pRow->flag >> 4) { // KV row } else if (pRow->flag >> 4) { // KV row
return tRowKVUpsertColData(pRow, pTSchema, aColData, nColData, flag); return tRowKVUpsertColData(pRow, pTSchema, aColData, nColData, flag);
@ -1134,6 +1153,171 @@ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, in
} }
} }
void tRowGetKey(SRow *pRow, SRowKey *key) {
key->ts = pRow->ts;
if ((pRow->flag & HAS_MULTI_KEY) == 0) {
key->numOfKeys = 0;
} else {
uint8_t *pEnd = ((uint8_t *)pRow) + pRow->len;
pEnd -= tGetU8(pEnd, &key->numOfKeys, false);
if (pRow->flag >> 4) { // Key-Value format
SKVIdx *pKVIdx = (SKVIdx *)pRow->data;
uint8_t *pv;
if (pRow->flag & KV_FLG_LIT) {
pv = pKVIdx->idx + pKVIdx->nCol;
} else if (pRow->flag & KV_FLG_MID) {
pv = pKVIdx->idx + (pKVIdx->nCol << 1);
} else {
pv = pKVIdx->idx + (pKVIdx->nCol << 2);
}
for (uint8_t iKey = 0; iKey < key->numOfKeys; iKey++) {
int32_t index;
uint8_t *pData;
STypeValue *pValue = &key->keys[iKey];
pEnd -= tGetI8(pEnd, &pValue->type, false);
pEnd -= tGetI32v(pEnd, &index, false);
if (pRow->flag & KV_FLG_LIT) {
pData = pv + ((uint8_t *)pKVIdx->idx)[index];
} else if (pRow->flag & KV_FLG_MID) {
pData = pv + ((uint16_t *)pKVIdx->idx)[index];
} else {
pData = pv + ((uint32_t *)pKVIdx->idx)[index];
}
pData += tGetI16v(pData, NULL, true);
if (IS_VAR_DATA_TYPE(pValue->type)) {
pData += tGetU32v(pData, &pValue->value.nData, true);
pValue->value.pData = pData;
} else {
memcpy(&pValue->value.val, pData, TYPE_BYTES[pValue->type]);
}
}
} else { // Tuple format
uint8_t *pf;
uint8_t *pv;
uint32_t fixedLen;
uint32_t numOfCols;
pEnd -= tGetU32v(pEnd, &numOfCols, false);
pEnd -= tGetU32v(pEnd, &fixedLen, false);
switch (pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) {
case (HAS_VALUE | HAS_NONE):
case (HAS_VALUE | HAS_NULL):
pf = pRow->data + BIT1_SIZE(numOfCols - 1);
pv = pf + fixedLen;
break;
case (HAS_VALUE | HAS_NULL | HAS_NONE):
pf = pRow->data + BIT2_SIZE(numOfCols - 1);
pv = pf + fixedLen;
break;
default:
ASSERTS(0, "invalid row format");
}
for (uint8_t iKey = 0; iKey < key->numOfKeys; iKey++) {
int32_t offset;
STypeValue *pValue = &key->keys[iKey];
pEnd -= tGetI8(pEnd, &pValue->type, false);
pEnd -= tGetI32v(pEnd, &offset, false);
if (IS_VAR_DATA_TYPE(key->keys[iKey].type)) {
pValue->value.pData = pv + *(int32_t *)(pf + offset);
pValue->value.pData += tGetU32v(pValue->value.pData, &pValue->value.nData, true);
} else {
memcpy(&key->keys[iKey].value.val, pf + offset, TYPE_BYTES[key->keys[iKey].type]);
}
}
}
}
}
#define T_COMPARE_SCALAR_VALUE(TYPE, V1, V2) \
do { \
if (*(TYPE *)(V1) < *(TYPE *)(V2)) { \
return -1; \
} else if (*(TYPE *)(V1) > *(TYPE *)(V2)) { \
return 1; \
} else { \
return 0; \
} \
} while (0)
static int32_t tTypeValueCmpr(const STypeValue *tv1, const STypeValue *tv2) {
ASSERT(tv1->type == tv2->type);
switch (tv1->type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
T_COMPARE_SCALAR_VALUE(int8_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_SMALLINT:
T_COMPARE_SCALAR_VALUE(int16_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_INT:
T_COMPARE_SCALAR_VALUE(int32_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
T_COMPARE_SCALAR_VALUE(int64_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_FLOAT:
T_COMPARE_SCALAR_VALUE(float, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_DOUBLE:
T_COMPARE_SCALAR_VALUE(double, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_UTINYINT:
T_COMPARE_SCALAR_VALUE(uint8_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_USMALLINT:
T_COMPARE_SCALAR_VALUE(uint16_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_UINT:
T_COMPARE_SCALAR_VALUE(uint32_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_UBIGINT:
T_COMPARE_SCALAR_VALUE(uint64_t, &tv1->value.val, &tv2->value.val);
case TSDB_DATA_TYPE_GEOMETRY:
case TSDB_DATA_TYPE_BINARY: {
return strcmp((const char *)tv1->value.pData, (const char *)tv2->value.pData);
}
case TSDB_DATA_TYPE_NCHAR: {
int32_t ret = tasoUcs4Compare((TdUcs4 *)tv1->value.pData, (TdUcs4 *)tv2->value.pData,
tv1->value.nData < tv2->value.nData ? tv1->value.nData : tv2->value.nData);
return ret ? ret : (tv1->value.nData < tv2->value.nData ? -1 : (tv1->value.nData > tv2->value.nData ? 1 : 0));
}
case TSDB_DATA_TYPE_VARBINARY: {
int32_t ret = memcmp(tv1->value.pData, tv2->value.pData,
tv1->value.nData < tv2->value.nData ? tv1->value.nData : tv2->value.nData);
return ret ? ret : (tv1->value.nData < tv2->value.nData ? -1 : (tv1->value.nData > tv2->value.nData ? 1 : 0));
}
case TSDB_DATA_TYPE_DECIMAL:
ASSERT(0);
break;
default:
ASSERT(0);
}
return 0;
}
int32_t tRowKeyCmpr(const void *p1, const void *p2) {
SRowKey *key1 = (SRowKey *)p1;
SRowKey *key2 = (SRowKey *)p2;
if (key1->ts < key2->ts) {
return -1;
} else if (key1->ts > key2->ts) {
return 1;
}
for (uint8_t iKey = 0; iKey < key1->numOfKeys; iKey++) {
int32_t ret = tTypeValueCmpr(&key1->keys[iKey], &key2->keys[iKey]);
if (ret) return ret;
}
return 0;
}
// STag ======================================== // STag ========================================
static int tTagValCmprFn(const void *p1, const void *p2) { static int tTagValCmprFn(const void *p1, const void *p2) {
if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) {
@ -1245,18 +1429,18 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
// key // key
if (isJson) { if (isJson) {
n += tPutCStr(p ? p + n : p, pTagVal->pKey); n += tPutCStr(p ? p + n : p, pTagVal->pKey, true);
} else { } else {
n += tPutI16v(p ? p + n : p, pTagVal->cid); n += tPutI16v(p ? p + n : p, pTagVal->cid, true);
ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid);
} }
// type // type
n += tPutI8(p ? p + n : p, pTagVal->type); n += tPutI8(p ? p + n : p, pTagVal->type, true);
// value // value
if (IS_VAR_DATA_TYPE(pTagVal->type)) { if (IS_VAR_DATA_TYPE(pTagVal->type)) {
n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData, true);
} else { } else {
p = p ? p + n : p; p = p ? p + n : p;
n += tDataTypes[pTagVal->type].bytes; n += tDataTypes[pTagVal->type].bytes;
@ -1270,17 +1454,17 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) {
// key // key
if (isJson) { if (isJson) {
n += tGetCStr(p + n, &pTagVal->pKey); n += tGetCStr(p + n, &pTagVal->pKey, true);
} else { } else {
n += tGetI16v(p + n, &pTagVal->cid); n += tGetI16v(p + n, &pTagVal->cid, true);
} }
// type // type
n += tGetI8(p + n, &pTagVal->type); n += tGetI8(p + n, &pTagVal->type, true);
// value // value
if (IS_VAR_DATA_TYPE(pTagVal->type)) { if (IS_VAR_DATA_TYPE(pTagVal->type)) {
n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData, true);
} else { } else {
memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes);
n += tDataTypes[pTagVal->type].bytes; n += tDataTypes[pTagVal->type].bytes;
@ -1510,7 +1694,7 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) {
offset = pTag->idx[iTag]; offset = pTag->idx[iTag];
} }
tPutI16v(p + offset, cid); tPutI16v(p + offset, cid, true);
} }
// STSchema ======================================== // STSchema ========================================
@ -1571,10 +1755,10 @@ void tColDataDestroy(void *ph) {
} }
} }
void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t cflag) {
pColData->cid = cid; pColData->cid = cid;
pColData->type = type; pColData->type = type;
pColData->smaOn = smaOn; pColData->cflag = cflag;
tColDataClear(pColData); tColDataClear(pColData);
} }
@ -2751,7 +2935,7 @@ static int32_t tColDataMergeSortMerge(SColData *aColData, int32_t start, int32_t
if (end > start) { if (end > start) {
aDstColData = taosMemoryCalloc(1, sizeof(SColData) * nColData); aDstColData = taosMemoryCalloc(1, sizeof(SColData) * nColData);
for (int c = 0; c < nColData; ++c) { for (int c = 0; c < nColData; ++c) {
tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].smaOn); tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].cflag);
} }
if (aDstColData == NULL) { if (aDstColData == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -3096,10 +3280,10 @@ _exit:
int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { int32_t tPutColData(uint8_t *pBuf, SColData *pColData) {
int32_t n = 0; int32_t n = 0;
n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid); n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid, true);
n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type); n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type, true);
n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal); n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal, true);
n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag); n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag, true);
// bitmap // bitmap
switch (pColData->flag) { switch (pColData->flag) {
@ -3123,7 +3307,7 @@ int32_t tPutColData(uint8_t *pBuf, SColData *pColData) {
if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2);
n += (pColData->nVal << 2); n += (pColData->nVal << 2);
n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData, true);
if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData);
n += pColData->nData; n += pColData->nData;
} else { } else {
@ -3138,10 +3322,10 @@ int32_t tPutColData(uint8_t *pBuf, SColData *pColData) {
int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { int32_t tGetColData(uint8_t *pBuf, SColData *pColData) {
int32_t n = 0; int32_t n = 0;
n += tGetI16v(pBuf + n, &pColData->cid); n += tGetI16v(pBuf + n, &pColData->cid, true);
n += tGetI8(pBuf + n, &pColData->type); n += tGetI8(pBuf + n, &pColData->type, true);
n += tGetI32v(pBuf + n, &pColData->nVal); n += tGetI32v(pBuf + n, &pColData->nVal, true);
n += tGetI8(pBuf + n, &pColData->flag); n += tGetI8(pBuf + n, &pColData->flag, true);
// bitmap // bitmap
switch (pColData->flag) { switch (pColData->flag) {
@ -3165,7 +3349,7 @@ int32_t tGetColData(uint8_t *pBuf, SColData *pColData) {
pColData->aOffset = (int32_t *)(pBuf + n); pColData->aOffset = (int32_t *)(pBuf + n);
n += (pColData->nVal << 2); n += (pColData->nVal << 2);
n += tGetI32v(pBuf + n, &pColData->nData); n += tGetI32v(pBuf + n, &pColData->nData, true);
pColData->pData = pBuf + n; pColData->pData = pBuf + n;
n += pColData->nData; n += pColData->nData;
} else { } else {

44
source/dnode/vnode/src/inc/tsdb.h
View File

@ -75,6 +75,7 @@ typedef struct SBlkInfo SBlkInfo;
typedef struct STsdbDataIter2 STsdbDataIter2; typedef struct STsdbDataIter2 STsdbDataIter2;
typedef struct STsdbFilterInfo STsdbFilterInfo; typedef struct STsdbFilterInfo STsdbFilterInfo;
typedef struct STFileSystem STFileSystem; typedef struct STFileSystem STFileSystem;
typedef struct STsdbRowKey STsdbRowKey;
#define TSDBROW_ROW_FMT ((int8_t)0x0) #define TSDBROW_ROW_FMT ((int8_t)0x0)
#define TSDBROW_COL_FMT ((int8_t)0x1) #define TSDBROW_COL_FMT ((int8_t)0x1)
@ -163,19 +164,20 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs);
#define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA))
#define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX]) #define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX])
int32_t tBlockDataCreate(SBlockData *pBlockData); int32_t tBlockDataCreate(SBlockData *pBlockData);
void tBlockDataDestroy(SBlockData *pBlockData); void tBlockDataDestroy(SBlockData *pBlockData);
int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid);
void tBlockDataReset(SBlockData *pBlockData); void tBlockDataReset(SBlockData *pBlockData);
int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, int8_t cflag, SColData **ppColData);
int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid); int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema);
int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid);
void tBlockDataClear(SBlockData *pBlockData); int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid);
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); void tBlockDataClear(SBlockData *pBlockData);
int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], SColData *tBlockDataGetColData(SBlockData *pBlockData, int16_t cid);
int32_t aBufN[]); int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); int32_t aBufN[]);
int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]);
// SDiskDataHdr // SDiskDataHdr
int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr);
int32_t tGetDiskDataHdr(uint8_t *p, void *ph); int32_t tGetDiskDataHdr(uint8_t *p, void *ph);
@ -288,14 +290,6 @@ typedef struct {
int32_t tsdbMerge(void *arg); int32_t tsdbMerge(void *arg);
// tsdbDiskData ==============================================================================================
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder);
int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
uint8_t calcSma);
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder);
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId);
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo);
// tsdbDataIter.c ============================================================================================== // tsdbDataIter.c ==============================================================================================
#define TSDB_MEM_TABLE_DATA_ITER 0 #define TSDB_MEM_TABLE_DATA_ITER 0
#define TSDB_DATA_FILE_DATA_ITER 1 #define TSDB_DATA_FILE_DATA_ITER 1
@ -436,6 +430,11 @@ struct TSDBROW {
}; };
}; };
struct STsdbRowKey {
SRowKey rowkey;
int64_t version;
};
struct SBlockIdx { struct SBlockIdx {
int64_t suid; int64_t suid;
int64_t uid; int64_t uid;
@ -453,7 +452,7 @@ struct SMapData {
struct SBlockCol { struct SBlockCol {
int16_t cid; int16_t cid;
int8_t type; int8_t type;
int8_t smaOn; int8_t cflag;
int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE
int32_t szOrigin; // original column value size (only save for variant data type) int32_t szOrigin; // original column value size (only save for variant data type)
int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL
@ -561,6 +560,7 @@ struct SDiskDataHdr {
int32_t szBlkCol; int32_t szBlkCol;
int32_t nRow; int32_t nRow;
int8_t cmprAlg; int8_t cmprAlg;
int8_t numPrimaryKeyCols;
}; };
struct SDelFile { struct SDelFile {

151
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -247,15 +247,28 @@ _exit:
return code; return code;
} }
int32_t tBlockColAndColumnCmpr(const void *p1, const void *p2) {
const SBlockCol *pBlockCol = (const SBlockCol *)p1;
const STColumn *pColumn = (const STColumn *)p2;
if (pBlockCol->cid < pColumn->colId) {
return -1;
} else if (pBlockCol->cid > pColumn->colId) {
return 1;
} else {
return 0;
}
}
int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData, int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData,
STSchema *pTSchema, int16_t cids[], int32_t ncid) { STSchema *pTSchema, int16_t cids[], int32_t ncid) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t n = 0;
SDiskDataHdr hdr;
SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL];
code = tBlockDataInit(bData, (TABLEID *)record, pTSchema, cids, ncid); // read key part
TSDB_CHECK_CODE(code, lino, _exit);
// uid + version + tskey
code = tRealloc(&reader->config->bufArr[0], record->blockKeySize); code = tRealloc(&reader->config->bufArr[0], record->blockKeySize);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
@ -263,34 +276,125 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
0); 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// hdr // decode header
SDiskDataHdr hdr[1]; n += tGetDiskDataHdr(reader->config->bufArr[0] + n, &hdr);
int32_t size = 0;
size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); tBlockDataReset(bData);
bData->suid = hdr.suid;
bData->uid = hdr.uid;
bData->nRow = hdr.nRow;
ASSERT(hdr->delimiter == TSDB_FILE_DLMT); // decode key part
ASSERT(record->uid == hdr->uid); for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) {
n += tGetBlockCol(reader->config->bufArr[0] + n, &primaryKeyBlockCols[i]);
bData->nRow = hdr->nRow; }
// uid // uid
ASSERT(hdr->uid); if (hdr.uid == 0) {
ASSERT(0);
}
// version // version
code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szVer, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg,
(uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr.nRow, &reader->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
size += hdr->szVer; n += hdr.szVer;
// ts // ts
code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr->cmprAlg, code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg,
(uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr->nRow, &reader->config->bufArr[1]); (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &reader->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
size += hdr->szKey; n += hdr.szKey;
ASSERT(size == record->blockKeySize); // primary key columns
for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) {
SColData *pColData;
code = tBlockDataAddColData(bData, primaryKeyBlockCols[i].cid, primaryKeyBlockCols[i].type,
primaryKeyBlockCols[i].cflag, &pColData);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDecmprColData(reader->config->bufArr[0] + n, &primaryKeyBlockCols[i], hdr.cmprAlg, hdr.nRow, pColData,
&reader->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit);
n += (primaryKeyBlockCols[i].szBitmap + primaryKeyBlockCols[i].szOffset + primaryKeyBlockCols[i].szValue);
}
ASSERT(n == record->blockKeySize);
// regular columns load
bool blockColLoaded = false;
int32_t decodedBufferSize = 0;
SBlockCol blockCol = {.cid = 0};
for (int32_t i = 0; i < ncid; i++) {
SColData *pColData = tBlockDataGetColData(bData, cids[i]);
if (pColData != NULL) continue;
// load the column index if not loaded yet
if (!blockColLoaded) {
if (hdr.szBlkCol > 0) {
code = tRealloc(&reader->config->bufArr[0], hdr.szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize,
reader->config->bufArr[0], hdr.szBlkCol, 0);
TSDB_CHECK_CODE(code, lino, _exit);
}
blockColLoaded = true;
}
// search the column index
for (;;) {
if (blockCol.cid >= cids[i]) {
break;
}
if (decodedBufferSize >= hdr.szBlkCol) {
blockCol.cid = INT16_MAX;
break;
}
decodedBufferSize += tGetBlockCol(reader->config->bufArr[0] + decodedBufferSize, &blockCol);
}
STColumn *pTColumn =
taosbsearch(&blockCol, pTSchema->columns, pTSchema->numOfCols, sizeof(STSchema), tBlockColAndColumnCmpr, TD_EQ);
ASSERT(pTColumn != NULL);
code = tBlockDataAddColData(bData, cids[i], pTColumn->type, pTColumn->flags, &pColData);
TSDB_CHECK_CODE(code, lino, _exit);
// fill the column data
if (blockCol.cid > cids[i]) {
// set as all NONE
for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { // all NONE
code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
TSDB_CHECK_CODE(code, lino, _exit);
}
} else if (blockCol.flag == HAS_NULL) { // all NULL
for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) {
code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type));
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
int32_t size1 = blockCol.szBitmap + blockCol.szOffset + blockCol.szValue;
code = tRealloc(&reader->config->bufArr[1], size1);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA],
record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset,
reader->config->bufArr[1], size1, 0);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDecmprColData(reader->config->bufArr[1], &blockCol, hdr.cmprAlg, hdr.nRow, pColData,
&reader->config->bufArr[2]);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
#if 0
// other columns // other columns
if (bData->nColData > 0) { if (bData->nColData > 0) {
if (hdr->szBlkCol > 0) { if (hdr->szBlkCol > 0) {
@ -386,6 +490,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe
} }
} }
} }
#endif
_exit: _exit:
if (code) { if (code) {
@ -877,7 +982,7 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData
// to .sma file // to .sma file
for (int32_t i = 0; i < bData->nColData; ++i) { for (int32_t i = 0; i < bData->nColData; ++i) {
SColData *colData = bData->aColData + i; SColData *colData = bData->aColData + i;
if ((!colData->smaOn) || ((colData->flag & HAS_VALUE) == 0)) continue; if ((colData->cflag & COL_SMA_ON) == 0 || ((colData->flag & HAS_VALUE) == 0)) continue;
SColumnDataAgg sma[1] = {{.colId = colData->cid}}; SColumnDataAgg sma[1] = {{.colId = colData->cid}};
tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull); tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull);

697
source/dnode/vnode/src/tsdb/tsdbDiskData.c
View File

@ -1,697 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "tsdb.h"
typedef struct SDiskColBuilder SDiskColBuilder;
struct SDiskColBuilder {
int16_t cid;
int8_t type;
uint8_t cmprAlg;
uint8_t calcSma;
int8_t flag;
int32_t nVal;
uint8_t *pBitMap;
int32_t offset;
SCompressor *pOffC;
SCompressor *pValC;
SColumnDataAgg sma;
uint8_t minSet;
uint8_t maxSet;
uint8_t *aBuf[2];
};
// SDiskData ================================================
static int32_t tDiskDataDestroy(SDiskData *pDiskData) {
int32_t code = 0;
pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol);
return code;
}
// SDiskColBuilder ================================================
#define tDiskColBuilderCreate() \
(SDiskColBuilder) { 0 }
static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) {
int32_t code = 0;
tFree(pBuilder->pBitMap);
if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC);
if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC);
for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
tFree(pBuilder->aBuf[iBuf]);
}
return code;
}
static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg,
uint8_t calcSma) {
int32_t code = 0;
pBuilder->cid = cid;
pBuilder->type = type;
pBuilder->cmprAlg = cmprAlg;
pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma;
pBuilder->flag = 0;
pBuilder->nVal = 0;
pBuilder->offset = 0;
if (IS_VAR_DATA_TYPE(type)) {
if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code;
code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg);
if (code) return code;
}
if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code;
code = tCompressStart(pBuilder->pValC, type, cmprAlg);
if (code) return code;
if (pBuilder->calcSma) {
pBuilder->sma = (SColumnDataAgg){.colId = cid};
pBuilder->minSet = 0;
pBuilder->maxSet = 0;
}
return code;
}
static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) {
int32_t code = 0;
ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE);
*pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid,
.type = pBuilder->type,
.smaOn = pBuilder->calcSma,
.flag = pBuilder->flag,
.szOrigin = 0,
.szBitmap = 0,
.szOffset = 0,
.szValue = 0,
.offset = 0},
.pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma};
if (pBuilder->flag == HAS_NULL) return code;
// BITMAP
if (pBuilder->flag != HAS_VALUE) {
int32_t nBit;
if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
nBit = BIT2_SIZE(pBuilder->nVal);
} else {
nBit = BIT1_SIZE(pBuilder->nVal);
}
code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES);
if (code) return code;
code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
if (code) return code;
pDiskCol->bCol.szBitmap =
tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES,
pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES);
pDiskCol->pBit = pBuilder->aBuf[0];
}
// OFFSET
if (IS_VAR_DATA_TYPE(pBuilder->type)) {
code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL);
if (code) return code;
}
// VALUE
if (pBuilder->flag != (HAS_NULL | HAS_NONE)) {
code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin);
if (code) return code;
}
return code;
}
static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
if (IS_VAR_DATA_TYPE(pColVal->type)) {
code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t));
if (code) return code;
pBuilder->offset += pColVal->value.nData;
code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData);
if (code) return code;
} else {
code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes);
if (code) return code;
}
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_VALUE;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_NONE;
return 0;
}
static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) {
pBuilder->flag = HAS_NULL;
return 0;
}
static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
// bit map
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
// value
pBuilder->flag |= HAS_VALUE;
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
pBuilder->flag |= HAS_NULL;
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
pBuilder->flag |= HAS_VALUE;
memset(pBuilder->pBitMap, 0, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
pBuilder->flag |= HAS_NONE;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_VALUE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal));
}
SET_BIT2(pBitMap, pBuilder->nVal, 2);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0});
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
code = tDiskColPutValue(pBuilder, &cv);
if (code) return code;
}
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return code;
}
static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) {
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NONE;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NULL;
int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1);
code = tRealloc(&pBuilder->pBitMap, nBit);
if (code) return code;
memset(pBuilder->pBitMap, 255, nBit);
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NULL;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0);
}
SET_BIT2(pBitMap, pBuilder->nVal, 1);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
pBuilder->flag |= HAS_NONE;
uint8_t *pBitMap = NULL;
code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) {
SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1);
}
SET_BIT2(pBitMap, pBuilder->nVal, 0);
tFree(pBuilder->pBitMap);
pBuilder->pBitMap = pBitMap;
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0);
return tDiskColPutValue(pBuilder, pColVal);
}
static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1));
if (code) return code;
SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1);
return tDiskColPutValue(pBuilder, pColVal);
}
static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = {
{tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0
{tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE
{tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL
{tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE
{tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE
{tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE
{tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL
{tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE
};
// extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet);
static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
int32_t code = 0;
if (pBuilder->calcSma) {
if (COL_VAL_IS_VALUE(pColVal)) {
// tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet);
} else {
pBuilder->sma.numOfNull++;
}
}
if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) {
code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal);
if (code) return code;
}
pBuilder->nVal++;
return code;
}
// SDiskDataBuilder ================================================
int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) {
int32_t code = 0;
*ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder));
if (*ppBuilder == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
return code;
}
void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
if (pBuilder == NULL) return NULL;
if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC);
if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC);
if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC);
if (pBuilder->aBuilder) {
for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
tDiskColBuilderDestroy(pDCBuilder);
}
taosArrayDestroy(pBuilder->aBuilder);
}
for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) {
tFree(pBuilder->aBuf[iBuf]);
}
tDiskDataDestroy(&pBuilder->dd);
taosMemoryFree(pBuilder);
return NULL;
}
int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg,
uint8_t calcSma) {
int32_t code = 0;
ASSERT(pId->suid || pId->uid);
pBuilder->suid = pId->suid;
pBuilder->uid = pId->uid;
pBuilder->nRow = 0;
pBuilder->cmprAlg = cmprAlg;
pBuilder->calcSma = calcSma;
pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX,
.maxUid = INT64_MIN,
.minKey = TSKEY_MAX,
.maxKey = TSKEY_MIN,
.minVer = VERSION_MAX,
.maxVer = VERSION_MIN,
.minTKey = TSDBKEY_MAX,
.maxTKey = TSDBKEY_MIN};
if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code;
code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
if (code) return code;
if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code;
code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg);
if (code) return code;
if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code;
code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg);
if (code) return code;
if (pBuilder->aBuilder == NULL) {
pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder));
if (pBuilder->aBuilder == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
pBuilder->nBuilder = 0;
for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) {
STColumn *pTColumn = &pTSchema->columns[iCol];
if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) {
SDiskColBuilder dc = tDiskColBuilderCreate();
if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder);
code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg,
(calcSma && (pTColumn->flags & COL_SMA_ON)));
if (code) return code;
pBuilder->nBuilder++;
}
return code;
}
int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) {
int32_t code = 0;
pBuilder->suid = 0;
pBuilder->uid = 0;
pBuilder->nRow = 0;
return code;
}
int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) {
int32_t code = 0;
ASSERT(pBuilder->suid || pBuilder->uid);
ASSERT(pId->suid == pBuilder->suid);
TSDBKEY kRow = TSDBROW_KEY(pRow);
if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow;
if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow;
// uid
if (pBuilder->uid && pBuilder->uid != pId->uid) {
ASSERT(pBuilder->suid);
for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) {
code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t));
if (code) return code;
}
pBuilder->uid = 0;
}
if (pBuilder->uid == 0) {
code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t));
if (code) return code;
}
if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid;
if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid;
// version
code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t));
if (code) return code;
if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version;
if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version;
// TSKEY
code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t));
if (code) return code;
if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts;
if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts;
STSDBRowIter iter = {0};
tsdbRowIterOpen(&iter, pRow, pTSchema);
SColVal *pColVal = tsdbRowIterNext(&iter);
for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
while (pColVal && pColVal->cid < pDCBuilder->cid) {
pColVal = tsdbRowIterNext(&iter);
}
if (pColVal && pColVal->cid == pDCBuilder->cid) {
code = tDiskColAddVal(pDCBuilder, pColVal);
if (code) return code;
pColVal = tsdbRowIterNext(&iter);
} else {
code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type));
if (code) return code;
}
}
pBuilder->nRow++;
return code;
}
int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) {
int32_t code = 0;
ASSERT(pBuilder->nRow);
*ppDiskData = NULL;
*ppBlkInfo = NULL;
SDiskData *pDiskData = &pBuilder->dd;
// reset SDiskData
pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT,
.fmtVer = 0,
.suid = pBuilder->suid,
.uid = pBuilder->uid,
.szUid = 0,
.szVer = 0,
.szKey = 0,
.szBlkCol = 0,
.nRow = pBuilder->nRow,
.cmprAlg = pBuilder->cmprAlg};
pDiskData->pUid = NULL;
pDiskData->pVer = NULL;
pDiskData->pKey = NULL;
// UID
if (pBuilder->uid == 0) {
code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL);
if (code) return code;
}
// VERSION
code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL);
if (code) return code;
// TSKEY
code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL);
if (code) return code;
// aDiskCol
if (pDiskData->aDiskCol) {
taosArrayClear(pDiskData->aDiskCol);
} else {
pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol));
if (pDiskData->aDiskCol == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
}
int32_t offset = 0;
for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) {
SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder);
if (pDCBuilder->flag == HAS_NONE) continue;
SDiskCol dCol;
code = tGnrtDiskCol(pDCBuilder, &dCol);
if (code) return code;
dCol.bCol.offset = offset;
offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue;
if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
return code;
}
pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol);
}
*ppDiskData = pDiskData;
*ppBlkInfo = &pBuilder->bi;
return code;
}

12
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -23,16 +23,16 @@ static int32_t tsdbFSToBinary(uint8_t *p, STsdbFS *pFS) {
uint32_t nSet = taosArrayGetSize(pFS->aDFileSet); uint32_t nSet = taosArrayGetSize(pFS->aDFileSet);
// version // version
n += tPutI8(p ? p + n : p, 0); n += tPutI8(p ? p + n : p, 0, true);
// SDelFile // SDelFile
n += tPutI8(p ? p + n : p, hasDel); n += tPutI8(p ? p + n : p, hasDel, true);
if (hasDel) { if (hasDel) {
n += tPutDelFile(p ? p + n : p, pFS->pDelFile); n += tPutDelFile(p ? p + n : p, pFS->pDelFile);
} }
// SArray<SDFileSet> // SArray<SDFileSet>
n += tPutU32v(p ? p + n : p, nSet); n += tPutU32v(p ? p + n : p, nSet, true);
for (uint32_t iSet = 0; iSet < nSet; iSet++) { for (uint32_t iSet = 0; iSet < nSet; iSet++) {
n += tPutDFileSet(p ? p + n : p, (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet)); n += tPutDFileSet(p ? p + n : p, (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet));
} }
@ -45,11 +45,11 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, STsdbFS *pFS) {
int32_t n = 0; int32_t n = 0;
// version // version
n += tGetI8(pData + n, NULL); n += tGetI8(pData + n, NULL, true);
// SDelFile // SDelFile
int8_t hasDel = 0; int8_t hasDel = 0;
n += tGetI8(pData + n, &hasDel); n += tGetI8(pData + n, &hasDel, true);
if (hasDel) { if (hasDel) {
pFS->pDelFile = (SDelFile *)taosMemoryCalloc(1, sizeof(SDelFile)); pFS->pDelFile = (SDelFile *)taosMemoryCalloc(1, sizeof(SDelFile));
if (pFS->pDelFile == NULL) { if (pFS->pDelFile == NULL) {
@ -66,7 +66,7 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, STsdbFS *pFS) {
// aDFileSet // aDFileSet
taosArrayClear(pFS->aDFileSet); taosArrayClear(pFS->aDFileSet);
uint32_t nSet = 0; uint32_t nSet = 0;
n += tGetU32v(pData + n, &nSet); n += tGetU32v(pData + n, &nSet, true);
for (uint32_t iSet = 0; iSet < nSet; iSet++) { for (uint32_t iSet = 0; iSet < nSet; iSet++) {
SDFileSet fSet = {0}; SDFileSet fSet = {0};

84
source/dnode/vnode/src/tsdb/tsdbFile.c
View File

@ -19,9 +19,9 @@
int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pHeadFile->commitID); n += tPutI64v(p ? p + n : p, pHeadFile->commitID, true);
n += tPutI64v(p ? p + n : p, pHeadFile->size); n += tPutI64v(p ? p + n : p, pHeadFile->size, true);
n += tPutI64v(p ? p + n : p, pHeadFile->offset); n += tPutI64v(p ? p + n : p, pHeadFile->offset, true);
return n; return n;
} }
@ -29,9 +29,9 @@ int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pHeadFile->commitID); n += tGetI64v(p + n, &pHeadFile->commitID, true);
n += tGetI64v(p + n, &pHeadFile->size); n += tGetI64v(p + n, &pHeadFile->size, true);
n += tGetI64v(p + n, &pHeadFile->offset); n += tGetI64v(p + n, &pHeadFile->offset, true);
return n; return n;
} }
@ -39,8 +39,8 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) {
int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) { int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pDataFile->commitID); n += tPutI64v(p ? p + n : p, pDataFile->commitID, true);
n += tPutI64v(p ? p + n : p, pDataFile->size); n += tPutI64v(p ? p + n : p, pDataFile->size, true);
return n; return n;
} }
@ -48,8 +48,8 @@ int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) {
static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pDataFile->commitID); n += tGetI64v(p + n, &pDataFile->commitID, true);
n += tGetI64v(p + n, &pDataFile->size); n += tGetI64v(p + n, &pDataFile->size, true);
return n; return n;
} }
@ -57,9 +57,9 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pSttFile->commitID); n += tPutI64v(p ? p + n : p, pSttFile->commitID, true);
n += tPutI64v(p ? p + n : p, pSttFile->size); n += tPutI64v(p ? p + n : p, pSttFile->size, true);
n += tPutI64v(p ? p + n : p, pSttFile->offset); n += tPutI64v(p ? p + n : p, pSttFile->offset, true);
return n; return n;
} }
@ -67,9 +67,9 @@ int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) {
static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) { static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pSttFile->commitID); n += tGetI64v(p + n, &pSttFile->commitID, true);
n += tGetI64v(p + n, &pSttFile->size); n += tGetI64v(p + n, &pSttFile->size, true);
n += tGetI64v(p + n, &pSttFile->offset); n += tGetI64v(p + n, &pSttFile->offset, true);
return n; return n;
} }
@ -77,8 +77,8 @@ static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) {
int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) { int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pSmaFile->commitID); n += tPutI64v(p ? p + n : p, pSmaFile->commitID, true);
n += tPutI64v(p ? p + n : p, pSmaFile->size); n += tPutI64v(p ? p + n : p, pSmaFile->size, true);
return n; return n;
} }
@ -86,18 +86,18 @@ int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) { static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pSmaFile->commitID); n += tGetI64v(p + n, &pSmaFile->commitID, true);
n += tGetI64v(p + n, &pSmaFile->size); n += tGetI64v(p + n, &pSmaFile->size, true);
return n; return n;
} }
// EXPOSED APIS ================================================== // EXPOSED APIS ==================================================
static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { static char *getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) {
const char* p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); const char *p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did);
int32_t len = strlen(p1); int32_t len = strlen(p1);
char* p = memcpy(fname, p1, len); char *p = memcpy(fname, p1, len);
p += len; p += len;
*(p++) = TD_DIRSEP[0]; *(p++) = TD_DIRSEP[0];
@ -125,25 +125,25 @@ static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t
} }
void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) { void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname);
memcpy(p, ".head", 5); memcpy(p, ".head", 5);
p[5] = 0; p[5] = 0;
} }
void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) { void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname);
memcpy(p, ".data", 5); memcpy(p, ".data", 5);
p[5] = 0; p[5] = 0;
} }
void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) { void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname);
memcpy(p, ".stt", 4); memcpy(p, ".stt", 4);
p[4] = 0; p[4] = 0;
} }
void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) {
char* p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); char *p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname);
memcpy(p, ".sma", 4); memcpy(p, ".sma", 4);
p[4] = 0; p[4] = 0;
} }
@ -221,9 +221,9 @@ _err:
int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t n = 0; int32_t n = 0;
n += tPutI32v(p ? p + n : p, pSet->diskId.level); n += tPutI32v(p ? p + n : p, pSet->diskId.level, true);
n += tPutI32v(p ? p + n : p, pSet->diskId.id); n += tPutI32v(p ? p + n : p, pSet->diskId.id, true);
n += tPutI32v(p ? p + n : p, pSet->fid); n += tPutI32v(p ? p + n : p, pSet->fid, true);
// data // data
n += tPutHeadFile(p ? p + n : p, pSet->pHeadF); n += tPutHeadFile(p ? p + n : p, pSet->pHeadF);
@ -231,7 +231,7 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF);
// stt // stt
n += tPutU8(p ? p + n : p, pSet->nSttF); n += tPutU8(p ? p + n : p, pSet->nSttF, true);
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
n += tPutSttFile(p ? p + n : p, pSet->aSttF[iStt]); n += tPutSttFile(p ? p + n : p, pSet->aSttF[iStt]);
} }
@ -242,9 +242,9 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
int32_t n = 0; int32_t n = 0;
n += tGetI32v(p + n, &pSet->diskId.level); n += tGetI32v(p + n, &pSet->diskId.level, true);
n += tGetI32v(p + n, &pSet->diskId.id); n += tGetI32v(p + n, &pSet->diskId.id, true);
n += tGetI32v(p + n, &pSet->fid); n += tGetI32v(p + n, &pSet->fid, true);
// head // head
pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile));
@ -271,7 +271,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
n += tGetSmaFile(p + n, pSet->pSmaF); n += tGetSmaFile(p + n, pSet->pSmaF);
// stt // stt
n += tGetU8(p + n, &pSet->nSttF); n += tGetU8(p + n, &pSet->nSttF, true);
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile)); pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile));
if (pSet->aSttF[iStt] == NULL) { if (pSet->aSttF[iStt] == NULL) {
@ -298,9 +298,9 @@ void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) {
int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) { int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) {
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pDelFile->commitID); n += tPutI64v(p ? p + n : p, pDelFile->commitID, true);
n += tPutI64v(p ? p + n : p, pDelFile->size); n += tPutI64v(p ? p + n : p, pDelFile->size, true);
n += tPutI64v(p ? p + n : p, pDelFile->offset); n += tPutI64v(p ? p + n : p, pDelFile->offset, true);
return n; return n;
} }
@ -308,9 +308,9 @@ int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) {
int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile) { int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile) {
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pDelFile->commitID); n += tGetI64v(p + n, &pDelFile->commitID, true);
n += tGetI64v(p + n, &pDelFile->size); n += tGetI64v(p + n, &pDelFile->size, true);
n += tGetI64v(p + n, &pDelFile->offset); n += tGetI64v(p + n, &pDelFile->offset, true);
return n; return n;
} }

150
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -191,35 +191,156 @@ _exit:
return code; return code;
} }
extern int32_t tBlockColAndColumnCmpr(const void *p1, const void *p2);
int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData, int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData,
STSchema *pTSchema, int16_t cids[], int32_t ncid) { STSchema *pTSchema, int16_t cids[], int32_t ncid) {
int32_t code = 0; int32_t code = 0;
int32_t lino = 0; int32_t lino = 0;
int32_t n = 0;
SDiskDataHdr hdr;
SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL];
TABLEID tbid = {.suid = sttBlk->suid}; // load key part
if (tbid.suid == 0) {
tbid.uid = sttBlk->minUid;
} else {
tbid.uid = 0;
}
code = tBlockDataInit(bData, &tbid, pTSchema, cids, ncid);
TSDB_CHECK_CODE(code, lino, _exit);
// uid + version + tskey
code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey); code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0); code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
// decode header
n += tGetDiskDataHdr(reader->config->bufArr[0] + n, &hdr);
ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
// set data container
tBlockDataReset(bData);
bData->suid = hdr.suid;
bData->uid = (sttBlk->suid == 0) ? sttBlk->minUid : 0;
bData->nRow = hdr.nRow;
// decode primary key column indices
for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) {
n += tGetBlockCol(reader->config->bufArr[0] + n, primaryKeyBlockCols + i);
}
// uid
if (hdr.uid == 0) {
ASSERT(hdr.szUid);
code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg,
(uint8_t **)&bData->aUid, sizeof(int64_t) * hdr.nRow, &reader->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
ASSERT(hdr.szUid == 0);
}
n += hdr.szUid;
// version
code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg,
(uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr.nRow, &reader->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit);
n += hdr.szVer;
// ts
code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg,
(uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &reader->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit);
n += hdr.szKey;
// decode primary key columns
for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) {
SColData *pColData;
code = tBlockDataAddColData(bData, primaryKeyBlockCols[i].cid, primaryKeyBlockCols[i].type,
primaryKeyBlockCols[i].cflag, &pColData);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDecmprColData(reader->config->bufArr[0] + n, &primaryKeyBlockCols[i], hdr.cmprAlg, hdr.nRow, pColData,
&reader->config->bufArr[1]);
TSDB_CHECK_CODE(code, lino, _exit);
n += (primaryKeyBlockCols[i].szBitmap + primaryKeyBlockCols[i].szOffset + primaryKeyBlockCols[i].szValue);
}
ASSERT(n == sttBlk->bInfo.szKey);
// regular columns load
bool blockColLoaded = false;
int32_t decodedBufferSize = 0;
SBlockCol blockCol = {.cid = 0};
for (int32_t i = 0; i < ncid; i++) {
SColData *pColData = tBlockDataGetColData(bData, cids[i]);
if (pColData != NULL) continue;
// load the column index if not loaded yet
if (!blockColLoaded) {
if (hdr.szBlkCol > 0) {
code = tRealloc(&reader->config->bufArr[0], hdr.szBlkCol);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, reader->config->bufArr[0],
hdr.szBlkCol, 0);
TSDB_CHECK_CODE(code, lino, _exit);
}
blockColLoaded = true;
}
// search the column index
for (;;) {
if (blockCol.cid >= cids[i]) {
break;
}
if (decodedBufferSize >= hdr.szBlkCol) {
blockCol.cid = INT16_MAX;
break;
}
decodedBufferSize += tGetBlockCol(reader->config->bufArr[0] + decodedBufferSize, &blockCol);
}
STColumn *pTColumn =
taosbsearch(&blockCol, pTSchema->columns, pTSchema->numOfCols, sizeof(STSchema), tBlockColAndColumnCmpr, TD_EQ);
ASSERT(pTColumn != NULL);
code = tBlockDataAddColData(bData, cids[i], pTColumn->type, pTColumn->flags, &pColData);
TSDB_CHECK_CODE(code, lino, _exit);
// fill the column data
if (blockCol.cid > cids[i]) {
// set as all NONE
for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { // all NONE
code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
TSDB_CHECK_CODE(code, lino, _exit);
}
} else if (blockCol.flag == HAS_NULL) { // all NULL
for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) {
code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type));
TSDB_CHECK_CODE(code, lino, _exit);
}
} else {
int32_t size1 = blockCol.szBitmap + blockCol.szOffset + blockCol.szValue;
code = tRealloc(&reader->config->bufArr[1], size1);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset,
reader->config->bufArr[1], size1, 0);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbDecmprColData(reader->config->bufArr[1], &blockCol, hdr.cmprAlg, hdr.nRow, pColData,
&reader->config->bufArr[2]);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
#if 0
// hdr // hdr
SDiskDataHdr hdr[1]; SDiskDataHdr hdr[1];
int32_t size = 0; int32_t size = 0;
size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr);
ASSERT(hdr->delimiter == TSDB_FILE_DLMT);
bData->nRow = hdr->nRow; bData->nRow = hdr->nRow;
bData->uid = hdr->uid; bData->uid = hdr->uid;
@ -307,6 +428,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *
} }
} }
} }
#endif
_exit: _exit:
if (code) { if (code) {

499
source/dnode/vnode/src/tsdb/tsdbUtil.c
View File

@ -123,15 +123,15 @@ _exit:
int32_t tPutMapData(uint8_t *p, SMapData *pMapData) { int32_t tPutMapData(uint8_t *p, SMapData *pMapData) {
int32_t n = 0; int32_t n = 0;
n += tPutI32v(p ? p + n : p, pMapData->nItem); n += tPutI32v(p ? p + n : p, pMapData->nItem, true);
if (pMapData->nItem) { if (pMapData->nItem) {
int32_t lOffset = 0; int32_t lOffset = 0;
for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) { for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset); n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset, true);
lOffset = pMapData->aOffset[iItem]; lOffset = pMapData->aOffset[iItem];
} }
n += tPutI32v(p ? p + n : p, pMapData->nData); n += tPutI32v(p ? p + n : p, pMapData->nData, true);
if (p) { if (p) {
memcpy(p + n, pMapData->pData, pMapData->nData); memcpy(p + n, pMapData->pData, pMapData->nData);
} }
@ -147,18 +147,18 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) {
tMapDataReset(pMapData); tMapDataReset(pMapData);
n += tGetI32v(p + n, &pMapData->nItem); n += tGetI32v(p + n, &pMapData->nItem, true);
if (pMapData->nItem) { if (pMapData->nItem) {
if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1; if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1;
int32_t lOffset = 0; int32_t lOffset = 0;
for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) { for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) {
n += tGetI32v(p + n, &pMapData->aOffset[iItem]); n += tGetI32v(p + n, &pMapData->aOffset[iItem], true);
pMapData->aOffset[iItem] += lOffset; pMapData->aOffset[iItem] += lOffset;
lOffset = pMapData->aOffset[iItem]; lOffset = pMapData->aOffset[iItem];
} }
n += tGetI32v(p + n, &pMapData->nData); n += tGetI32v(p + n, &pMapData->nData, true);
if (tRealloc(&pMapData->pData, pMapData->nData)) return -1; if (tRealloc(&pMapData->pData, pMapData->nData)) return -1;
memcpy(pMapData->pData, p + n, pMapData->nData); memcpy(pMapData->pData, p + n, pMapData->nData);
n += pMapData->nData; n += pMapData->nData;
@ -192,10 +192,10 @@ int32_t tPutBlockIdx(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlockIdx *pBlockIdx = (SBlockIdx *)ph; SBlockIdx *pBlockIdx = (SBlockIdx *)ph;
n += tPutI64(p ? p + n : p, pBlockIdx->suid); n += tPutI64(p ? p + n : p, pBlockIdx->suid, true);
n += tPutI64(p ? p + n : p, pBlockIdx->uid); n += tPutI64(p ? p + n : p, pBlockIdx->uid, true);
n += tPutI64v(p ? p + n : p, pBlockIdx->offset); n += tPutI64v(p ? p + n : p, pBlockIdx->offset, true);
n += tPutI64v(p ? p + n : p, pBlockIdx->size); n += tPutI64v(p ? p + n : p, pBlockIdx->size, true);
return n; return n;
} }
@ -204,10 +204,10 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlockIdx *pBlockIdx = (SBlockIdx *)ph; SBlockIdx *pBlockIdx = (SBlockIdx *)ph;
n += tGetI64(p + n, &pBlockIdx->suid); n += tGetI64(p + n, &pBlockIdx->suid, true);
n += tGetI64(p + n, &pBlockIdx->uid); n += tGetI64(p + n, &pBlockIdx->uid, true);
n += tGetI64v(p + n, &pBlockIdx->offset); n += tGetI64v(p + n, &pBlockIdx->offset, true);
n += tGetI64v(p + n, &pBlockIdx->size); n += tGetI64v(p + n, &pBlockIdx->size, true);
return n; return n;
} }
@ -259,23 +259,23 @@ int32_t tPutDataBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SDataBlk *pDataBlk = (SDataBlk *)ph; SDataBlk *pDataBlk = (SDataBlk *)ph;
n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version); n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version, true);
n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts); n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts, true);
n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version); n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version, true);
n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts); n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts, true);
n += tPutI64v(p ? p + n : p, pDataBlk->minVer); n += tPutI64v(p ? p + n : p, pDataBlk->minVer, true);
n += tPutI64v(p ? p + n : p, pDataBlk->maxVer); n += tPutI64v(p ? p + n : p, pDataBlk->maxVer, true);
n += tPutI32v(p ? p + n : p, pDataBlk->nRow); n += tPutI32v(p ? p + n : p, pDataBlk->nRow, true);
n += tPutI8(p ? p + n : p, pDataBlk->hasDup); n += tPutI8(p ? p + n : p, pDataBlk->hasDup, true);
n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock); n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock, true);
for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset); n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset, true);
n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock); n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock, true);
n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey); n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey, true);
} }
if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset); n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset, true);
n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size); n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size, true);
} }
return n; return n;
@ -285,23 +285,23 @@ int32_t tGetDataBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SDataBlk *pDataBlk = (SDataBlk *)ph; SDataBlk *pDataBlk = (SDataBlk *)ph;
n += tGetI64v(p + n, &pDataBlk->minKey.version); n += tGetI64v(p + n, &pDataBlk->minKey.version, true);
n += tGetI64v(p + n, &pDataBlk->minKey.ts); n += tGetI64v(p + n, &pDataBlk->minKey.ts, true);
n += tGetI64v(p + n, &pDataBlk->maxKey.version); n += tGetI64v(p + n, &pDataBlk->maxKey.version, true);
n += tGetI64v(p + n, &pDataBlk->maxKey.ts); n += tGetI64v(p + n, &pDataBlk->maxKey.ts, true);
n += tGetI64v(p + n, &pDataBlk->minVer); n += tGetI64v(p + n, &pDataBlk->minVer, true);
n += tGetI64v(p + n, &pDataBlk->maxVer); n += tGetI64v(p + n, &pDataBlk->maxVer, true);
n += tGetI32v(p + n, &pDataBlk->nRow); n += tGetI32v(p + n, &pDataBlk->nRow, true);
n += tGetI8(p + n, &pDataBlk->hasDup); n += tGetI8(p + n, &pDataBlk->hasDup, true);
n += tGetI8(p + n, &pDataBlk->nSubBlock); n += tGetI8(p + n, &pDataBlk->nSubBlock, true);
for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset); n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset, true);
n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock); n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock, true);
n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey); n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey, true);
} }
if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) {
n += tGetI64v(p + n, &pDataBlk->smaInfo.offset); n += tGetI64v(p + n, &pDataBlk->smaInfo.offset, true);
n += tGetI32v(p + n, &pDataBlk->smaInfo.size); n += tGetI32v(p + n, &pDataBlk->smaInfo.size, true);
} else { } else {
pDataBlk->smaInfo.offset = 0; pDataBlk->smaInfo.offset = 0;
pDataBlk->smaInfo.size = 0; pDataBlk->smaInfo.size = 0;
@ -335,17 +335,17 @@ int32_t tPutSttBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SSttBlk *pSttBlk = (SSttBlk *)ph; SSttBlk *pSttBlk = (SSttBlk *)ph;
n += tPutI64(p ? p + n : p, pSttBlk->suid); n += tPutI64(p ? p + n : p, pSttBlk->suid, true);
n += tPutI64(p ? p + n : p, pSttBlk->minUid); n += tPutI64(p ? p + n : p, pSttBlk->minUid, true);
n += tPutI64(p ? p + n : p, pSttBlk->maxUid); n += tPutI64(p ? p + n : p, pSttBlk->maxUid, true);
n += tPutI64v(p ? p + n : p, pSttBlk->minKey); n += tPutI64v(p ? p + n : p, pSttBlk->minKey, true);
n += tPutI64v(p ? p + n : p, pSttBlk->maxKey); n += tPutI64v(p ? p + n : p, pSttBlk->maxKey, true);
n += tPutI64v(p ? p + n : p, pSttBlk->minVer); n += tPutI64v(p ? p + n : p, pSttBlk->minVer, true);
n += tPutI64v(p ? p + n : p, pSttBlk->maxVer); n += tPutI64v(p ? p + n : p, pSttBlk->maxVer, true);
n += tPutI32v(p ? p + n : p, pSttBlk->nRow); n += tPutI32v(p ? p + n : p, pSttBlk->nRow, true);
n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset); n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset, true);
n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock); n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock, true);
n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey); n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey, true);
return n; return n;
} }
@ -354,17 +354,17 @@ int32_t tGetSttBlk(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SSttBlk *pSttBlk = (SSttBlk *)ph; SSttBlk *pSttBlk = (SSttBlk *)ph;
n += tGetI64(p + n, &pSttBlk->suid); n += tGetI64(p + n, &pSttBlk->suid, true);
n += tGetI64(p + n, &pSttBlk->minUid); n += tGetI64(p + n, &pSttBlk->minUid, true);
n += tGetI64(p + n, &pSttBlk->maxUid); n += tGetI64(p + n, &pSttBlk->maxUid, true);
n += tGetI64v(p + n, &pSttBlk->minKey); n += tGetI64v(p + n, &pSttBlk->minKey, true);
n += tGetI64v(p + n, &pSttBlk->maxKey); n += tGetI64v(p + n, &pSttBlk->maxKey, true);
n += tGetI64v(p + n, &pSttBlk->minVer); n += tGetI64v(p + n, &pSttBlk->minVer, true);
n += tGetI64v(p + n, &pSttBlk->maxVer); n += tGetI64v(p + n, &pSttBlk->maxVer, true);
n += tGetI32v(p + n, &pSttBlk->nRow); n += tGetI32v(p + n, &pSttBlk->nRow, true);
n += tGetI64v(p + n, &pSttBlk->bInfo.offset); n += tGetI64v(p + n, &pSttBlk->bInfo.offset, true);
n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock); n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock, true);
n += tGetI32v(p + n, &pSttBlk->bInfo.szKey); n += tGetI32v(p + n, &pSttBlk->bInfo.szKey, true);
return n; return n;
} }
@ -376,26 +376,26 @@ int32_t tPutBlockCol(uint8_t *p, void *ph) {
ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE));
n += tPutI16v(p ? p + n : p, pBlockCol->cid); n += tPutI16v(p ? p + n : p, pBlockCol->cid, true);
n += tPutI8(p ? p + n : p, pBlockCol->type); n += tPutI8(p ? p + n : p, pBlockCol->type, true);
n += tPutI8(p ? p + n : p, pBlockCol->smaOn); n += tPutI8(p ? p + n : p, pBlockCol->cflag, true);
n += tPutI8(p ? p + n : p, pBlockCol->flag); n += tPutI8(p ? p + n : p, pBlockCol->flag, true);
n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin, true);
if (pBlockCol->flag != HAS_NULL) { if (pBlockCol->flag != HAS_NULL) {
if (pBlockCol->flag != HAS_VALUE) { if (pBlockCol->flag != HAS_VALUE) {
n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap, true);
} }
if (IS_VAR_DATA_TYPE(pBlockCol->type)) { if (IS_VAR_DATA_TYPE(pBlockCol->type)) {
n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); n += tPutI32v(p ? p + n : p, pBlockCol->szOffset, true);
} }
if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) {
n += tPutI32v(p ? p + n : p, pBlockCol->szValue); n += tPutI32v(p ? p + n : p, pBlockCol->szValue, true);
} }
n += tPutI32v(p ? p + n : p, pBlockCol->offset); n += tPutI32v(p ? p + n : p, pBlockCol->offset, true);
} }
_exit: _exit:
@ -406,11 +406,11 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SBlockCol *pBlockCol = (SBlockCol *)ph; SBlockCol *pBlockCol = (SBlockCol *)ph;
n += tGetI16v(p + n, &pBlockCol->cid); n += tGetI16v(p + n, &pBlockCol->cid, true);
n += tGetI8(p + n, &pBlockCol->type); n += tGetI8(p + n, &pBlockCol->type, true);
n += tGetI8(p + n, &pBlockCol->smaOn); n += tGetI8(p + n, &pBlockCol->cflag, true);
n += tGetI8(p + n, &pBlockCol->flag); n += tGetI8(p + n, &pBlockCol->flag, true);
n += tGetI32v(p + n, &pBlockCol->szOrigin); n += tGetI32v(p + n, &pBlockCol->szOrigin, true);
ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE));
@ -421,18 +421,18 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) {
if (pBlockCol->flag != HAS_NULL) { if (pBlockCol->flag != HAS_NULL) {
if (pBlockCol->flag != HAS_VALUE) { if (pBlockCol->flag != HAS_VALUE) {
n += tGetI32v(p + n, &pBlockCol->szBitmap); n += tGetI32v(p + n, &pBlockCol->szBitmap, true);
} }
if (IS_VAR_DATA_TYPE(pBlockCol->type)) { if (IS_VAR_DATA_TYPE(pBlockCol->type)) {
n += tGetI32v(p + n, &pBlockCol->szOffset); n += tGetI32v(p + n, &pBlockCol->szOffset, true);
} }
if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) {
n += tGetI32v(p + n, &pBlockCol->szValue); n += tGetI32v(p + n, &pBlockCol->szValue, true);
} }
n += tGetI32v(p + n, &pBlockCol->offset); n += tGetI32v(p + n, &pBlockCol->offset, true);
} }
return n; return n;
@ -472,10 +472,10 @@ int32_t tPutDelIdx(uint8_t *p, void *ph) {
SDelIdx *pDelIdx = (SDelIdx *)ph; SDelIdx *pDelIdx = (SDelIdx *)ph;
int32_t n = 0; int32_t n = 0;
n += tPutI64(p ? p + n : p, pDelIdx->suid); n += tPutI64(p ? p + n : p, pDelIdx->suid, true);
n += tPutI64(p ? p + n : p, pDelIdx->uid); n += tPutI64(p ? p + n : p, pDelIdx->uid, true);
n += tPutI64v(p ? p + n : p, pDelIdx->offset); n += tPutI64v(p ? p + n : p, pDelIdx->offset, true);
n += tPutI64v(p ? p + n : p, pDelIdx->size); n += tPutI64v(p ? p + n : p, pDelIdx->size, true);
return n; return n;
} }
@ -484,10 +484,10 @@ int32_t tGetDelIdx(uint8_t *p, void *ph) {
SDelIdx *pDelIdx = (SDelIdx *)ph; SDelIdx *pDelIdx = (SDelIdx *)ph;
int32_t n = 0; int32_t n = 0;
n += tGetI64(p + n, &pDelIdx->suid); n += tGetI64(p + n, &pDelIdx->suid, true);
n += tGetI64(p + n, &pDelIdx->uid); n += tGetI64(p + n, &pDelIdx->uid, true);
n += tGetI64v(p + n, &pDelIdx->offset); n += tGetI64v(p + n, &pDelIdx->offset, true);
n += tGetI64v(p + n, &pDelIdx->size); n += tGetI64v(p + n, &pDelIdx->size, true);
return n; return n;
} }
@ -497,9 +497,9 @@ int32_t tPutDelData(uint8_t *p, void *ph) {
SDelData *pDelData = (SDelData *)ph; SDelData *pDelData = (SDelData *)ph;
int32_t n = 0; int32_t n = 0;
n += tPutI64v(p ? p + n : p, pDelData->version); n += tPutI64v(p ? p + n : p, pDelData->version, true);
n += tPutI64(p ? p + n : p, pDelData->sKey); n += tPutI64(p ? p + n : p, pDelData->sKey, true);
n += tPutI64(p ? p + n : p, pDelData->eKey); n += tPutI64(p ? p + n : p, pDelData->eKey, true);
return n; return n;
} }
@ -508,9 +508,9 @@ int32_t tGetDelData(uint8_t *p, void *ph) {
SDelData *pDelData = (SDelData *)ph; SDelData *pDelData = (SDelData *)ph;
int32_t n = 0; int32_t n = 0;
n += tGetI64v(p + n, &pDelData->version); n += tGetI64v(p + n, &pDelData->version, true);
n += tGetI64(p + n, &pDelData->sKey); n += tGetI64(p + n, &pDelData->sKey, true);
n += tGetI64(p + n, &pDelData->eKey); n += tGetI64(p + n, &pDelData->eKey, true);
return n; return n;
} }
@ -575,9 +575,7 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
if (pRow->type == TSDBROW_ROW_FMT) { if (pRow->type == TSDBROW_ROW_FMT) {
tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal);
} else if (pRow->type == TSDBROW_COL_FMT) { } else if (pRow->type == TSDBROW_COL_FMT) {
SColData *pColData; SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId);
tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData);
if (pColData) { if (pColData) {
tColDataGetValue(pColData, pRow->iRow, pColVal); tColDataGetValue(pColData, pRow->iRow, pColVal);
@ -589,8 +587,50 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal *
} }
} }
static void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key) {
if (row->type == TSDBROW_ROW_FMT) {
key->version = row->version;
tRowGetKey(row->pTSRow, &key->rowkey);
} else {
key->version = row->pBlockData->aVersion[row->iRow];
key->rowkey.ts = row->pBlockData->aTSKEY[row->iRow];
key->rowkey.numOfKeys = 0;
for (int32_t i = 0; i < row->pBlockData->nColData; i++) {
SColData *pColData = &row->pBlockData->aColData[i];
if (pColData->cflag & COL_IS_KEY) {
SColVal cv;
tColDataGetValue(pColData, row->iRow, &cv);
key->rowkey.keys[key->rowkey.numOfKeys].type = pColData->type;
key->rowkey.keys[key->rowkey.numOfKeys].value = cv.value;
key->rowkey.numOfKeys++;
} else {
break;
}
}
}
}
int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2) {
int32_t c = tRowKeyCmpr(&key1->rowkey, &key2->rowkey);
if (c) {
return c;
}
if (key1->version < key2->version) {
return -1;
} else if (key1->version > key2->version) {
return 1;
}
return 0;
}
int32_t tsdbRowCmprFn(const void *p1, const void *p2) { int32_t tsdbRowCmprFn(const void *p1, const void *p2) {
return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); STsdbRowKey key1, key2;
tsdbRowGetKey((TSDBROW *)p1, &key1);
tsdbRowGetKey((TSDBROW *)p2, &key2);
return tsdbRowKeyCmpr(&key1, &key2);
} }
// STSDBRowIter ====================================================== // STSDBRowIter ======================================================
@ -1126,8 +1166,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
continue; continue;
} }
tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, pTColumn->flags);
(pTColumn->flags & COL_SMA_ON) ? 1 : 0);
iColumn++; iColumn++;
pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL;
@ -1138,8 +1177,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema,
for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
STColumn *pTColumn = &pTSchema->columns[iColData + 1]; STColumn *pTColumn = &pTSchema->columns[iColData + 1];
tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, pTColumn->flags);
(pTColumn->flags & COL_SMA_ON) ? 1 : 0);
} }
} }
@ -1167,6 +1205,23 @@ void tBlockDataClear(SBlockData *pBlockData) {
} }
} }
int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, int8_t cflag, SColData **ppColData) {
ASSERT(pBlockData->nColData == 0 || pBlockData->aColData[pBlockData->nColData - 1].cid < cid);
SColData *newColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SColData) * (pBlockData->nColData + 1));
if (newColData == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pBlockData->aColData = newColData;
pBlockData->nColData++;
*ppColData = &pBlockData->aColData[pBlockData->nColData - 1];
tColDataInit(*ppColData, cid, type, cflag);
return 0;
}
/* flag > 0: forward update /* flag > 0: forward update
* flag == 0: insert * flag == 0: insert
* flag < 0: backward update * flag < 0: backward update
@ -1287,7 +1342,7 @@ int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS
} }
} }
void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) { SColData *tBlockDataGetColData(SBlockData *pBlockData, int16_t cid) {
ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID);
int32_t lidx = 0; int32_t lidx = 0;
int32_t ridx = pBlockData->nColData - 1; int32_t ridx = pBlockData->nColData - 1;
@ -1298,8 +1353,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD
int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1);
if (c == 0) { if (c == 0) {
*ppColData = pColData; return pColData;
return;
} else if (c < 0) { } else if (c < 0) {
lidx = midx + 1; lidx = midx + 1;
} else { } else {
@ -1307,42 +1361,49 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD
} }
} }
*ppColData = NULL; return NULL;
} }
int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[],
int32_t aBufN[]) { int32_t aBufSize[]) {
int32_t code = 0; int32_t code = 0;
SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL];
SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, SDiskDataHdr hdr = {
.fmtVer = 0, .delimiter = TSDB_FILE_DLMT,
.suid = pBlockData->suid, .fmtVer = 0,
.uid = pBlockData->uid, .suid = pBlockData->suid,
.nRow = pBlockData->nRow, .uid = pBlockData->uid,
.cmprAlg = cmprAlg}; .nRow = pBlockData->nRow,
.cmprAlg = cmprAlg,
};
// encode ================= // encode =================
// columns AND SBlockCol // columns AND SBlockCol
aBufN[0] = 0; aBufSize[0] = 0;
for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
ASSERT(pColData->flag); ASSERT(pColData->flag);
if (pColData->flag == HAS_NONE) continue; if (pColData->flag == HAS_NONE) continue;
if (pColData->cflag & COL_IS_KEY) {
ASSERT(pColData->flag == HAS_VALUE);
hdr.fmtVer = 1;
continue;
}
SBlockCol blockCol = {.cid = pColData->cid, SBlockCol blockCol = {.cid = pColData->cid,
.type = pColData->type, .type = pColData->type,
.smaOn = pColData->smaOn, .cflag = pColData->cflag,
.flag = pColData->flag, .flag = pColData->flag,
.szOrigin = pColData->nData}; .szOrigin = pColData->nData};
if (pColData->flag != HAS_NULL) { if (pColData->flag != HAS_NULL) {
code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], aBufN[0], &aBuf[2]); code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], aBufSize[0], &aBuf[2]);
if (code) goto _exit; if (code) goto _exit;
blockCol.offset = aBufN[0]; blockCol.offset = aBufSize[0];
aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; aBufSize[0] = aBufSize[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue;
} }
code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol));
@ -1351,46 +1412,74 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
} }
// SBlockCol // SBlockCol
aBufN[1] = hdr.szBlkCol; aBufSize[1] = hdr.szBlkCol;
// uid + version + tskey aBufSize[2] = 0;
aBufN[2] = 0; // uid
if (pBlockData->uid == 0) { if (pBlockData->uid == 0) {
code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg,
&aBuf[2], aBufN[2], &hdr.szUid, &aBuf[3]); &aBuf[2], aBufSize[2], &hdr.szUid, &aBuf[3]);
if (code) goto _exit; if (code) goto _exit;
} }
aBufN[2] += hdr.szUid; aBufSize[2] += hdr.szUid;
// version
code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT,
cmprAlg, &aBuf[2], aBufN[2], &hdr.szVer, &aBuf[3]); cmprAlg, &aBuf[2], aBufSize[2], &hdr.szVer, &aBuf[3]);
if (code) goto _exit; if (code) goto _exit;
aBufN[2] += hdr.szVer; aBufSize[2] += hdr.szVer;
// ts
code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP,
cmprAlg, &aBuf[2], aBufN[2], &hdr.szKey, &aBuf[3]); cmprAlg, &aBuf[2], aBufSize[2], &hdr.szKey, &aBuf[3]);
if (code) goto _exit; if (code) goto _exit;
aBufN[2] += hdr.szKey; aBufSize[2] += hdr.szKey;
// primary key columns
if (hdr.fmtVer == 1) {
for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
if ((pColData->cflag & COL_IS_KEY) == 0) break;
SBlockCol *pBlockCol = &primaryKeyBlockCols[hdr.numPrimaryKeyCols];
pBlockCol->cid = pColData->cid;
pBlockCol->type = pColData->type;
pBlockCol->cflag = pColData->cflag;
pBlockCol->flag = pColData->flag;
pBlockCol->szOrigin = pColData->nData;
code = tsdbCmprColData(pColData, cmprAlg, pBlockCol, &aBuf[2], aBufSize[2], &aBuf[3]);
if (code) goto _exit;
aBufSize[2] = aBufSize[2] + pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue;
hdr.numPrimaryKeyCols++;
}
}
// hdr // hdr
aBufN[3] = tPutDiskDataHdr(NULL, &hdr); aBufSize[3] = tPutDiskDataHdr(NULL, &hdr);
code = tRealloc(&aBuf[3], aBufN[3]); code = tRealloc(&aBuf[3], aBufSize[3]);
if (code) goto _exit; if (code) goto _exit;
tPutDiskDataHdr(aBuf[3], &hdr); tPutDiskDataHdr(aBuf[3], &hdr);
for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) {
code = tRealloc(&aBuf[3], aBufSize[3] + tPutBlockCol(NULL, &primaryKeyBlockCols[i]));
if (code) goto _exit;
aBufSize[3] += tPutBlockCol(aBuf[3] + aBufSize[3], &primaryKeyBlockCols[i]);
}
// aggragate // aggragate
if (ppOut) { if (ppOut) {
*szOut = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; *szOut = aBufSize[0] + aBufSize[1] + aBufSize[2] + aBufSize[3];
code = tRealloc(ppOut, *szOut); code = tRealloc(ppOut, *szOut);
if (code) goto _exit; if (code) goto _exit;
memcpy(*ppOut, aBuf[3], aBufN[3]); memcpy(*ppOut, aBuf[3], aBufSize[3]);
memcpy(*ppOut + aBufN[3], aBuf[2], aBufN[2]); memcpy(*ppOut + aBufSize[3], aBuf[2], aBufSize[2]);
if (aBufN[1]) { if (aBufSize[1]) {
memcpy(*ppOut + aBufN[3] + aBufN[2], aBuf[1], aBufN[1]); memcpy(*ppOut + aBufSize[3] + aBufSize[2], aBuf[1], aBufSize[1]);
} }
if (aBufN[0]) { if (aBufSize[0]) {
memcpy(*ppOut + aBufN[3] + aBufN[2] + aBufN[1], aBuf[0], aBufN[0]); memcpy(*ppOut + aBufSize[3] + aBufSize[2] + aBufSize[1], aBuf[0], aBufSize[0]);
} }
} }
@ -1405,6 +1494,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
int32_t n = 0; int32_t n = 0;
SDiskDataHdr hdr = {0}; SDiskDataHdr hdr = {0};
SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL];
// SDiskDataHdr // SDiskDataHdr
n += tGetDiskDataHdr(pIn + n, &hdr); n += tGetDiskDataHdr(pIn + n, &hdr);
@ -1414,6 +1504,11 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
pBlockData->uid = hdr.uid; pBlockData->uid = hdr.uid;
pBlockData->nRow = hdr.nRow; pBlockData->nRow = hdr.nRow;
// primary SBlockCol
for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) {
n += tGetBlockCol(pIn + n, &primaryKeyBlockCols[i]);
}
// uid // uid
if (hdr.uid == 0) { if (hdr.uid == 0) {
ASSERT(hdr.szUid); ASSERT(hdr.szUid);
@ -1437,39 +1532,41 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
if (code) goto _exit; if (code) goto _exit;
n += hdr.szKey; n += hdr.szKey;
// loop to decode each column data // Primary key columns
if (hdr.szBlkCol == 0) goto _exit; for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) {
SColData *pColData;
int32_t nColData = 0; code = tBlockDataAddColData(pBlockData, primaryKeyBlockCols[i].cid, primaryKeyBlockCols[i].type,
int32_t nt = 0; primaryKeyBlockCols[i].cflag, &pColData);
while (nt < hdr.szBlkCol) { if (code) goto _exit;
SBlockCol blockCol = {0};
nt += tGetBlockCol(pIn + n + nt, &blockCol); code = tsdbDecmprColData(pIn + n, &primaryKeyBlockCols[i], hdr.cmprAlg, hdr.nRow, pColData, &aBuf[0]);
++nColData; if (code) goto _exit;
n = n + primaryKeyBlockCols[i].szBitmap + primaryKeyBlockCols[i].szOffset + primaryKeyBlockCols[i].szValue;
} }
ASSERT(nt == hdr.szBlkCol);
code = tBlockDataAdjustColData(pBlockData, nColData); // regular key columns
if (code) goto _exit; if (hdr.szBlkCol > 0) {
int32_t nt = 0;
while (nt < hdr.szBlkCol) {
SColData *pColData;
SBlockCol blockCol = {0};
nt += tGetBlockCol(pIn + n + nt, &blockCol);
nt = 0; code = tBlockDataAddColData(pBlockData, blockCol.cid, blockCol.type, blockCol.cflag, &pColData);
int32_t iColData = 0; if (code) goto _exit;
while (nt < hdr.szBlkCol) {
SBlockCol blockCol = {0};
nt += tGetBlockCol(pIn + n + nt, &blockCol);
SColData *pColData = &pBlockData->aColData[iColData++]; if (blockCol.flag == HAS_NULL) {
for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) {
tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type));
if (blockCol.flag == HAS_NULL) { if (code) goto _exit;
for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { }
code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); } else {
code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData,
&aBuf[0]);
if (code) goto _exit; if (code) goto _exit;
} }
} else {
code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData,
&aBuf[0]);
if (code) goto _exit;
} }
} }
@ -1481,16 +1578,19 @@ _exit:
int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) { int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) {
int32_t n = 0; int32_t n = 0;
n += tPutU32(p ? p + n : p, pHdr->delimiter); n += tPutU32(p ? p + n : p, pHdr->delimiter, true);
n += tPutU32v(p ? p + n : p, pHdr->fmtVer); n += tPutU32v(p ? p + n : p, pHdr->fmtVer, true);
n += tPutI64(p ? p + n : p, pHdr->suid); n += tPutI64(p ? p + n : p, pHdr->suid, true);
n += tPutI64(p ? p + n : p, pHdr->uid); n += tPutI64(p ? p + n : p, pHdr->uid, true);
n += tPutI32v(p ? p + n : p, pHdr->szUid); n += tPutI32v(p ? p + n : p, pHdr->szUid, true);
n += tPutI32v(p ? p + n : p, pHdr->szVer); n += tPutI32v(p ? p + n : p, pHdr->szVer, true);
n += tPutI32v(p ? p + n : p, pHdr->szKey); n += tPutI32v(p ? p + n : p, pHdr->szKey, true);
n += tPutI32v(p ? p + n : p, pHdr->szBlkCol); n += tPutI32v(p ? p + n : p, pHdr->szBlkCol, true);
n += tPutI32v(p ? p + n : p, pHdr->nRow); n += tPutI32v(p ? p + n : p, pHdr->nRow, true);
n += tPutI8(p ? p + n : p, pHdr->cmprAlg); n += tPutI8(p ? p + n : p, pHdr->cmprAlg, true);
if (pHdr->fmtVer == 1) {
n += tPutI8(p ? p + n : p, pHdr->numPrimaryKeyCols, true);
}
return n; return n;
} }
@ -1499,16 +1599,21 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) {
int32_t n = 0; int32_t n = 0;
SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; SDiskDataHdr *pHdr = (SDiskDataHdr *)ph;
n += tGetU32(p + n, &pHdr->delimiter); n += tGetU32(p + n, &pHdr->delimiter, true);
n += tGetU32v(p + n, &pHdr->fmtVer); n += tGetU32v(p + n, &pHdr->fmtVer, true);
n += tGetI64(p + n, &pHdr->suid); n += tGetI64(p + n, &pHdr->suid, true);
n += tGetI64(p + n, &pHdr->uid); n += tGetI64(p + n, &pHdr->uid, true);
n += tGetI32v(p + n, &pHdr->szUid); n += tGetI32v(p + n, &pHdr->szUid, true);
n += tGetI32v(p + n, &pHdr->szVer); n += tGetI32v(p + n, &pHdr->szVer, true);
n += tGetI32v(p + n, &pHdr->szKey); n += tGetI32v(p + n, &pHdr->szKey, true);
n += tGetI32v(p + n, &pHdr->szBlkCol); n += tGetI32v(p + n, &pHdr->szBlkCol, true);
n += tGetI32v(p + n, &pHdr->nRow); n += tGetI32v(p + n, &pHdr->nRow, true);
n += tGetI8(p + n, &pHdr->cmprAlg); n += tGetI8(p + n, &pHdr->cmprAlg, true);
if (pHdr->fmtVer == 1) {
n += tGetI8(p + n, &pHdr->numPrimaryKeyCols, true);
} else {
pHdr->numPrimaryKeyCols = 0;
}
return n; return n;
} }
@ -1517,11 +1622,11 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) {
int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
int32_t n = 0; int32_t n = 0;
n += tPutI16v(p ? p + n : p, pColAgg->colId); n += tPutI16v(p ? p + n : p, pColAgg->colId, true);
n += tPutI16v(p ? p + n : p, pColAgg->numOfNull); n += tPutI16v(p ? p + n : p, pColAgg->numOfNull, true);
n += tPutI64(p ? p + n : p, pColAgg->sum); n += tPutI64(p ? p + n : p, pColAgg->sum, true);
n += tPutI64(p ? p + n : p, pColAgg->max); n += tPutI64(p ? p + n : p, pColAgg->max, true);
n += tPutI64(p ? p + n : p, pColAgg->min); n += tPutI64(p ? p + n : p, pColAgg->min, true);
return n; return n;
} }
@ -1529,11 +1634,11 @@ int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) {
int32_t n = 0; int32_t n = 0;
n += tGetI16v(p + n, &pColAgg->colId); n += tGetI16v(p + n, &pColAgg->colId, true);
n += tGetI16v(p + n, &pColAgg->numOfNull); n += tGetI16v(p + n, &pColAgg->numOfNull, true);
n += tGetI64(p + n, &pColAgg->sum); n += tGetI64(p + n, &pColAgg->sum, true);
n += tGetI64(p + n, &pColAgg->max); n += tGetI64(p + n, &pColAgg->max, true);
n += tGetI64(p + n, &pColAgg->min); n += tGetI64(p + n, &pColAgg->min, true);
return n; return n;
} }
@ -1656,7 +1761,7 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
ASSERT(pColData->cid == pBlockCol->cid); ASSERT(pColData->cid == pBlockCol->cid);
ASSERT(pColData->type == pBlockCol->type); ASSERT(pColData->type == pBlockCol->type);
pColData->smaOn = pBlockCol->smaOn; pColData->cflag = pBlockCol->cflag;
pColData->flag = pBlockCol->flag; pColData->flag = pBlockCol->flag;
pColData->nVal = nVal; pColData->nVal = nVal;
pColData->nData = pBlockCol->szOrigin; pColData->nData = pBlockCol->szOrigin;

24
source/util/src/tcompare.c
View File

@ -17,13 +17,13 @@
#define _XOPEN_SOURCE #define _XOPEN_SOURCE
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "tcompare.h" #include "tcompare.h"
#include "osString.h"
#include "regex.h" #include "regex.h"
#include "tdef.h" #include "tdef.h"
#include "thash.h" #include "thash.h"
#include "tlog.h" #include "tlog.h"
#include "tutil.h" #include "tutil.h"
#include "types.h" #include "types.h"
#include "osString.h"
int32_t setChkInBytes1(const void *pLeft, const void *pRight) { int32_t setChkInBytes1(const void *pLeft, const void *pRight) {
return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0;
@ -209,11 +209,11 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) {
int32_t len1 = varDataLen(pLeft); int32_t len1 = varDataLen(pLeft);
int32_t len2 = varDataLen(pRight); int32_t len2 = varDataLen(pRight);
int32_t ret = tasoUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1>len2 ? len2:len1); int32_t ret = tasoUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1 > len2 ? len2 : len1);
if (ret == 0) { if (ret == 0) {
if (len1 > len2) if (len1 > len2)
return 1; return 1;
else if(len1 < len2) else if (len1 < len2)
return -1; return -1;
else else
return 0; return 0;
@ -242,9 +242,7 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight) {
} }
} }
int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight) { int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight) { return compareLenBinaryVal(pRight, pLeft); }
return compareLenBinaryVal(pRight, pLeft);
}
// string > number > bool > null // string > number > bool > null
// ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison
@ -328,7 +326,7 @@ int32_t compareInt8Uint16(const void *pLeft, const void *pRight) {
} }
int32_t compareInt8Uint32(const void *pLeft, const void *pRight) { int32_t compareInt8Uint32(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft); int8_t left = GET_INT8_VAL(pLeft);
if (left < 0) return -1; if (left < 0) return -1;
uint32_t right = GET_UINT32_VAL(pRight); uint32_t right = GET_UINT32_VAL(pRight);
if ((uint32_t)left > right) return 1; if ((uint32_t)left > right) return 1;
@ -337,7 +335,7 @@ int32_t compareInt8Uint32(const void *pLeft, const void *pRight) {
} }
int32_t compareInt8Uint64(const void *pLeft, const void *pRight) { int32_t compareInt8Uint64(const void *pLeft, const void *pRight) {
int8_t left = GET_INT8_VAL(pLeft); int8_t left = GET_INT8_VAL(pLeft);
if (left < 0) return -1; if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight); uint64_t right = GET_UINT64_VAL(pRight);
if ((uint64_t)left > right) return 1; if ((uint64_t)left > right) return 1;
@ -402,7 +400,7 @@ int32_t compareInt16Uint16(const void *pLeft, const void *pRight) {
} }
int32_t compareInt16Uint32(const void *pLeft, const void *pRight) { int32_t compareInt16Uint32(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft); int16_t left = GET_INT16_VAL(pLeft);
if (left < 0) return -1; if (left < 0) return -1;
uint32_t right = GET_UINT32_VAL(pRight); uint32_t right = GET_UINT32_VAL(pRight);
if ((uint32_t)left > right) return 1; if ((uint32_t)left > right) return 1;
@ -411,7 +409,7 @@ int32_t compareInt16Uint32(const void *pLeft, const void *pRight) {
} }
int32_t compareInt16Uint64(const void *pLeft, const void *pRight) { int32_t compareInt16Uint64(const void *pLeft, const void *pRight) {
int16_t left = GET_INT16_VAL(pLeft); int16_t left = GET_INT16_VAL(pLeft);
if (left < 0) return -1; if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight); uint64_t right = GET_UINT64_VAL(pRight);
if ((uint64_t)left > right) return 1; if ((uint64_t)left > right) return 1;
@ -476,7 +474,7 @@ int32_t compareInt32Uint16(const void *pLeft, const void *pRight) {
} }
int32_t compareInt32Uint32(const void *pLeft, const void *pRight) { int32_t compareInt32Uint32(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft); int32_t left = GET_INT32_VAL(pLeft);
if (left < 0) return -1; if (left < 0) return -1;
uint32_t right = GET_UINT32_VAL(pRight); uint32_t right = GET_UINT32_VAL(pRight);
if ((uint32_t)left > right) return 1; if ((uint32_t)left > right) return 1;
@ -485,7 +483,7 @@ int32_t compareInt32Uint32(const void *pLeft, const void *pRight) {
} }
int32_t compareInt32Uint64(const void *pLeft, const void *pRight) { int32_t compareInt32Uint64(const void *pLeft, const void *pRight) {
int32_t left = GET_INT32_VAL(pLeft); int32_t left = GET_INT32_VAL(pLeft);
if (left < 0) return -1; if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight); uint64_t right = GET_UINT64_VAL(pRight);
if ((uint64_t)left > right) return 1; if ((uint64_t)left > right) return 1;
@ -558,7 +556,7 @@ int32_t compareInt64Uint32(const void *pLeft, const void *pRight) {
} }
int32_t compareInt64Uint64(const void *pLeft, const void *pRight) { int32_t compareInt64Uint64(const void *pLeft, const void *pRight) {
int64_t left = GET_INT64_VAL(pLeft); int64_t left = GET_INT64_VAL(pLeft);
if (left < 0) return -1; if (left < 0) return -1;
uint64_t right = GET_UINT64_VAL(pRight); uint64_t right = GET_UINT64_VAL(pRight);
if ((uint64_t)left > right) return 1; if ((uint64_t)left > right) return 1;