diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index aed1d03fc1..c63b26ef09 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -27,7 +27,6 @@ extern "C" { #endif -typedef struct SBuffer SBuffer; typedef struct SSchema SSchema; typedef struct SSchema2 SSchema2; typedef struct STColumn STColumn; @@ -39,10 +38,12 @@ typedef struct SRowIter SRowIter; typedef struct STagVal STagVal; typedef struct STag STag; typedef struct SColData SColData; +typedef struct SRowKey SRowKey; -#define HAS_NONE ((uint8_t)0x1) -#define HAS_NULL ((uint8_t)0x2) -#define HAS_VALUE ((uint8_t)0x4) +#define HAS_NONE ((uint8_t)0x1) +#define HAS_NULL ((uint8_t)0x2) +#define HAS_VALUE ((uint8_t)0x4) +#define HAS_MULTI_KEY ((uint8_t)0x8) // bitmap ================================ const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, @@ -78,19 +79,6 @@ const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b001111 } while (0) #define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) -// SBuffer ================================ -struct SBuffer { - int64_t nBuf; - uint8_t *pBuf; -}; - -#define tBufferCreate() \ - (SBuffer) { .nBuf = 0, .pBuf = NULL } -void tBufferDestroy(SBuffer *pBuffer); -int32_t tBufferInit(SBuffer *pBuffer, int64_t size); -int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData); -int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData); - // SColVal ================================ #define CV_FLAG_VALUE ((int8_t)0x0) #define CV_FLAG_NONE ((int8_t)0x1) @@ -111,6 +99,8 @@ void tRowDestroy(SRow *pRow); int32_t tRowSort(SArray *aRowP); int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, int8_t flag); int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); +void tRowGetKey(SRow *pRow, SRowKey *key); +int32_t tRowKeyCmpr(const void *p1, const void *p2); // SRowIter ================================ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); @@ -134,7 +124,7 @@ int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, voi // SColData ================================ typedef void *(*xMallocFn)(void *, int32_t); void tColDataDestroy(void *ph); -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t cflag); void tColDataClear(SColData *pColData); void tColDataDeepClear(SColData *pColData); int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); @@ -172,6 +162,15 @@ struct STSchema { STColumn columns[]; }; +/* + * 1. Tuple format: + * SRow + [bit map +] fix-length data + [var-length data +] [(type + offset) * numPrimaryKeyCols + fixedLen + + * numOfCols + numPrimaryKeyCols] + * + * 2. K-V format: + * SRow + numColsNotNone + u8/u16/u32 * numColsNotNone + ([-]cid [+ data]) * numColsNotNone + [(type + index) * + * numPrimaryKeyCols + numPrimaryKeyCols】 + */ struct SRow { uint8_t flag; uint8_t rsv; @@ -191,6 +190,18 @@ struct SValue { }; }; +#define TD_MAX_PRIMARY_KEY_COL 8 +typedef struct { + int8_t type; + SValue value; +} STypeValue; + +struct SRowKey { + TSKEY ts; + uint8_t numOfKeys; + STypeValue keys[TD_MAX_PRIMARY_KEY_COL]; +}; + struct SColVal { int16_t cid; int8_t type; @@ -201,7 +212,7 @@ struct SColVal { struct SColData { int16_t cid; int8_t type; - int8_t smaOn; + int8_t cflag; int32_t numOfNone; // # of none int32_t numOfNull; // # of null int32_t numOfValue; // # of vale diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d776972d01..66c1c15812 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1393,7 +1393,7 @@ void tFreeSCompactDbReq(SCompactDbReq* pReq); typedef struct { int32_t compactId; - int8_t bAccepted; + int8_t bAccepted; } SCompactDbRsp; int32_t tSerializeSCompactDbRsp(void* buf, int32_t bufLen, SCompactDbRsp* pRsp); @@ -1407,7 +1407,7 @@ typedef struct { int32_t tSerializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq); int32_t tDeserializeSKillCompactReq(void* buf, int32_t bufLen, SKillCompactReq* pReq); -void tFreeSKillCompactReq(SKillCompactReq *pReq); +void tFreeSKillCompactReq(SKillCompactReq* pReq); typedef struct { char name[TSDB_FUNC_NAME_LEN]; @@ -1741,9 +1741,9 @@ int32_t tSerializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* int32_t tDeserializeSCompactVnodeReq(void* buf, int32_t bufLen, SCompactVnodeReq* pReq); typedef struct { - int32_t compactId; - int32_t vgId; - int32_t dnodeId; + int32_t compactId; + int32_t vgId; + int32_t dnodeId; } SVKillCompactReq; int32_t tSerializeSVKillCompactReq(void* buf, int32_t bufLen, SVKillCompactReq* pReq); @@ -1944,9 +1944,9 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; char tb[TSDB_TABLE_NAME_LEN]; char user[TSDB_USER_LEN]; - char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns + char filterTb[TSDB_TABLE_NAME_LEN]; // for ins_columns int64_t showId; - int64_t compactId; // for compact + int64_t compactId; // for compact } SRetrieveTableReq; typedef struct SSysTableSchema { diff --git a/include/util/tdef.h b/include/util/tdef.h index aee20514ad..be5e4971a9 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -444,6 +444,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_VARBINARY_LEN TSDB_MAX_FIELD_LEN // 16384-8:65519 #define PRIMARYKEY_TIMESTAMP_COL_ID 1 +#define PRIMARYKEY_COL_ID 2 // for another primary key column in addition to timestamp #define COL_REACH_END(colId, maxColId) ((colId) > (maxColId)) #ifdef WINDOWS diff --git a/include/util/tencode.h b/include/util/tencode.h index d05d4914e3..25d5dbbe73 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -468,272 +468,217 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { } // =========================================== -#define tPutV(p, v) \ - do { \ - int32_t n = 0; \ - for (;;) { \ - if (v <= 0x7f) { \ - if (p) p[n] = v; \ - n++; \ - break; \ - } \ - if (p) p[n] = (v & 0x7f) | 0x80; \ - n++; \ - v >>= 7; \ - } \ - return n; \ +#define TPUT(TYPE, BUF, VAL, FORWARD) \ + do { \ + if (BUF) { \ + if (FORWARD) { \ + *(TYPE*)(BUF) = (VAL); \ + } else { \ + *(TYPE*)((BUF) - sizeof(TYPE)) = (VAL); \ + } \ + } \ + return sizeof(TYPE); \ + } while (0) + +#define TGET(TYPE, BUF, VAL, FORWARD) \ + do { \ + if (FORWARD) { \ + *(TYPE*)(VAL) = *(TYPE*)(BUF); \ + } else { \ + *(TYPE*)(VAL) = *(TYPE*)((BUF) - sizeof(TYPE)); \ + } \ + return sizeof(TYPE); \ + } while (0) + +#define TPUTV(BUF, VAL, FORWARD) \ + do { \ + int32_t n = 0; \ + for (;;) { \ + if ((VAL) < 0x80) { \ + if (BUF) { \ + if (FORWARD) { \ + (BUF)[n] = (VAL); \ + } else { \ + (BUF)[-(n + 1)] = (VAL); \ + } \ + } \ + n++; \ + break; \ + } else { \ + if (BUF) { \ + if (FORWARD) { \ + (BUF)[n] = ((VAL)&0x7f) | 0x80; \ + } else { \ + (BUF)[-(n + 1)] = ((VAL)&0x7f) | 0x80; \ + } \ + } \ + n++; \ + (VAL) >>= 7; \ + } \ + } \ + return n; \ + } while (0) + +#define TGETV(BUF, VAL, FORWARD) \ + do { \ + int32_t n = 0; \ + if (VAL) { \ + *(VAL) = 0; \ + } \ + for (;;) { \ + if (FORWARD) { \ + if (VAL) { \ + (*(VAL)) |= ((BUF)[n] & 0x7f) << (7 * n); \ + } \ + if ((BUF)[n++] < 0x80) break; \ + } else { \ + if (VAL) { \ + (*(VAL)) |= ((BUF)[-(n + 1)] & 0x7f) << (7 * n); \ + } \ + if ((BUF)[-(n++)] < 0x80) break; \ + } \ + } \ + return n; \ } while (0) // PUT -static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { - if (p) ((uint8_t*)p)[0] = v; - return sizeof(uint8_t); -} +static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v, bool forward) { TPUT(uint8_t, p, v, forward); } +static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v, bool forward) { TPUT(int8_t, p, v, forward); } +static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v, bool forward) { TPUT(uint16_t, p, v, forward); } +static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v, bool forward) { TPUT(int16_t, p, v, forward); } +static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v, bool forward) { TPUT(uint32_t, p, v, forward); } +static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v, bool forward) { TPUT(int32_t, p, v, forward); } +static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v, bool forward) { TPUT(uint64_t, p, v, forward); } +static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v, bool forward) { TPUT(int64_t, p, v, forward); } -static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { - if (p) ((int8_t*)p)[0] = v; - return sizeof(int8_t); -} - -static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) { - if (p) ((uint16_t*)p)[0] = v; - return sizeof(uint16_t); -} - -static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) { - if (p) ((int16_t*)p)[0] = v; - return sizeof(int16_t); -} - -static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) { - if (p) ((uint32_t*)p)[0] = v; - return sizeof(uint32_t); -} - -static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) { - if (p) ((int32_t*)p)[0] = v; - return sizeof(int32_t); -} - -static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) { - if (p) ((uint64_t*)p)[0] = v; - return sizeof(uint64_t); -} - -static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { - if (p) ((int64_t*)p)[0] = v; - return sizeof(int64_t); -} - -static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { +static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f, bool forward) { union { uint32_t ui; float f; - } v; - v.f = f; - - return tPutU32(p, v.ui); + } v = {.f = f}; + return tPutU32(p, v.ui, forward); } -static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { +static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d, bool forward) { union { uint64_t ui; double d; - } v; - v.d = d; - - return tPutU64(p, v.ui); + } v = {.d = d}; + return tPutU64(p, v.ui, forward); } -static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } - -static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } - -static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } - -static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } - -static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } - -static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } +static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v, bool forward) { TPUTV(p, v, forward); } +static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v, bool forward) { + return tPutU16v(p, ZIGZAGE(int16_t, v), forward); +} +static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v, bool forward) { TPUTV(p, v, forward); } +static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v, bool forward) { + return tPutU32v(p, ZIGZAGE(int32_t, v), forward); +} +static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v, bool forward) { TPUTV(p, v, forward); } +static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v, bool forward) { + return tPutU64v(p, ZIGZAGE(int64_t, v), forward); +} // GET -static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { - if (v) *v = ((uint8_t*)p)[0]; - return sizeof(uint8_t); -} - -static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { - if (v) *v = ((int8_t*)p)[0]; - return sizeof(int8_t); -} - -static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) { - if (v) *v = ((uint16_t*)p)[0]; - return sizeof(uint16_t); -} - -static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) { - if (v) *v = ((int16_t*)p)[0]; - return sizeof(int16_t); -} - -static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) { - if (v) *v = ((uint32_t*)p)[0]; - return sizeof(uint32_t); -} - -static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) { - if (v) *v = ((int32_t*)p)[0]; - return sizeof(int32_t); -} - -static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) { - if (v) *v = ((uint64_t*)p)[0]; - return sizeof(uint64_t); -} - -static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { - if (v) *v = ((int64_t*)p)[0]; - return sizeof(int64_t); -} - -static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { - int32_t n = 0; - - if (v) *v = 0; - for (;;) { - if (p[n] <= 0x7f) { - if (v) (*v) |= (((uint16_t)p[n]) << (7 * n)); - n++; - break; - } - if (v) (*v) |= (((uint16_t)(p[n] & 0x7f)) << (7 * n)); - n++; - } - - return n; -} - -static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { - int32_t n; +static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v, bool forward) { TGET(uint8_t, p, v, forward); } +static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v, bool forward) { TGET(int8_t, p, v, forward); } +static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v, bool forward) { TGET(uint16_t, p, v, forward); } +static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v, bool forward) { TGET(int16_t, p, v, forward); } +static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v, bool forward) { TGET(uint32_t, p, v, forward); } +static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v, bool forward) { TGET(int32_t, p, v, forward); } +static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v, bool forward) { TGET(uint64_t, p, v, forward); } +static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v, bool forward) { TGET(int64_t, p, v, forward); } +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v, bool forward) { TGETV(p, v, forward); } +static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v, bool forward) { uint16_t tv; - - n = tGetU16v(p, &tv); - if (v) *v = ZIGZAGD(int16_t, tv); - - return n; -} - -static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { - int32_t n = 0; - - if (v) *v = 0; - for (;;) { - if (p[n] <= 0x7f) { - if (v) (*v) |= (((uint32_t)p[n]) << (7 * n)); - n++; - break; - } - if (v) (*v) |= (((uint32_t)(p[n] & 0x7f)) << (7 * n)); - n++; + int32_t n = tGetU16v(p, &tv, forward); + if (v) { + *v = ZIGZAGD(int16_t, tv); } - return n; } - -static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { - int32_t n; +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v, bool forward) { TGETV(p, v, forward); } +static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v, bool forward) { uint32_t tv; - - n = tGetU32v(p, &tv); - if (v) *v = ZIGZAGD(int32_t, tv); - - return n; -} - -static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { - int32_t n = 0; - - if (v) *v = 0; - for (;;) { - if (p[n] <= 0x7f) { - if (v) (*v) |= (((uint64_t)p[n]) << (7 * n)); - n++; - break; - } - if (v) (*v) |= (((uint64_t)(p[n] & 0x7f)) << (7 * n)); - n++; + int32_t n = tGetU32v(p, &tv, forward); + if (v) { + *v = ZIGZAGD(int32_t, tv); } - return n; } - -static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { - int32_t n; +static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v, bool forward) { TGETV(p, v, forward); } +static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v, bool forward) { uint64_t tv; - - n = tGetU64v(p, &tv); - if (v) *v = ZIGZAGD(int64_t, tv); - + int32_t n = tGetU64v(p, &tv, forward); + if (v) { + *v = ZIGZAGD(int64_t, tv); + } return n; } - -static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) { - int32_t n = 0; - +static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f, bool forward) { union { uint32_t ui; float f; } v; - n = tGetU32(p, &v.ui); - - *f = v.f; + int32_t n = tGetU32(p, &v.ui, forward); + if (f) { + *f = v.f; + } return n; } -static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) { - int32_t n = 0; - +static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d, bool forward) { union { uint64_t ui; double d; } v; - n = tGetU64(p, &v.ui); - - *d = v.d; + int32_t n = tGetU64(p, &v.ui, forward); + if (d) { + *d = v.d; + } return n; } // ===================== -static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { - int n = 0; - - n += tPutU32v(p ? p + n : p, nData); - if (p) memcpy(p + n, pData, nData); +static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData, bool forward) { + int32_t n = tPutU32v(p, nData, forward); + if (p) { + if (forward) { + memcpy(p + n, pData, nData); + } else { + memcpy(p - n - nData, pData, nData); + } + } n += nData; return n; } -static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData) { - int32_t n = 0; +static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* nData, bool forward) { uint32_t nt; - - n += tGetU32v(p, &nt); - if (nData) *nData = nt; - if (ppData) *ppData = p + n; - n += nt; - - return n; + int32_t n = tGetU32v(p, &nt, forward); + if (nData) { + *nData = nt; + } + if (ppData) { + if (forward) { + *ppData = p + n; + } else { + *ppData = p - n - nt; + } + } + return n + nt; } -static FORCE_INLINE int32_t tPutCStr(uint8_t* p, char* pData) { - return tPutBinary(p, (uint8_t*)pData, strlen(pData) + 1); +static FORCE_INLINE int32_t tPutCStr(uint8_t* p, char* pData, bool forward) { + return tPutBinary(p, (uint8_t*)pData, strlen(pData) + 1, forward); +} +static FORCE_INLINE int32_t tGetCStr(uint8_t* p, char** ppData, bool forward) { + return tGetBinary(p, (uint8_t**)ppData, NULL, forward); } -static FORCE_INLINE int32_t tGetCStr(uint8_t* p, char** ppData) { return tGetBinary(p, (uint8_t**)ppData, NULL); } #ifdef __cplusplus } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 02dfbfebfe..2a9d35db9a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -23,46 +23,13 @@ static int32_t (*tColDataAppendValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData); static int32_t (*tColDataUpdateValueImpl[8][3])(SColData *pColData, uint8_t *pData, uint32_t nData, bool forward); -// SBuffer ================================ -void tBufferDestroy(SBuffer *pBuffer) { - tFree(pBuffer->pBuf); - pBuffer->pBuf = NULL; -} - -int32_t tBufferInit(SBuffer *pBuffer, int64_t size) { - pBuffer->nBuf = 0; - return tRealloc(&pBuffer->pBuf, size); -} - -int32_t tBufferPut(SBuffer *pBuffer, const void *pData, int64_t nData) { - int32_t code = 0; - - code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); - if (code) return code; - - memcpy(pBuffer->pBuf + pBuffer->nBuf, pData, nData); - pBuffer->nBuf += nData; - - return code; -} - -int32_t tBufferReserve(SBuffer *pBuffer, int64_t nData, void **ppData) { - int32_t code = tRealloc(&pBuffer->pBuf, pBuffer->nBuf + nData); - if (code) return code; - - *ppData = pBuffer->pBuf + pBuffer->nBuf; - pBuffer->nBuf += nData; - - return code; -} - // ================================ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); // SRow ======================================================================== #define KV_FLG_LIT ((uint8_t)0x10) #define KV_FLG_MID ((uint8_t)0x20) -#define KV_FLG_BIG ((uint8_t)0x30) +#define KV_FLG_BIG ((uint8_t)0x40) #define BIT_FLG_NONE ((uint8_t)0x0) #define BIT_FLG_NULL ((uint8_t)0x1) @@ -71,7 +38,7 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); #pragma pack(push, 1) typedef struct { int16_t nCol; - char idx[]; // uint8_t * | uint16_t * | uint32_t * + uint8_t idx[]; // uint8_t * | uint16_t * | uint32_t * } SKVIdx; #pragma pack(pop) @@ -118,6 +85,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { int32_t nkv = 0; int32_t maxIdx = 0; int32_t nIdx = 0; + uint8_t numPrimaryKeyCols = 0; while (pTColumn) { if (pColVal) { if (pColVal->cid == pTColumn->colId) { @@ -125,18 +93,33 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { flag |= HAS_VALUE; maxIdx = nkv; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - ntp = ntp + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; - nkv = nkv + tPutI16v(NULL, pTColumn->colId) + tPutU32v(NULL, pColVal->value.nData) + pColVal->value.nData; + ntp = ntp + tPutU32v(NULL, pColVal->value.nData, true) + pColVal->value.nData; + nkv = nkv + tPutI16v(NULL, pTColumn->colId, true) + tPutU32v(NULL, pColVal->value.nData, true) + + pColVal->value.nData; } else { - nkv = nkv + tPutI16v(NULL, pTColumn->colId) + pTColumn->bytes; + nkv = nkv + tPutI16v(NULL, pTColumn->colId, true) + pTColumn->bytes; } + + if (pTColumn->flags & COL_IS_KEY) { + flag |= HAS_MULTI_KEY; + numPrimaryKeyCols++; + ntp += (tPutI8(NULL, pTColumn->type, false) // type + + tPutI32v(NULL, pTColumn->offset, false) // offset + ); + nkv += (tPutI8(NULL, pTColumn->type, false) // type + + tPutI32v(NULL, nIdx, false) // index + ); + } + nIdx++; } else if (COL_VAL_IS_NONE(pColVal)) { // NONE + ASSERT((pTColumn->flags & COL_IS_KEY) == 0); flag |= HAS_NONE; } else if (COL_VAL_IS_NULL(pColVal)) { // NULL + ASSERT((pTColumn->flags & COL_IS_KEY) == 0); flag |= HAS_NULL; maxIdx = nkv; - nkv += tPutI16v(NULL, -pTColumn->colId); + nkv += tPutI16v(NULL, -pTColumn->colId, true); nIdx++; } else { if (ASSERTS(0, "invalid input")) { @@ -148,19 +131,22 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } else if (pColVal->cid > pTColumn->colId) { // NONE + ASSERT((pTColumn->flags & COL_IS_KEY) == 0); flag |= HAS_NONE; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } else { pColVal = (++iColVal < nColVal) ? &colVals[iColVal] : NULL; } } else { // NONE + ASSERT((pTColumn->flags & COL_IS_KEY) == 0); flag |= HAS_NONE; pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; } } // compare --------------- - switch (flag) { + // Tuple Row Format + switch (flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) { case HAS_NONE: case HAS_NULL: ntp = sizeof(SRow); @@ -184,6 +170,13 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { goto _exit; } } + if (flag & HAS_MULTI_KEY) { + ntp += (tPutI8(NULL, numPrimaryKeyCols, false) // numPrimaryKeyCols + + tPutU32v(NULL, pTSchema->numOfCols, false) // numOfCols + + tPutU32v(NULL, pTSchema->flen, false) // fixedLen + ); + } + // Key-Value Row Format if (maxIdx <= UINT8_MAX) { nkv = sizeof(SRow) + sizeof(SKVIdx) + nIdx + nkv; flag |= KV_FLG_LIT; @@ -194,16 +187,19 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { nkv = sizeof(SRow) + sizeof(SKVIdx) + (nIdx << 2) + nkv; flag |= KV_FLG_BIG; } - int32_t nRow; + if (flag & HAS_MULTI_KEY) { + nkv += tPutI8(NULL, numPrimaryKeyCols, false); + } + int32_t rowLength; if (nkv < ntp) { - nRow = nkv; + rowLength = nkv; } else { - nRow = ntp; + rowLength = ntp; flag &= ((uint8_t)0x0f); } // alloc -------------- - pRow = taosMemoryMalloc(nRow); + pRow = taosMemoryMalloc(rowLength); if (NULL == pRow) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -215,7 +211,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { pRow->flag = flag; pRow->rsv = 0; pRow->sver = pTSchema->version; - pRow->len = nRow; + pRow->len = rowLength; memcpy(&pRow->ts, &pColVal->value.val, sizeof(TSKEY)); if (flag == HAS_NONE || flag == HAS_NULL) { @@ -226,6 +222,13 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { pColVal = (iColVal < nColVal) ? &colVals[iColVal] : NULL; iTColumn = 1; pTColumn = pTSchema->columns + iTColumn; + uint8_t *pEnd = ((uint8_t *)pRow) + rowLength; + + if (pRow->flag & HAS_MULTI_KEY) { + pEnd -= tPutI8(pEnd, numPrimaryKeyCols, false); // numPrimaryKeyCols + pEnd -= tPutU32v(pEnd, pTSchema->numOfCols, false); // numOfCols + pEnd -= tPutU32v(NULL, pTSchema->flen, false); // fixedLen + } if (flag >> 4) { // KV SKVIdx *pIdx = (SKVIdx *)pRow->data; int32_t iIdx = 0; @@ -251,17 +254,23 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { } else { ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; } - iIdx++; - nv += tPutI16v(pv + nv, pTColumn->colId); + nv += tPutI16v(pv + nv, pTColumn->colId, true); if (IS_VAR_DATA_TYPE(pTColumn->type)) { - nv += tPutU32v(pv + nv, pColVal->value.nData); + nv += tPutU32v(pv + nv, pColVal->value.nData, true); memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); nv += pColVal->value.nData; } else { memcpy(pv + nv, &pColVal->value.val, pTColumn->bytes); nv += pTColumn->bytes; } + + if (pRow->flag & HAS_MULTI_KEY) { + pEnd -= tPutI8(pEnd, pTColumn->type, false); + pEnd -= tPutI32v(pEnd, iIdx, false); + } + + iIdx++; } else if (COL_VAL_IS_NULL(pColVal)) { if (flag & KV_FLG_LIT) { ((uint8_t *)pIdx->idx)[iIdx] = (uint8_t)nv; @@ -270,8 +279,8 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { } else { ((uint32_t *)pIdx->idx)[iIdx] = (uint32_t)nv; } + nv += tPutI16v(pv + nv, -pTColumn->colId, true); iIdx++; - nv += tPutI16v(pv + nv, -pTColumn->colId); } pTColumn = (++iTColumn < pTSchema->numOfCols) ? pTSchema->columns + iTColumn : NULL; @@ -291,7 +300,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { uint8_t *pv = NULL; int32_t nv = 0; - switch (flag) { + switch (flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) { case (HAS_NULL | HAS_NONE): pb = pRow->data; break; @@ -334,7 +343,7 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { if (IS_VAR_DATA_TYPE(pTColumn->type)) { *(int32_t *)(pf + pTColumn->offset) = nv; - nv += tPutU32v(pv + nv, pColVal->value.nData); + nv += tPutU32v(pv + nv, pColVal->value.nData, true); if (pColVal->value.nData) { memcpy(pv + nv, pColVal->value.pData, pColVal->value.nData); nv += pColVal->value.nData; @@ -342,6 +351,11 @@ int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow) { } else { memcpy(pf + pTColumn->offset, &pColVal->value.val, TYPE_BYTES[pTColumn->type]); } + + if (pTColumn->flags & COL_IS_KEY) { + pEnd -= tPutI8(pEnd, pTColumn->type, false); + pEnd -= tPutI32v(pEnd, pTColumn->offset, false); + } } else if (COL_VAL_IS_NONE(pColVal)) { // NONE ROW_SET_BITMAP(pb, flag, iTColumn - 1, BIT_FLG_NONE); if (pf) memset(pf + pTColumn->offset, 0, TYPE_BYTES[pTColumn->type]); @@ -391,12 +405,12 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) return 0; } - if (pRow->flag == HAS_NONE) { + if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NONE) { *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); return 0; } - if (pRow->flag == HAS_NULL) { + if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NULL) { *pColVal = COL_VAL_NULL(pTColumn->colId, pTColumn->type); return 0; } @@ -426,7 +440,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) } int16_t cid; - pData += tGetI16v(pData, &cid); + pData += tGetI16v(pData, &cid, true); if (TABS(cid) == pTColumn->colId) { if (cid < 0) { @@ -437,7 +451,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pData += tGetU32v(pData, &pColVal->value.nData); + pData += tGetU32v(pData, &pColVal->value.nData, true); if (pColVal->value.nData > 0) { pColVal->value.pData = pData; } else { @@ -457,13 +471,13 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) *pColVal = COL_VAL_NONE(pTColumn->colId, pTColumn->type); } else { // Tuple Row - if (pRow->flag == HAS_VALUE) { + if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_VALUE) { pColVal->cid = pTColumn->colId; pColVal->type = pTColumn->type; pColVal->flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { uint8_t *pData = pRow->data + pTSchema->flen + *(int32_t *)(pRow->data + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); + pData += tGetU32v(pData, &pColVal->value.nData, true); if (pColVal->value.nData) { pColVal->value.pData = pData; } else { @@ -477,7 +491,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) uint8_t *pv; uint8_t bv = BIT_FLG_VALUE; - switch (pRow->flag) { + switch (pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) { case (HAS_NULL | HAS_NONE): bv = GET_BIT1(pRow->data, iCol - 1); break; @@ -516,7 +530,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) pColVal->flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); - pData += tGetU32v(pData, &pColVal->value.nData); + pData += tGetU32v(pData, &pColVal->value.nData, true); if (pColVal->value.nData) { pColVal->value.pData = pData; } else { @@ -682,7 +696,9 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { pIter->pTSchema = pTSchema; pIter->iTColumn = 0; - if (pRow->flag == HAS_NONE || pRow->flag == HAS_NULL) goto _exit; + if ((pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NONE || + (pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) == HAS_NULL) + goto _exit; if (pRow->flag >> 4) { pIter->iCol = 0; @@ -695,7 +711,7 @@ int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter) { pIter->pv = pIter->pIdx->idx + (pIter->pIdx->nCol << 2); // * sizeof(uint32_t) } } else { - switch (pRow->flag) { + switch (pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) { case (HAS_NULL | HAS_NONE): pIter->pb = pRow->data; break; @@ -753,12 +769,13 @@ SColVal *tRowIterNext(SRowIter *pIter) { goto _exit; } - if (pIter->pRow->flag == HAS_NONE) { + uint8_t tflag = (pIter->pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)); + if (tflag == HAS_NONE) { pIter->cv = COL_VAL_NONE(pTColumn->colId, pTColumn->type); goto _exit; } - if (pIter->pRow->flag == HAS_NULL) { + if (tflag == HAS_NULL) { pIter->cv = COL_VAL_NULL(pTColumn->colId, pTColumn->type); goto _exit; } @@ -776,7 +793,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { } int16_t cid; - pData += tGetI16v(pData, &cid); + pData += tGetI16v(pData, &cid, true); if (TABS(cid) == pTColumn->colId) { if (cid < 0) { @@ -787,7 +804,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pData += tGetU32v(pData, &pIter->cv.value.nData); + pData += tGetU32v(pData, &pIter->cv.value.nData, true); if (pIter->cv.value.nData > 0) { pIter->cv.value.pData = pData; } else { @@ -813,7 +830,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { } else { // Tuple uint8_t bv = BIT_FLG_VALUE; if (pIter->pb) { - switch (pIter->pRow->flag) { + switch (tflag) { case (HAS_NULL | HAS_NONE): bv = GET_BIT1(pIter->pb, pIter->iTColumn - 1); break; @@ -846,7 +863,7 @@ SColVal *tRowIterNext(SRowIter *pIter) { pIter->cv.flag = CV_FLAG_VALUE; if (IS_VAR_DATA_TYPE(pTColumn->type)) { uint8_t *pData = pIter->pv + *(int32_t *)(pIter->pf + pTColumn->offset); - pData += tGetU32v(pData, &pIter->cv.value.nData); + pData += tGetU32v(pData, &pIter->cv.value.nData, true); if (pIter->cv.value.nData > 0) { pIter->cv.value.pData = pData; } else { @@ -921,7 +938,8 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * uint8_t *pb = NULL, *pf = NULL, *pv = NULL; - switch (pRow->flag) { + uint8_t tflag = pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE); + switch (tflag) { case HAS_VALUE: pf = pRow->data; pv = pf + pTSchema->flen; @@ -951,7 +969,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * ASSERT(pTColumn->type == pColData->type); if (pb) { uint8_t bv; - switch (pRow->flag) { + switch (tflag) { case (HAS_NULL | HAS_NONE): bv = GET_BIT1(pb, iTColumn - 1); break; @@ -988,7 +1006,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * if (IS_VAR_DATA_TYPE(pColData->type)) { uint8_t *pData = pv + *(int32_t *)(pf + pTColumn->offset); uint32_t nData; - pData += tGetU32v(pData, &nData); + pData += tGetU32v(pData, &nData, true); if (flag == 0) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_VALUE](pColData, pData, nData); } else { @@ -1062,7 +1080,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo } int16_t cid; - pData += tGetI16v(pData, &cid); + pData += tGetI16v(pData, &cid, true); if (TABS(cid) == pTColumn->colId) { if (cid < 0) { @@ -1075,7 +1093,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo } else { uint32_t nData; if (IS_VAR_DATA_TYPE(pTColumn->type)) { - pData += tGetU32v(pData, &nData); + pData += tGetU32v(pData, &nData, true); } else { nData = 0; } @@ -1123,9 +1141,10 @@ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, in ASSERT(pRow->sver == pTSchema->version); ASSERT(nColData > 0); - if (pRow->flag == HAS_NONE) { + uint8_t tflag = pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE); + if (tflag == HAS_NONE) { return tRowNoneUpsertColData(aColData, nColData, flag); - } else if (pRow->flag == HAS_NULL) { + } else if (tflag == HAS_NULL) { return tRowNullUpsertColData(aColData, nColData, pTSchema, flag); } else if (pRow->flag >> 4) { // KV row return tRowKVUpsertColData(pRow, pTSchema, aColData, nColData, flag); @@ -1134,6 +1153,171 @@ int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, in } } +void tRowGetKey(SRow *pRow, SRowKey *key) { + key->ts = pRow->ts; + if ((pRow->flag & HAS_MULTI_KEY) == 0) { + key->numOfKeys = 0; + } else { + uint8_t *pEnd = ((uint8_t *)pRow) + pRow->len; + + pEnd -= tGetU8(pEnd, &key->numOfKeys, false); + + if (pRow->flag >> 4) { // Key-Value format + SKVIdx *pKVIdx = (SKVIdx *)pRow->data; + uint8_t *pv; + + if (pRow->flag & KV_FLG_LIT) { + pv = pKVIdx->idx + pKVIdx->nCol; + } else if (pRow->flag & KV_FLG_MID) { + pv = pKVIdx->idx + (pKVIdx->nCol << 1); + } else { + pv = pKVIdx->idx + (pKVIdx->nCol << 2); + } + + for (uint8_t iKey = 0; iKey < key->numOfKeys; iKey++) { + int32_t index; + uint8_t *pData; + STypeValue *pValue = &key->keys[iKey]; + + pEnd -= tGetI8(pEnd, &pValue->type, false); + pEnd -= tGetI32v(pEnd, &index, false); + + if (pRow->flag & KV_FLG_LIT) { + pData = pv + ((uint8_t *)pKVIdx->idx)[index]; + } else if (pRow->flag & KV_FLG_MID) { + pData = pv + ((uint16_t *)pKVIdx->idx)[index]; + } else { + pData = pv + ((uint32_t *)pKVIdx->idx)[index]; + } + + pData += tGetI16v(pData, NULL, true); + if (IS_VAR_DATA_TYPE(pValue->type)) { + pData += tGetU32v(pData, &pValue->value.nData, true); + pValue->value.pData = pData; + } else { + memcpy(&pValue->value.val, pData, TYPE_BYTES[pValue->type]); + } + } + } else { // Tuple format + uint8_t *pf; + uint8_t *pv; + uint32_t fixedLen; + uint32_t numOfCols; + + pEnd -= tGetU32v(pEnd, &numOfCols, false); + pEnd -= tGetU32v(pEnd, &fixedLen, false); + + switch (pRow->flag & (HAS_VALUE | HAS_NULL | HAS_NONE)) { + case (HAS_VALUE | HAS_NONE): + case (HAS_VALUE | HAS_NULL): + pf = pRow->data + BIT1_SIZE(numOfCols - 1); + pv = pf + fixedLen; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + pf = pRow->data + BIT2_SIZE(numOfCols - 1); + pv = pf + fixedLen; + break; + default: + ASSERTS(0, "invalid row format"); + } + + for (uint8_t iKey = 0; iKey < key->numOfKeys; iKey++) { + int32_t offset; + STypeValue *pValue = &key->keys[iKey]; + + pEnd -= tGetI8(pEnd, &pValue->type, false); + pEnd -= tGetI32v(pEnd, &offset, false); + + if (IS_VAR_DATA_TYPE(key->keys[iKey].type)) { + pValue->value.pData = pv + *(int32_t *)(pf + offset); + pValue->value.pData += tGetU32v(pValue->value.pData, &pValue->value.nData, true); + } else { + memcpy(&key->keys[iKey].value.val, pf + offset, TYPE_BYTES[key->keys[iKey].type]); + } + } + } + } +} + +#define T_COMPARE_SCALAR_VALUE(TYPE, V1, V2) \ + do { \ + if (*(TYPE *)(V1) < *(TYPE *)(V2)) { \ + return -1; \ + } else if (*(TYPE *)(V1) > *(TYPE *)(V2)) { \ + return 1; \ + } else { \ + return 0; \ + } \ + } while (0) + +static int32_t tTypeValueCmpr(const STypeValue *tv1, const STypeValue *tv2) { + ASSERT(tv1->type == tv2->type); + + switch (tv1->type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + T_COMPARE_SCALAR_VALUE(int8_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_SMALLINT: + T_COMPARE_SCALAR_VALUE(int16_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_INT: + T_COMPARE_SCALAR_VALUE(int32_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + T_COMPARE_SCALAR_VALUE(int64_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_FLOAT: + T_COMPARE_SCALAR_VALUE(float, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_DOUBLE: + T_COMPARE_SCALAR_VALUE(double, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_UTINYINT: + T_COMPARE_SCALAR_VALUE(uint8_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_USMALLINT: + T_COMPARE_SCALAR_VALUE(uint16_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_UINT: + T_COMPARE_SCALAR_VALUE(uint32_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_UBIGINT: + T_COMPARE_SCALAR_VALUE(uint64_t, &tv1->value.val, &tv2->value.val); + case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_BINARY: { + return strcmp((const char *)tv1->value.pData, (const char *)tv2->value.pData); + } + case TSDB_DATA_TYPE_NCHAR: { + int32_t ret = tasoUcs4Compare((TdUcs4 *)tv1->value.pData, (TdUcs4 *)tv2->value.pData, + tv1->value.nData < tv2->value.nData ? tv1->value.nData : tv2->value.nData); + return ret ? ret : (tv1->value.nData < tv2->value.nData ? -1 : (tv1->value.nData > tv2->value.nData ? 1 : 0)); + } + case TSDB_DATA_TYPE_VARBINARY: { + int32_t ret = memcmp(tv1->value.pData, tv2->value.pData, + tv1->value.nData < tv2->value.nData ? tv1->value.nData : tv2->value.nData); + return ret ? ret : (tv1->value.nData < tv2->value.nData ? -1 : (tv1->value.nData > tv2->value.nData ? 1 : 0)); + } + case TSDB_DATA_TYPE_DECIMAL: + ASSERT(0); + break; + default: + ASSERT(0); + } + + return 0; +} + +int32_t tRowKeyCmpr(const void *p1, const void *p2) { + SRowKey *key1 = (SRowKey *)p1; + SRowKey *key2 = (SRowKey *)p2; + + if (key1->ts < key2->ts) { + return -1; + } else if (key1->ts > key2->ts) { + return 1; + } + + for (uint8_t iKey = 0; iKey < key1->numOfKeys; iKey++) { + int32_t ret = tTypeValueCmpr(&key1->keys[iKey], &key2->keys[iKey]); + if (ret) return ret; + } + + return 0; +} + // STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { @@ -1245,18 +1429,18 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { // key if (isJson) { - n += tPutCStr(p ? p + n : p, pTagVal->pKey); + n += tPutCStr(p ? p + n : p, pTagVal->pKey, true); } else { - n += tPutI16v(p ? p + n : p, pTagVal->cid); + n += tPutI16v(p ? p + n : p, pTagVal->cid, true); ASSERTS(pTagVal->cid > 0, "Invalid tag cid:%" PRIi16, pTagVal->cid); } // type - n += tPutI8(p ? p + n : p, pTagVal->type); + n += tPutI8(p ? p + n : p, pTagVal->type, true); // value if (IS_VAR_DATA_TYPE(pTagVal->type)) { - n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); + n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData, true); } else { p = p ? p + n : p; n += tDataTypes[pTagVal->type].bytes; @@ -1270,17 +1454,17 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { // key if (isJson) { - n += tGetCStr(p + n, &pTagVal->pKey); + n += tGetCStr(p + n, &pTagVal->pKey, true); } else { - n += tGetI16v(p + n, &pTagVal->cid); + n += tGetI16v(p + n, &pTagVal->cid, true); } // type - n += tGetI8(p + n, &pTagVal->type); + n += tGetI8(p + n, &pTagVal->type, true); // value if (IS_VAR_DATA_TYPE(pTagVal->type)) { - n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); + n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData, true); } else { memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); n += tDataTypes[pTagVal->type].bytes; @@ -1510,7 +1694,7 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid) { offset = pTag->idx[iTag]; } - tPutI16v(p + offset, cid); + tPutI16v(p + offset, cid, true); } // STSchema ======================================== @@ -1571,10 +1755,10 @@ void tColDataDestroy(void *ph) { } } -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t cflag) { pColData->cid = cid; pColData->type = type; - pColData->smaOn = smaOn; + pColData->cflag = cflag; tColDataClear(pColData); } @@ -2751,7 +2935,7 @@ static int32_t tColDataMergeSortMerge(SColData *aColData, int32_t start, int32_t if (end > start) { aDstColData = taosMemoryCalloc(1, sizeof(SColData) * nColData); for (int c = 0; c < nColData; ++c) { - tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].smaOn); + tColDataInit(&aDstColData[c], aColData[c].cid, aColData[c].type, aColData[c].cflag); } if (aDstColData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3096,10 +3280,10 @@ _exit: int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { int32_t n = 0; - n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid); - n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type); - n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal); - n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag); + n += tPutI16v(pBuf ? pBuf + n : NULL, pColData->cid, true); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->type, true); + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nVal, true); + n += tPutI8(pBuf ? pBuf + n : NULL, pColData->flag, true); // bitmap switch (pColData->flag) { @@ -3123,7 +3307,7 @@ int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { if (pBuf) memcpy(pBuf + n, pColData->aOffset, pColData->nVal << 2); n += (pColData->nVal << 2); - n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData); + n += tPutI32v(pBuf ? pBuf + n : NULL, pColData->nData, true); if (pBuf) memcpy(pBuf + n, pColData->pData, pColData->nData); n += pColData->nData; } else { @@ -3138,10 +3322,10 @@ int32_t tPutColData(uint8_t *pBuf, SColData *pColData) { int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { int32_t n = 0; - n += tGetI16v(pBuf + n, &pColData->cid); - n += tGetI8(pBuf + n, &pColData->type); - n += tGetI32v(pBuf + n, &pColData->nVal); - n += tGetI8(pBuf + n, &pColData->flag); + n += tGetI16v(pBuf + n, &pColData->cid, true); + n += tGetI8(pBuf + n, &pColData->type, true); + n += tGetI32v(pBuf + n, &pColData->nVal, true); + n += tGetI8(pBuf + n, &pColData->flag, true); // bitmap switch (pColData->flag) { @@ -3165,7 +3349,7 @@ int32_t tGetColData(uint8_t *pBuf, SColData *pColData) { pColData->aOffset = (int32_t *)(pBuf + n); n += (pColData->nVal << 2); - n += tGetI32v(pBuf + n, &pColData->nData); + n += tGetI32v(pBuf + n, &pColData->nData, true); pColData->pData = pBuf + n; n += pColData->nData; } else { diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 9d8d5013fa..b2893ffc0c 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -75,6 +75,7 @@ typedef struct SBlkInfo SBlkInfo; typedef struct STsdbDataIter2 STsdbDataIter2; typedef struct STsdbFilterInfo STsdbFilterInfo; typedef struct STFileSystem STFileSystem; +typedef struct STsdbRowKey STsdbRowKey; #define TSDBROW_ROW_FMT ((int8_t)0x0) #define TSDBROW_COL_FMT ((int8_t)0x1) @@ -163,19 +164,20 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs); #define tBlockDataLastKey(PBLOCKDATA) TSDBROW_KEY(&tBlockDataLastRow(PBLOCKDATA)) #define tBlockDataGetColDataByIdx(PBLOCKDATA, IDX) (&(PBLOCKDATA)->aColData[IDX]) -int32_t tBlockDataCreate(SBlockData *pBlockData); -void tBlockDataDestroy(SBlockData *pBlockData); -int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); -void tBlockDataReset(SBlockData *pBlockData); -int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); -int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid); -int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); -void tBlockDataClear(SBlockData *pBlockData); -void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData); -int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], - int32_t aBufN[]); -int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); +int32_t tBlockDataCreate(SBlockData *pBlockData); +void tBlockDataDestroy(SBlockData *pBlockData); +int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, int16_t *aCid, int32_t nCid); +void tBlockDataReset(SBlockData *pBlockData); +int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, int8_t cflag, SColData **ppColData); +int32_t tBlockDataAppendRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +int32_t tBlockDataUpdateRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema); +int32_t tBlockDataTryUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, int64_t uid); +int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTSchema, int64_t uid); +void tBlockDataClear(SBlockData *pBlockData); +SColData *tBlockDataGetColData(SBlockData *pBlockData, int16_t cid); +int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], + int32_t aBufN[]); +int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]); // SDiskDataHdr int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr); int32_t tGetDiskDataHdr(uint8_t *p, void *ph); @@ -288,14 +290,6 @@ typedef struct { int32_t tsdbMerge(void *arg); -// tsdbDiskData ============================================================================================== -int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder); -void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder); -int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, - uint8_t calcSma); -int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder); -int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId); -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo); // tsdbDataIter.c ============================================================================================== #define TSDB_MEM_TABLE_DATA_ITER 0 #define TSDB_DATA_FILE_DATA_ITER 1 @@ -436,6 +430,11 @@ struct TSDBROW { }; }; +struct STsdbRowKey { + SRowKey rowkey; + int64_t version; +}; + struct SBlockIdx { int64_t suid; int64_t uid; @@ -453,7 +452,7 @@ struct SMapData { struct SBlockCol { int16_t cid; int8_t type; - int8_t smaOn; + int8_t cflag; int8_t flag; // HAS_NONE|HAS_NULL|HAS_VALUE int32_t szOrigin; // original column value size (only save for variant data type) int32_t szBitmap; // bitmap size, 0 only for flag == HAS_VAL @@ -561,6 +560,7 @@ struct SDiskDataHdr { int32_t szBlkCol; int32_t nRow; int8_t cmprAlg; + int8_t numPrimaryKeyCols; }; struct SDelFile { diff --git a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c index e1625c9ddb..0793ce6ea4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbDataFileRW.c @@ -247,15 +247,28 @@ _exit: return code; } +int32_t tBlockColAndColumnCmpr(const void *p1, const void *p2) { + const SBlockCol *pBlockCol = (const SBlockCol *)p1; + const STColumn *pColumn = (const STColumn *)p2; + + if (pBlockCol->cid < pColumn->colId) { + return -1; + } else if (pBlockCol->cid > pColumn->colId) { + return 1; + } else { + return 0; + } +} + int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRecord *record, SBlockData *bData, STSchema *pTSchema, int16_t cids[], int32_t ncid) { - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0; + int32_t lino = 0; + int32_t n = 0; + SDiskDataHdr hdr; + SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL]; - code = tBlockDataInit(bData, (TABLEID *)record, pTSchema, cids, ncid); - TSDB_CHECK_CODE(code, lino, _exit); - - // uid + version + tskey + // read key part code = tRealloc(&reader->config->bufArr[0], record->blockKeySize); TSDB_CHECK_CODE(code, lino, _exit); @@ -263,34 +276,125 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe 0); TSDB_CHECK_CODE(code, lino, _exit); - // hdr - SDiskDataHdr hdr[1]; - int32_t size = 0; + // decode header + n += tGetDiskDataHdr(reader->config->bufArr[0] + n, &hdr); - size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); + tBlockDataReset(bData); + bData->suid = hdr.suid; + bData->uid = hdr.uid; + bData->nRow = hdr.nRow; - ASSERT(hdr->delimiter == TSDB_FILE_DLMT); - ASSERT(record->uid == hdr->uid); - - bData->nRow = hdr->nRow; + // decode key part + for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) { + n += tGetBlockCol(reader->config->bufArr[0] + n, &primaryKeyBlockCols[i]); + } // uid - ASSERT(hdr->uid); + if (hdr.uid == 0) { + ASSERT(0); + } // version - code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szVer, TSDB_DATA_TYPE_BIGINT, hdr->cmprAlg, - (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr->nRow, &reader->config->bufArr[1]); + code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, + (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr.nRow, &reader->config->bufArr[1]); TSDB_CHECK_CODE(code, lino, _exit); - size += hdr->szVer; + n += hdr.szVer; // ts - code = tsdbDecmprData(reader->config->bufArr[0] + size, hdr->szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr->cmprAlg, - (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr->nRow, &reader->config->bufArr[1]); + code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, + (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &reader->config->bufArr[1]); TSDB_CHECK_CODE(code, lino, _exit); - size += hdr->szKey; + n += hdr.szKey; - ASSERT(size == record->blockKeySize); + // primary key columns + for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) { + SColData *pColData; + code = tBlockDataAddColData(bData, primaryKeyBlockCols[i].cid, primaryKeyBlockCols[i].type, + primaryKeyBlockCols[i].cflag, &pColData); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[0] + n, &primaryKeyBlockCols[i], hdr.cmprAlg, hdr.nRow, pColData, + &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + + n += (primaryKeyBlockCols[i].szBitmap + primaryKeyBlockCols[i].szOffset + primaryKeyBlockCols[i].szValue); + } + + ASSERT(n == record->blockKeySize); + + // regular columns load + bool blockColLoaded = false; + int32_t decodedBufferSize = 0; + SBlockCol blockCol = {.cid = 0}; + for (int32_t i = 0; i < ncid; i++) { + SColData *pColData = tBlockDataGetColData(bData, cids[i]); + if (pColData != NULL) continue; + + // load the column index if not loaded yet + if (!blockColLoaded) { + if (hdr.szBlkCol > 0) { + code = tRealloc(&reader->config->bufArr[0], hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], record->blockOffset + record->blockKeySize, + reader->config->bufArr[0], hdr.szBlkCol, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } + blockColLoaded = true; + } + + // search the column index + for (;;) { + if (blockCol.cid >= cids[i]) { + break; + } + + if (decodedBufferSize >= hdr.szBlkCol) { + blockCol.cid = INT16_MAX; + break; + } + + decodedBufferSize += tGetBlockCol(reader->config->bufArr[0] + decodedBufferSize, &blockCol); + } + + STColumn *pTColumn = + taosbsearch(&blockCol, pTSchema->columns, pTSchema->numOfCols, sizeof(STSchema), tBlockColAndColumnCmpr, TD_EQ); + ASSERT(pTColumn != NULL); + + code = tBlockDataAddColData(bData, cids[i], pTColumn->type, pTColumn->flags, &pColData); + TSDB_CHECK_CODE(code, lino, _exit); + + // fill the column data + if (blockCol.cid > cids[i]) { + // set as all NONE + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { // all NONE + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else if (blockCol.flag == HAS_NULL) { // all NULL + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + int32_t size1 = blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; + + code = tRealloc(&reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd[TSDB_FTYPE_DATA], + record->blockOffset + record->blockKeySize + hdr.szBlkCol + blockCol.offset, + reader->config->bufArr[1], size1, 0); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[1], &blockCol, hdr.cmprAlg, hdr.nRow, pColData, + &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +#if 0 // other columns if (bData->nColData > 0) { if (hdr->szBlkCol > 0) { @@ -386,6 +490,7 @@ int32_t tsdbDataFileReadBlockDataByColumn(SDataFileReader *reader, const SBrinRe } } } +#endif _exit: if (code) { @@ -877,7 +982,7 @@ static int32_t tsdbDataFileDoWriteBlockData(SDataFileWriter *writer, SBlockData // to .sma file for (int32_t i = 0; i < bData->nColData; ++i) { SColData *colData = bData->aColData + i; - if ((!colData->smaOn) || ((colData->flag & HAS_VALUE) == 0)) continue; + if ((colData->cflag & COL_SMA_ON) == 0 || ((colData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma[1] = {{.colId = colData->cid}}; tColDataCalcSMA[colData->type](colData, &sma->sum, &sma->max, &sma->min, &sma->numOfNull); diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c deleted file mode 100644 index ae9af11f5a..0000000000 --- a/source/dnode/vnode/src/tsdb/tsdbDiskData.c +++ /dev/null @@ -1,697 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tsdb.h" - -typedef struct SDiskColBuilder SDiskColBuilder; - -struct SDiskColBuilder { - int16_t cid; - int8_t type; - uint8_t cmprAlg; - uint8_t calcSma; - int8_t flag; - int32_t nVal; - uint8_t *pBitMap; - int32_t offset; - SCompressor *pOffC; - SCompressor *pValC; - SColumnDataAgg sma; - uint8_t minSet; - uint8_t maxSet; - uint8_t *aBuf[2]; -}; - -// SDiskData ================================================ -static int32_t tDiskDataDestroy(SDiskData *pDiskData) { - int32_t code = 0; - pDiskData->aDiskCol = taosArrayDestroy(pDiskData->aDiskCol); - return code; -} - -// SDiskColBuilder ================================================ -#define tDiskColBuilderCreate() \ - (SDiskColBuilder) { 0 } - -static int32_t tDiskColBuilderDestroy(SDiskColBuilder *pBuilder) { - int32_t code = 0; - - tFree(pBuilder->pBitMap); - if (pBuilder->pOffC) tCompressorDestroy(pBuilder->pOffC); - if (pBuilder->pValC) tCompressorDestroy(pBuilder->pValC); - for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { - tFree(pBuilder->aBuf[iBuf]); - } - - return code; -} - -static int32_t tDiskColBuilderInit(SDiskColBuilder *pBuilder, int16_t cid, int8_t type, uint8_t cmprAlg, - uint8_t calcSma) { - int32_t code = 0; - - pBuilder->cid = cid; - pBuilder->type = type; - pBuilder->cmprAlg = cmprAlg; - pBuilder->calcSma = IS_VAR_DATA_TYPE(type) ? 0 : calcSma; - pBuilder->flag = 0; - pBuilder->nVal = 0; - pBuilder->offset = 0; - - if (IS_VAR_DATA_TYPE(type)) { - if (pBuilder->pOffC == NULL && (code = tCompressorCreate(&pBuilder->pOffC))) return code; - code = tCompressStart(pBuilder->pOffC, TSDB_DATA_TYPE_INT, cmprAlg); - if (code) return code; - } - - if (pBuilder->pValC == NULL && (code = tCompressorCreate(&pBuilder->pValC))) return code; - code = tCompressStart(pBuilder->pValC, type, cmprAlg); - if (code) return code; - - if (pBuilder->calcSma) { - pBuilder->sma = (SColumnDataAgg){.colId = cid}; - pBuilder->minSet = 0; - pBuilder->maxSet = 0; - } - - return code; -} - -static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) { - int32_t code = 0; - - ASSERT(pBuilder->flag && pBuilder->flag != HAS_NONE); - - *pDiskCol = (SDiskCol){(SBlockCol){.cid = pBuilder->cid, - .type = pBuilder->type, - .smaOn = pBuilder->calcSma, - .flag = pBuilder->flag, - .szOrigin = 0, - .szBitmap = 0, - .szOffset = 0, - .szValue = 0, - .offset = 0}, - .pBit = NULL, .pOff = NULL, .pVal = NULL, .agg = pBuilder->sma}; - - if (pBuilder->flag == HAS_NULL) return code; - - // BITMAP - if (pBuilder->flag != HAS_VALUE) { - int32_t nBit; - if (pBuilder->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { - nBit = BIT2_SIZE(pBuilder->nVal); - } else { - nBit = BIT1_SIZE(pBuilder->nVal); - } - - code = tRealloc(&pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES); - if (code) return code; - - code = tRealloc(&pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); - if (code) return code; - - pDiskCol->bCol.szBitmap = - tsCompressTinyint(pBuilder->pBitMap, nBit, nBit, pBuilder->aBuf[0], nBit + COMP_OVERFLOW_BYTES, - pBuilder->cmprAlg, pBuilder->aBuf[1], nBit + COMP_OVERFLOW_BYTES); - pDiskCol->pBit = pBuilder->aBuf[0]; - } - - // OFFSET - if (IS_VAR_DATA_TYPE(pBuilder->type)) { - code = tCompressEnd(pBuilder->pOffC, &pDiskCol->pOff, &pDiskCol->bCol.szOffset, NULL); - if (code) return code; - } - - // VALUE - if (pBuilder->flag != (HAS_NULL | HAS_NONE)) { - code = tCompressEnd(pBuilder->pValC, &pDiskCol->pVal, &pDiskCol->bCol.szValue, &pDiskCol->bCol.szOrigin); - if (code) return code; - } - - return code; -} - -static FORCE_INLINE int32_t tDiskColPutValue(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - if (IS_VAR_DATA_TYPE(pColVal->type)) { - code = tCompress(pBuilder->pOffC, &pBuilder->offset, sizeof(int32_t)); - if (code) return code; - pBuilder->offset += pColVal->value.nData; - - code = tCompress(pBuilder->pValC, pColVal->value.pData, pColVal->value.nData); - if (code) return code; - } else { - code = tCompress(pBuilder->pValC, &pColVal->value.val, tDataTypes[pColVal->type].bytes); - if (code) return code; - } - - return code; -} -static FORCE_INLINE int32_t tDiskColAddVal00(SDiskColBuilder *pBuilder, SColVal *pColVal) { - pBuilder->flag = HAS_VALUE; - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal01(SDiskColBuilder *pBuilder, SColVal *pColVal) { - pBuilder->flag = HAS_NONE; - return 0; -} -static FORCE_INLINE int32_t tDiskColAddVal02(SDiskColBuilder *pBuilder, SColVal *pColVal) { - pBuilder->flag = HAS_NULL; - return 0; -} -static FORCE_INLINE int32_t tDiskColAddVal10(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - // bit map - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) return code; - - memset(pBuilder->pBitMap, 0, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - // value - pBuilder->flag |= HAS_VALUE; - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) return code; - } - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal12(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) return code; - - memset(pBuilder->pBitMap, 0, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - pBuilder->flag |= HAS_NULL; - - return code; -} -static FORCE_INLINE int32_t tDiskColAddVal20(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) return code; - - pBuilder->flag |= HAS_VALUE; - - memset(pBuilder->pBitMap, 0, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) return code; - } - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal21(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) return code; - - pBuilder->flag |= HAS_NONE; - - memset(pBuilder->pBitMap, 255, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - return code; -} -static FORCE_INLINE int32_t tDiskColAddVal30(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - pBuilder->flag |= HAS_VALUE; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) return code; - - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal)); - } - SET_BIT2(pBitMap, pBuilder->nVal, 2); - - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - - SColVal cv = COL_VAL_VALUE(pColVal->cid, pColVal->type, (SValue){0}); - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - code = tDiskColPutValue(pBuilder, &cv); - if (code) return code; - } - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal31(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - return code; -} -static FORCE_INLINE int32_t tDiskColAddVal32(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - return code; -} -static FORCE_INLINE int32_t tDiskColAddVal40(SDiskColBuilder *pBuilder, SColVal *pColVal) { - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal41(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - pBuilder->flag |= HAS_NONE; - - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) return code; - - memset(pBuilder->pBitMap, 255, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal42(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - pBuilder->flag |= HAS_NULL; - - int32_t nBit = BIT1_SIZE(pBuilder->nVal + 1); - code = tRealloc(&pBuilder->pBitMap, nBit); - if (code) return code; - - memset(pBuilder->pBitMap, 255, nBit); - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal50(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal51(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal52(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - pBuilder->flag |= HAS_NULL; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) return code; - - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 0); - } - SET_BIT2(pBitMap, pBuilder->nVal, 1); - - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal60(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 1); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal61(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - pBuilder->flag |= HAS_NONE; - - uint8_t *pBitMap = NULL; - code = tRealloc(&pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) return code; - - for (int32_t iVal = 0; iVal < pBuilder->nVal; iVal++) { - SET_BIT2(pBitMap, iVal, GET_BIT1(pBuilder->pBitMap, iVal) ? 2 : 1); - } - SET_BIT2(pBitMap, pBuilder->nVal, 0); - - tFree(pBuilder->pBitMap); - pBuilder->pBitMap = pBitMap; - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal62(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT1_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT1(pBuilder->pBitMap, pBuilder->nVal, 0); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal70(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 2); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal71(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 0); - - return tDiskColPutValue(pBuilder, pColVal); -} -static FORCE_INLINE int32_t tDiskColAddVal72(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - code = tRealloc(&pBuilder->pBitMap, BIT2_SIZE(pBuilder->nVal + 1)); - if (code) return code; - SET_BIT2(pBuilder->pBitMap, pBuilder->nVal, 1); - - return tDiskColPutValue(pBuilder, pColVal); -} -static int32_t (*tDiskColAddValImpl[8][3])(SDiskColBuilder *pBuilder, SColVal *pColVal) = { - {tDiskColAddVal00, tDiskColAddVal01, tDiskColAddVal02}, // 0 - {tDiskColAddVal10, NULL, tDiskColAddVal12}, // HAS_NONE - {tDiskColAddVal20, tDiskColAddVal21, NULL}, // HAS_NULL - {tDiskColAddVal30, tDiskColAddVal31, tDiskColAddVal32}, // HAS_NULL|HAS_NONE - {tDiskColAddVal40, tDiskColAddVal41, tDiskColAddVal42}, // HAS_VALUE - {tDiskColAddVal50, tDiskColAddVal51, tDiskColAddVal52}, // HAS_VALUE|HAS_NONE - {tDiskColAddVal60, tDiskColAddVal61, tDiskColAddVal62}, // HAS_VALUE|HAS_NULL - {tDiskColAddVal70, tDiskColAddVal71, tDiskColAddVal72} // HAS_VALUE|HAS_NULL|HAS_NONE -}; -// extern void (*tSmaUpdateImpl[])(SColumnDataAgg *pColAgg, SColVal *pColVal, uint8_t *minSet, uint8_t *maxSet); -static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { - int32_t code = 0; - - if (pBuilder->calcSma) { - if (COL_VAL_IS_VALUE(pColVal)) { - // tSmaUpdateImpl[pBuilder->type](&pBuilder->sma, pColVal, &pBuilder->minSet, &pBuilder->maxSet); - } else { - pBuilder->sma.numOfNull++; - } - } - - if (tDiskColAddValImpl[pBuilder->flag][pColVal->flag]) { - code = tDiskColAddValImpl[pBuilder->flag][pColVal->flag](pBuilder, pColVal); - if (code) return code; - } - - pBuilder->nVal++; - - return code; -} - -// SDiskDataBuilder ================================================ -int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { - int32_t code = 0; - - *ppBuilder = (SDiskDataBuilder *)taosMemoryCalloc(1, sizeof(SDiskDataBuilder)); - if (*ppBuilder == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } - - return code; -} - -void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { - if (pBuilder == NULL) return NULL; - - if (pBuilder->pUidC) tCompressorDestroy(pBuilder->pUidC); - if (pBuilder->pVerC) tCompressorDestroy(pBuilder->pVerC); - if (pBuilder->pKeyC) tCompressorDestroy(pBuilder->pKeyC); - - if (pBuilder->aBuilder) { - for (int32_t iBuilder = 0; iBuilder < taosArrayGetSize(pBuilder->aBuilder); iBuilder++) { - SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - tDiskColBuilderDestroy(pDCBuilder); - } - taosArrayDestroy(pBuilder->aBuilder); - } - for (int32_t iBuf = 0; iBuf < sizeof(pBuilder->aBuf) / sizeof(pBuilder->aBuf[0]); iBuf++) { - tFree(pBuilder->aBuf[iBuf]); - } - tDiskDataDestroy(&pBuilder->dd); - taosMemoryFree(pBuilder); - - return NULL; -} - -int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TABLEID *pId, uint8_t cmprAlg, - uint8_t calcSma) { - int32_t code = 0; - - ASSERT(pId->suid || pId->uid); - - pBuilder->suid = pId->suid; - pBuilder->uid = pId->uid; - pBuilder->nRow = 0; - pBuilder->cmprAlg = cmprAlg; - pBuilder->calcSma = calcSma; - pBuilder->bi = (SBlkInfo){.minUid = INT64_MAX, - .maxUid = INT64_MIN, - .minKey = TSKEY_MAX, - .maxKey = TSKEY_MIN, - .minVer = VERSION_MAX, - .maxVer = VERSION_MIN, - .minTKey = TSDBKEY_MAX, - .maxTKey = TSDBKEY_MIN}; - - if (pBuilder->pUidC == NULL && (code = tCompressorCreate(&pBuilder->pUidC))) return code; - code = tCompressStart(pBuilder->pUidC, TSDB_DATA_TYPE_BIGINT, cmprAlg); - if (code) return code; - - if (pBuilder->pVerC == NULL && (code = tCompressorCreate(&pBuilder->pVerC))) return code; - code = tCompressStart(pBuilder->pVerC, TSDB_DATA_TYPE_BIGINT, cmprAlg); - if (code) return code; - - if (pBuilder->pKeyC == NULL && (code = tCompressorCreate(&pBuilder->pKeyC))) return code; - code = tCompressStart(pBuilder->pKeyC, TSDB_DATA_TYPE_TIMESTAMP, cmprAlg); - if (code) return code; - - if (pBuilder->aBuilder == NULL) { - pBuilder->aBuilder = taosArrayInit(pTSchema->numOfCols - 1, sizeof(SDiskColBuilder)); - if (pBuilder->aBuilder == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } - } - - pBuilder->nBuilder = 0; - for (int32_t iCol = 1; iCol < pTSchema->numOfCols; iCol++) { - STColumn *pTColumn = &pTSchema->columns[iCol]; - - if (pBuilder->nBuilder >= taosArrayGetSize(pBuilder->aBuilder)) { - SDiskColBuilder dc = tDiskColBuilderCreate(); - if (taosArrayPush(pBuilder->aBuilder, &dc) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } - } - - SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, pBuilder->nBuilder); - - code = tDiskColBuilderInit(pDCBuilder, pTColumn->colId, pTColumn->type, cmprAlg, - (calcSma && (pTColumn->flags & COL_SMA_ON))); - if (code) return code; - - pBuilder->nBuilder++; - } - - return code; -} - -int32_t tDiskDataBuilderClear(SDiskDataBuilder *pBuilder) { - int32_t code = 0; - pBuilder->suid = 0; - pBuilder->uid = 0; - pBuilder->nRow = 0; - return code; -} - -int32_t tDiskDataAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, TABLEID *pId) { - int32_t code = 0; - - ASSERT(pBuilder->suid || pBuilder->uid); - ASSERT(pId->suid == pBuilder->suid); - - TSDBKEY kRow = TSDBROW_KEY(pRow); - if (tsdbKeyCmprFn(&pBuilder->bi.minTKey, &kRow) > 0) pBuilder->bi.minTKey = kRow; - if (tsdbKeyCmprFn(&pBuilder->bi.maxTKey, &kRow) < 0) pBuilder->bi.maxTKey = kRow; - - // uid - if (pBuilder->uid && pBuilder->uid != pId->uid) { - ASSERT(pBuilder->suid); - for (int32_t iRow = 0; iRow < pBuilder->nRow; iRow++) { - code = tCompress(pBuilder->pUidC, &pBuilder->uid, sizeof(int64_t)); - if (code) return code; - } - pBuilder->uid = 0; - } - if (pBuilder->uid == 0) { - code = tCompress(pBuilder->pUidC, &pId->uid, sizeof(int64_t)); - if (code) return code; - } - if (pBuilder->bi.minUid > pId->uid) pBuilder->bi.minUid = pId->uid; - if (pBuilder->bi.maxUid < pId->uid) pBuilder->bi.maxUid = pId->uid; - - // version - code = tCompress(pBuilder->pVerC, &kRow.version, sizeof(int64_t)); - if (code) return code; - if (pBuilder->bi.minVer > kRow.version) pBuilder->bi.minVer = kRow.version; - if (pBuilder->bi.maxVer < kRow.version) pBuilder->bi.maxVer = kRow.version; - - // TSKEY - code = tCompress(pBuilder->pKeyC, &kRow.ts, sizeof(int64_t)); - if (code) return code; - if (pBuilder->bi.minKey > kRow.ts) pBuilder->bi.minKey = kRow.ts; - if (pBuilder->bi.maxKey < kRow.ts) pBuilder->bi.maxKey = kRow.ts; - - STSDBRowIter iter = {0}; - tsdbRowIterOpen(&iter, pRow, pTSchema); - - SColVal *pColVal = tsdbRowIterNext(&iter); - for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { - SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - - while (pColVal && pColVal->cid < pDCBuilder->cid) { - pColVal = tsdbRowIterNext(&iter); - } - - if (pColVal && pColVal->cid == pDCBuilder->cid) { - code = tDiskColAddVal(pDCBuilder, pColVal); - if (code) return code; - pColVal = tsdbRowIterNext(&iter); - } else { - code = tDiskColAddVal(pDCBuilder, &COL_VAL_NONE(pDCBuilder->cid, pDCBuilder->type)); - if (code) return code; - } - } - pBuilder->nRow++; - - return code; -} - -int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, const SDiskData **ppDiskData, const SBlkInfo **ppBlkInfo) { - int32_t code = 0; - - ASSERT(pBuilder->nRow); - - *ppDiskData = NULL; - *ppBlkInfo = NULL; - - SDiskData *pDiskData = &pBuilder->dd; - // reset SDiskData - pDiskData->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, - .fmtVer = 0, - .suid = pBuilder->suid, - .uid = pBuilder->uid, - .szUid = 0, - .szVer = 0, - .szKey = 0, - .szBlkCol = 0, - .nRow = pBuilder->nRow, - .cmprAlg = pBuilder->cmprAlg}; - pDiskData->pUid = NULL; - pDiskData->pVer = NULL; - pDiskData->pKey = NULL; - - // UID - if (pBuilder->uid == 0) { - code = tCompressEnd(pBuilder->pUidC, &pDiskData->pUid, &pDiskData->hdr.szUid, NULL); - if (code) return code; - } - - // VERSION - code = tCompressEnd(pBuilder->pVerC, &pDiskData->pVer, &pDiskData->hdr.szVer, NULL); - if (code) return code; - - // TSKEY - code = tCompressEnd(pBuilder->pKeyC, &pDiskData->pKey, &pDiskData->hdr.szKey, NULL); - if (code) return code; - - // aDiskCol - if (pDiskData->aDiskCol) { - taosArrayClear(pDiskData->aDiskCol); - } else { - pDiskData->aDiskCol = taosArrayInit(pBuilder->nBuilder, sizeof(SDiskCol)); - if (pDiskData->aDiskCol == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } - } - - int32_t offset = 0; - for (int32_t iBuilder = 0; iBuilder < pBuilder->nBuilder; iBuilder++) { - SDiskColBuilder *pDCBuilder = (SDiskColBuilder *)taosArrayGet(pBuilder->aBuilder, iBuilder); - - if (pDCBuilder->flag == HAS_NONE) continue; - - SDiskCol dCol; - - code = tGnrtDiskCol(pDCBuilder, &dCol); - if (code) return code; - - dCol.bCol.offset = offset; - offset = offset + dCol.bCol.szBitmap + dCol.bCol.szOffset + dCol.bCol.szValue; - - if (taosArrayPush(pDiskData->aDiskCol, &dCol) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - return code; - } - - pDiskData->hdr.szBlkCol += tPutBlockCol(NULL, &dCol.bCol); - } - - *ppDiskData = pDiskData; - *ppBlkInfo = &pBuilder->bi; - return code; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index a796f5121c..51b8a4bcc4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -23,16 +23,16 @@ static int32_t tsdbFSToBinary(uint8_t *p, STsdbFS *pFS) { uint32_t nSet = taosArrayGetSize(pFS->aDFileSet); // version - n += tPutI8(p ? p + n : p, 0); + n += tPutI8(p ? p + n : p, 0, true); // SDelFile - n += tPutI8(p ? p + n : p, hasDel); + n += tPutI8(p ? p + n : p, hasDel, true); if (hasDel) { n += tPutDelFile(p ? p + n : p, pFS->pDelFile); } // SArray - n += tPutU32v(p ? p + n : p, nSet); + n += tPutU32v(p ? p + n : p, nSet, true); for (uint32_t iSet = 0; iSet < nSet; iSet++) { n += tPutDFileSet(p ? p + n : p, (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet)); } @@ -45,11 +45,11 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, STsdbFS *pFS) { int32_t n = 0; // version - n += tGetI8(pData + n, NULL); + n += tGetI8(pData + n, NULL, true); // SDelFile int8_t hasDel = 0; - n += tGetI8(pData + n, &hasDel); + n += tGetI8(pData + n, &hasDel, true); if (hasDel) { pFS->pDelFile = (SDelFile *)taosMemoryCalloc(1, sizeof(SDelFile)); if (pFS->pDelFile == NULL) { @@ -66,7 +66,7 @@ static int32_t tsdbBinaryToFS(uint8_t *pData, int64_t nData, STsdbFS *pFS) { // aDFileSet taosArrayClear(pFS->aDFileSet); uint32_t nSet = 0; - n += tGetU32v(pData + n, &nSet); + n += tGetU32v(pData + n, &nSet, true); for (uint32_t iSet = 0; iSet < nSet; iSet++) { SDFileSet fSet = {0}; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 3ee0c482a7..1a38296ded 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -19,9 +19,9 @@ int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pHeadFile->commitID); - n += tPutI64v(p ? p + n : p, pHeadFile->size); - n += tPutI64v(p ? p + n : p, pHeadFile->offset); + n += tPutI64v(p ? p + n : p, pHeadFile->commitID, true); + n += tPutI64v(p ? p + n : p, pHeadFile->size, true); + n += tPutI64v(p ? p + n : p, pHeadFile->offset, true); return n; } @@ -29,9 +29,9 @@ int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile) { static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { int32_t n = 0; - n += tGetI64v(p + n, &pHeadFile->commitID); - n += tGetI64v(p + n, &pHeadFile->size); - n += tGetI64v(p + n, &pHeadFile->offset); + n += tGetI64v(p + n, &pHeadFile->commitID, true); + n += tGetI64v(p + n, &pHeadFile->size, true); + n += tGetI64v(p + n, &pHeadFile->offset, true); return n; } @@ -39,8 +39,8 @@ static int32_t tGetHeadFile(uint8_t *p, SHeadFile *pHeadFile) { int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pDataFile->commitID); - n += tPutI64v(p ? p + n : p, pDataFile->size); + n += tPutI64v(p ? p + n : p, pDataFile->commitID, true); + n += tPutI64v(p ? p + n : p, pDataFile->size, true); return n; } @@ -48,8 +48,8 @@ int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile) { static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { int32_t n = 0; - n += tGetI64v(p + n, &pDataFile->commitID); - n += tGetI64v(p + n, &pDataFile->size); + n += tGetI64v(p + n, &pDataFile->commitID, true); + n += tGetI64v(p + n, &pDataFile->size, true); return n; } @@ -57,9 +57,9 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pSttFile->commitID); - n += tPutI64v(p ? p + n : p, pSttFile->size); - n += tPutI64v(p ? p + n : p, pSttFile->offset); + n += tPutI64v(p ? p + n : p, pSttFile->commitID, true); + n += tPutI64v(p ? p + n : p, pSttFile->size, true); + n += tPutI64v(p ? p + n : p, pSttFile->offset, true); return n; } @@ -67,9 +67,9 @@ int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) { static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t n = 0; - n += tGetI64v(p + n, &pSttFile->commitID); - n += tGetI64v(p + n, &pSttFile->size); - n += tGetI64v(p + n, &pSttFile->offset); + n += tGetI64v(p + n, &pSttFile->commitID, true); + n += tGetI64v(p + n, &pSttFile->size, true); + n += tGetI64v(p + n, &pSttFile->offset, true); return n; } @@ -77,8 +77,8 @@ static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pSmaFile->commitID); - n += tPutI64v(p ? p + n : p, pSmaFile->size); + n += tPutI64v(p ? p + n : p, pSmaFile->commitID, true); + n += tPutI64v(p ? p + n : p, pSmaFile->size, true); return n; } @@ -86,18 +86,18 @@ int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile) { static int32_t tGetSmaFile(uint8_t *p, SSmaFile *pSmaFile) { int32_t n = 0; - n += tGetI64v(p + n, &pSmaFile->commitID); - n += tGetI64v(p + n, &pSmaFile->size); + n += tGetI64v(p + n, &pSmaFile->commitID, true); + n += tGetI64v(p + n, &pSmaFile->size, true); return n; } // EXPOSED APIS ================================================== -static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { - const char* p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); - int32_t len = strlen(p1); +static char *getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t commitId, char fname[]) { + const char *p1 = tfsGetDiskPath(pTsdb->pVnode->pTfs, did); + int32_t len = strlen(p1); - char* p = memcpy(fname, p1, len); + char *p = memcpy(fname, p1, len); p += len; *(p++) = TD_DIRSEP[0]; @@ -125,25 +125,25 @@ static char* getFileNamePrefix(STsdb *pTsdb, SDiskID did, int32_t fid, uint64_t } void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pHeadF->commitID, fname); memcpy(p, ".head", 5); p[5] = 0; } void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pDataF->commitID, fname); memcpy(p, ".data", 5); p[5] = 0; } void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pSttF->commitID, fname); memcpy(p, ".stt", 4); p[4] = 0; } void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { - char* p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); + char *p = getFileNamePrefix(pTsdb, did, fid, pSmaF->commitID, fname); memcpy(p, ".sma", 4); p[4] = 0; } @@ -221,9 +221,9 @@ _err: int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t n = 0; - n += tPutI32v(p ? p + n : p, pSet->diskId.level); - n += tPutI32v(p ? p + n : p, pSet->diskId.id); - n += tPutI32v(p ? p + n : p, pSet->fid); + n += tPutI32v(p ? p + n : p, pSet->diskId.level, true); + n += tPutI32v(p ? p + n : p, pSet->diskId.id, true); + n += tPutI32v(p ? p + n : p, pSet->fid, true); // data n += tPutHeadFile(p ? p + n : p, pSet->pHeadF); @@ -231,7 +231,7 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); // stt - n += tPutU8(p ? p + n : p, pSet->nSttF); + n += tPutU8(p ? p + n : p, pSet->nSttF, true); for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { n += tPutSttFile(p ? p + n : p, pSet->aSttF[iStt]); } @@ -242,9 +242,9 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { int32_t n = 0; - n += tGetI32v(p + n, &pSet->diskId.level); - n += tGetI32v(p + n, &pSet->diskId.id); - n += tGetI32v(p + n, &pSet->fid); + n += tGetI32v(p + n, &pSet->diskId.level, true); + n += tGetI32v(p + n, &pSet->diskId.id, true); + n += tGetI32v(p + n, &pSet->fid, true); // head pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); @@ -271,7 +271,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { n += tGetSmaFile(p + n, pSet->pSmaF); // stt - n += tGetU8(p + n, &pSet->nSttF); + n += tGetU8(p + n, &pSet->nSttF, true); for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile)); if (pSet->aSttF[iStt] == NULL) { @@ -298,9 +298,9 @@ void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]) { int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pDelFile->commitID); - n += tPutI64v(p ? p + n : p, pDelFile->size); - n += tPutI64v(p ? p + n : p, pDelFile->offset); + n += tPutI64v(p ? p + n : p, pDelFile->commitID, true); + n += tPutI64v(p ? p + n : p, pDelFile->size, true); + n += tPutI64v(p ? p + n : p, pDelFile->offset, true); return n; } @@ -308,9 +308,9 @@ int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile) { int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile) { int32_t n = 0; - n += tGetI64v(p + n, &pDelFile->commitID); - n += tGetI64v(p + n, &pDelFile->size); - n += tGetI64v(p + n, &pDelFile->offset); + n += tGetI64v(p + n, &pDelFile->commitID, true); + n += tGetI64v(p + n, &pDelFile->size, true); + n += tGetI64v(p + n, &pDelFile->offset, true); return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c index f26c6540df..6db52a9e54 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c +++ b/source/dnode/vnode/src/tsdb/tsdbSttFileRW.c @@ -191,35 +191,156 @@ _exit: return code; } +extern int32_t tBlockColAndColumnCmpr(const void *p1, const void *p2); + int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk *sttBlk, SBlockData *bData, STSchema *pTSchema, int16_t cids[], int32_t ncid) { - int32_t code = 0; - int32_t lino = 0; + int32_t code = 0; + int32_t lino = 0; + int32_t n = 0; + SDiskDataHdr hdr; + SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL]; - TABLEID tbid = {.suid = sttBlk->suid}; - if (tbid.suid == 0) { - tbid.uid = sttBlk->minUid; - } else { - tbid.uid = 0; - } - - code = tBlockDataInit(bData, &tbid, pTSchema, cids, ncid); - TSDB_CHECK_CODE(code, lino, _exit); - - // uid + version + tskey + // load key part code = tRealloc(&reader->config->bufArr[0], sttBlk->bInfo.szKey); TSDB_CHECK_CODE(code, lino, _exit); code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset, reader->config->bufArr[0], sttBlk->bInfo.szKey, 0); TSDB_CHECK_CODE(code, lino, _exit); + // decode header + n += tGetDiskDataHdr(reader->config->bufArr[0] + n, &hdr); + + ASSERT(hdr.delimiter == TSDB_FILE_DLMT); + + // set data container + tBlockDataReset(bData); + bData->suid = hdr.suid; + bData->uid = (sttBlk->suid == 0) ? sttBlk->minUid : 0; + bData->nRow = hdr.nRow; + + // decode primary key column indices + for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) { + n += tGetBlockCol(reader->config->bufArr[0] + n, primaryKeyBlockCols + i); + } + + // uid + if (hdr.uid == 0) { + ASSERT(hdr.szUid); + code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, + (uint8_t **)&bData->aUid, sizeof(int64_t) * hdr.nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + ASSERT(hdr.szUid == 0); + } + n += hdr.szUid; + + // version + code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, + (uint8_t **)&bData->aVersion, sizeof(int64_t) * hdr.nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + n += hdr.szVer; + + // ts + code = tsdbDecmprData(reader->config->bufArr[0] + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, + (uint8_t **)&bData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + n += hdr.szKey; + + // decode primary key columns + for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) { + SColData *pColData; + + code = tBlockDataAddColData(bData, primaryKeyBlockCols[i].cid, primaryKeyBlockCols[i].type, + primaryKeyBlockCols[i].cflag, &pColData); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[0] + n, &primaryKeyBlockCols[i], hdr.cmprAlg, hdr.nRow, pColData, + &reader->config->bufArr[1]); + TSDB_CHECK_CODE(code, lino, _exit); + + n += (primaryKeyBlockCols[i].szBitmap + primaryKeyBlockCols[i].szOffset + primaryKeyBlockCols[i].szValue); + } + + ASSERT(n == sttBlk->bInfo.szKey); + + // regular columns load + bool blockColLoaded = false; + int32_t decodedBufferSize = 0; + SBlockCol blockCol = {.cid = 0}; + for (int32_t i = 0; i < ncid; i++) { + SColData *pColData = tBlockDataGetColData(bData, cids[i]); + if (pColData != NULL) continue; + + // load the column index if not loaded yet + if (!blockColLoaded) { + if (hdr.szBlkCol > 0) { + code = tRealloc(&reader->config->bufArr[0], hdr.szBlkCol); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey, reader->config->bufArr[0], + hdr.szBlkCol, 0); + TSDB_CHECK_CODE(code, lino, _exit); + } + blockColLoaded = true; + } + + // search the column index + for (;;) { + if (blockCol.cid >= cids[i]) { + break; + } + + if (decodedBufferSize >= hdr.szBlkCol) { + blockCol.cid = INT16_MAX; + break; + } + + decodedBufferSize += tGetBlockCol(reader->config->bufArr[0] + decodedBufferSize, &blockCol); + } + + STColumn *pTColumn = + taosbsearch(&blockCol, pTSchema->columns, pTSchema->numOfCols, sizeof(STSchema), tBlockColAndColumnCmpr, TD_EQ); + ASSERT(pTColumn != NULL); + + code = tBlockDataAddColData(bData, cids[i], pTColumn->type, pTColumn->flags, &pColData); + TSDB_CHECK_CODE(code, lino, _exit); + + // fill the column data + if (blockCol.cid > cids[i]) { + // set as all NONE + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { // all NONE + code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else if (blockCol.flag == HAS_NULL) { // all NULL + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + TSDB_CHECK_CODE(code, lino, _exit); + } + } else { + int32_t size1 = blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; + + code = tRealloc(&reader->config->bufArr[1], size1); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbReadFile(reader->fd, sttBlk->bInfo.offset + sttBlk->bInfo.szKey + hdr.szBlkCol + blockCol.offset, + reader->config->bufArr[1], size1, 0); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tsdbDecmprColData(reader->config->bufArr[1], &blockCol, hdr.cmprAlg, hdr.nRow, pColData, + &reader->config->bufArr[2]); + TSDB_CHECK_CODE(code, lino, _exit); + } + } + +#if 0 // hdr SDiskDataHdr hdr[1]; int32_t size = 0; size += tGetDiskDataHdr(reader->config->bufArr[0] + size, hdr); - ASSERT(hdr->delimiter == TSDB_FILE_DLMT); bData->nRow = hdr->nRow; bData->uid = hdr->uid; @@ -307,6 +428,7 @@ int32_t tsdbSttFileReadBlockDataByColumn(SSttFileReader *reader, const SSttBlk * } } } +#endif _exit: if (code) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 87917cd243..04140d5428 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -123,15 +123,15 @@ _exit: int32_t tPutMapData(uint8_t *p, SMapData *pMapData) { int32_t n = 0; - n += tPutI32v(p ? p + n : p, pMapData->nItem); + n += tPutI32v(p ? p + n : p, pMapData->nItem, true); if (pMapData->nItem) { int32_t lOffset = 0; for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) { - n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset); + n += tPutI32v(p ? p + n : p, pMapData->aOffset[iItem] - lOffset, true); lOffset = pMapData->aOffset[iItem]; } - n += tPutI32v(p ? p + n : p, pMapData->nData); + n += tPutI32v(p ? p + n : p, pMapData->nData, true); if (p) { memcpy(p + n, pMapData->pData, pMapData->nData); } @@ -147,18 +147,18 @@ int32_t tGetMapData(uint8_t *p, SMapData *pMapData) { tMapDataReset(pMapData); - n += tGetI32v(p + n, &pMapData->nItem); + n += tGetI32v(p + n, &pMapData->nItem, true); if (pMapData->nItem) { if (tRealloc((uint8_t **)&pMapData->aOffset, sizeof(int32_t) * pMapData->nItem)) return -1; int32_t lOffset = 0; for (int32_t iItem = 0; iItem < pMapData->nItem; iItem++) { - n += tGetI32v(p + n, &pMapData->aOffset[iItem]); + n += tGetI32v(p + n, &pMapData->aOffset[iItem], true); pMapData->aOffset[iItem] += lOffset; lOffset = pMapData->aOffset[iItem]; } - n += tGetI32v(p + n, &pMapData->nData); + n += tGetI32v(p + n, &pMapData->nData, true); if (tRealloc(&pMapData->pData, pMapData->nData)) return -1; memcpy(pMapData->pData, p + n, pMapData->nData); n += pMapData->nData; @@ -192,10 +192,10 @@ int32_t tPutBlockIdx(uint8_t *p, void *ph) { int32_t n = 0; SBlockIdx *pBlockIdx = (SBlockIdx *)ph; - n += tPutI64(p ? p + n : p, pBlockIdx->suid); - n += tPutI64(p ? p + n : p, pBlockIdx->uid); - n += tPutI64v(p ? p + n : p, pBlockIdx->offset); - n += tPutI64v(p ? p + n : p, pBlockIdx->size); + n += tPutI64(p ? p + n : p, pBlockIdx->suid, true); + n += tPutI64(p ? p + n : p, pBlockIdx->uid, true); + n += tPutI64v(p ? p + n : p, pBlockIdx->offset, true); + n += tPutI64v(p ? p + n : p, pBlockIdx->size, true); return n; } @@ -204,10 +204,10 @@ int32_t tGetBlockIdx(uint8_t *p, void *ph) { int32_t n = 0; SBlockIdx *pBlockIdx = (SBlockIdx *)ph; - n += tGetI64(p + n, &pBlockIdx->suid); - n += tGetI64(p + n, &pBlockIdx->uid); - n += tGetI64v(p + n, &pBlockIdx->offset); - n += tGetI64v(p + n, &pBlockIdx->size); + n += tGetI64(p + n, &pBlockIdx->suid, true); + n += tGetI64(p + n, &pBlockIdx->uid, true); + n += tGetI64v(p + n, &pBlockIdx->offset, true); + n += tGetI64v(p + n, &pBlockIdx->size, true); return n; } @@ -259,23 +259,23 @@ int32_t tPutDataBlk(uint8_t *p, void *ph) { int32_t n = 0; SDataBlk *pDataBlk = (SDataBlk *)ph; - n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version); - n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts); - n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version); - n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts); - n += tPutI64v(p ? p + n : p, pDataBlk->minVer); - n += tPutI64v(p ? p + n : p, pDataBlk->maxVer); - n += tPutI32v(p ? p + n : p, pDataBlk->nRow); - n += tPutI8(p ? p + n : p, pDataBlk->hasDup); - n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock); + n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version, true); + n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts, true); + n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version, true); + n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts, true); + n += tPutI64v(p ? p + n : p, pDataBlk->minVer, true); + n += tPutI64v(p ? p + n : p, pDataBlk->maxVer, true); + n += tPutI32v(p ? p + n : p, pDataBlk->nRow, true); + n += tPutI8(p ? p + n : p, pDataBlk->hasDup, true); + n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock, true); for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { - n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset); - n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock); - n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey); + n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset, true); + n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock, true); + n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey, true); } if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { - n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset); - n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size); + n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset, true); + n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size, true); } return n; @@ -285,23 +285,23 @@ int32_t tGetDataBlk(uint8_t *p, void *ph) { int32_t n = 0; SDataBlk *pDataBlk = (SDataBlk *)ph; - n += tGetI64v(p + n, &pDataBlk->minKey.version); - n += tGetI64v(p + n, &pDataBlk->minKey.ts); - n += tGetI64v(p + n, &pDataBlk->maxKey.version); - n += tGetI64v(p + n, &pDataBlk->maxKey.ts); - n += tGetI64v(p + n, &pDataBlk->minVer); - n += tGetI64v(p + n, &pDataBlk->maxVer); - n += tGetI32v(p + n, &pDataBlk->nRow); - n += tGetI8(p + n, &pDataBlk->hasDup); - n += tGetI8(p + n, &pDataBlk->nSubBlock); + n += tGetI64v(p + n, &pDataBlk->minKey.version, true); + n += tGetI64v(p + n, &pDataBlk->minKey.ts, true); + n += tGetI64v(p + n, &pDataBlk->maxKey.version, true); + n += tGetI64v(p + n, &pDataBlk->maxKey.ts, true); + n += tGetI64v(p + n, &pDataBlk->minVer, true); + n += tGetI64v(p + n, &pDataBlk->maxVer, true); + n += tGetI32v(p + n, &pDataBlk->nRow, true); + n += tGetI8(p + n, &pDataBlk->hasDup, true); + n += tGetI8(p + n, &pDataBlk->nSubBlock, true); for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { - n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset); - n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock); - n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey); + n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset, true); + n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock, true); + n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey, true); } if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { - n += tGetI64v(p + n, &pDataBlk->smaInfo.offset); - n += tGetI32v(p + n, &pDataBlk->smaInfo.size); + n += tGetI64v(p + n, &pDataBlk->smaInfo.offset, true); + n += tGetI32v(p + n, &pDataBlk->smaInfo.size, true); } else { pDataBlk->smaInfo.offset = 0; pDataBlk->smaInfo.size = 0; @@ -335,17 +335,17 @@ int32_t tPutSttBlk(uint8_t *p, void *ph) { int32_t n = 0; SSttBlk *pSttBlk = (SSttBlk *)ph; - n += tPutI64(p ? p + n : p, pSttBlk->suid); - n += tPutI64(p ? p + n : p, pSttBlk->minUid); - n += tPutI64(p ? p + n : p, pSttBlk->maxUid); - n += tPutI64v(p ? p + n : p, pSttBlk->minKey); - n += tPutI64v(p ? p + n : p, pSttBlk->maxKey); - n += tPutI64v(p ? p + n : p, pSttBlk->minVer); - n += tPutI64v(p ? p + n : p, pSttBlk->maxVer); - n += tPutI32v(p ? p + n : p, pSttBlk->nRow); - n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset); - n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock); - n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey); + n += tPutI64(p ? p + n : p, pSttBlk->suid, true); + n += tPutI64(p ? p + n : p, pSttBlk->minUid, true); + n += tPutI64(p ? p + n : p, pSttBlk->maxUid, true); + n += tPutI64v(p ? p + n : p, pSttBlk->minKey, true); + n += tPutI64v(p ? p + n : p, pSttBlk->maxKey, true); + n += tPutI64v(p ? p + n : p, pSttBlk->minVer, true); + n += tPutI64v(p ? p + n : p, pSttBlk->maxVer, true); + n += tPutI32v(p ? p + n : p, pSttBlk->nRow, true); + n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset, true); + n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock, true); + n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey, true); return n; } @@ -354,17 +354,17 @@ int32_t tGetSttBlk(uint8_t *p, void *ph) { int32_t n = 0; SSttBlk *pSttBlk = (SSttBlk *)ph; - n += tGetI64(p + n, &pSttBlk->suid); - n += tGetI64(p + n, &pSttBlk->minUid); - n += tGetI64(p + n, &pSttBlk->maxUid); - n += tGetI64v(p + n, &pSttBlk->minKey); - n += tGetI64v(p + n, &pSttBlk->maxKey); - n += tGetI64v(p + n, &pSttBlk->minVer); - n += tGetI64v(p + n, &pSttBlk->maxVer); - n += tGetI32v(p + n, &pSttBlk->nRow); - n += tGetI64v(p + n, &pSttBlk->bInfo.offset); - n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock); - n += tGetI32v(p + n, &pSttBlk->bInfo.szKey); + n += tGetI64(p + n, &pSttBlk->suid, true); + n += tGetI64(p + n, &pSttBlk->minUid, true); + n += tGetI64(p + n, &pSttBlk->maxUid, true); + n += tGetI64v(p + n, &pSttBlk->minKey, true); + n += tGetI64v(p + n, &pSttBlk->maxKey, true); + n += tGetI64v(p + n, &pSttBlk->minVer, true); + n += tGetI64v(p + n, &pSttBlk->maxVer, true); + n += tGetI32v(p + n, &pSttBlk->nRow, true); + n += tGetI64v(p + n, &pSttBlk->bInfo.offset, true); + n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock, true); + n += tGetI32v(p + n, &pSttBlk->bInfo.szKey, true); return n; } @@ -376,26 +376,26 @@ int32_t tPutBlockCol(uint8_t *p, void *ph) { ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); - n += tPutI16v(p ? p + n : p, pBlockCol->cid); - n += tPutI8(p ? p + n : p, pBlockCol->type); - n += tPutI8(p ? p + n : p, pBlockCol->smaOn); - n += tPutI8(p ? p + n : p, pBlockCol->flag); - n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin); + n += tPutI16v(p ? p + n : p, pBlockCol->cid, true); + n += tPutI8(p ? p + n : p, pBlockCol->type, true); + n += tPutI8(p ? p + n : p, pBlockCol->cflag, true); + n += tPutI8(p ? p + n : p, pBlockCol->flag, true); + n += tPutI32v(p ? p + n : p, pBlockCol->szOrigin, true); if (pBlockCol->flag != HAS_NULL) { if (pBlockCol->flag != HAS_VALUE) { - n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap); + n += tPutI32v(p ? p + n : p, pBlockCol->szBitmap, true); } if (IS_VAR_DATA_TYPE(pBlockCol->type)) { - n += tPutI32v(p ? p + n : p, pBlockCol->szOffset); + n += tPutI32v(p ? p + n : p, pBlockCol->szOffset, true); } if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { - n += tPutI32v(p ? p + n : p, pBlockCol->szValue); + n += tPutI32v(p ? p + n : p, pBlockCol->szValue, true); } - n += tPutI32v(p ? p + n : p, pBlockCol->offset); + n += tPutI32v(p ? p + n : p, pBlockCol->offset, true); } _exit: @@ -406,11 +406,11 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) { int32_t n = 0; SBlockCol *pBlockCol = (SBlockCol *)ph; - n += tGetI16v(p + n, &pBlockCol->cid); - n += tGetI8(p + n, &pBlockCol->type); - n += tGetI8(p + n, &pBlockCol->smaOn); - n += tGetI8(p + n, &pBlockCol->flag); - n += tGetI32v(p + n, &pBlockCol->szOrigin); + n += tGetI16v(p + n, &pBlockCol->cid, true); + n += tGetI8(p + n, &pBlockCol->type, true); + n += tGetI8(p + n, &pBlockCol->cflag, true); + n += tGetI8(p + n, &pBlockCol->flag, true); + n += tGetI32v(p + n, &pBlockCol->szOrigin, true); ASSERT(pBlockCol->flag && (pBlockCol->flag != HAS_NONE)); @@ -421,18 +421,18 @@ int32_t tGetBlockCol(uint8_t *p, void *ph) { if (pBlockCol->flag != HAS_NULL) { if (pBlockCol->flag != HAS_VALUE) { - n += tGetI32v(p + n, &pBlockCol->szBitmap); + n += tGetI32v(p + n, &pBlockCol->szBitmap, true); } if (IS_VAR_DATA_TYPE(pBlockCol->type)) { - n += tGetI32v(p + n, &pBlockCol->szOffset); + n += tGetI32v(p + n, &pBlockCol->szOffset, true); } if (pBlockCol->flag != (HAS_NULL | HAS_NONE)) { - n += tGetI32v(p + n, &pBlockCol->szValue); + n += tGetI32v(p + n, &pBlockCol->szValue, true); } - n += tGetI32v(p + n, &pBlockCol->offset); + n += tGetI32v(p + n, &pBlockCol->offset, true); } return n; @@ -472,10 +472,10 @@ int32_t tPutDelIdx(uint8_t *p, void *ph) { SDelIdx *pDelIdx = (SDelIdx *)ph; int32_t n = 0; - n += tPutI64(p ? p + n : p, pDelIdx->suid); - n += tPutI64(p ? p + n : p, pDelIdx->uid); - n += tPutI64v(p ? p + n : p, pDelIdx->offset); - n += tPutI64v(p ? p + n : p, pDelIdx->size); + n += tPutI64(p ? p + n : p, pDelIdx->suid, true); + n += tPutI64(p ? p + n : p, pDelIdx->uid, true); + n += tPutI64v(p ? p + n : p, pDelIdx->offset, true); + n += tPutI64v(p ? p + n : p, pDelIdx->size, true); return n; } @@ -484,10 +484,10 @@ int32_t tGetDelIdx(uint8_t *p, void *ph) { SDelIdx *pDelIdx = (SDelIdx *)ph; int32_t n = 0; - n += tGetI64(p + n, &pDelIdx->suid); - n += tGetI64(p + n, &pDelIdx->uid); - n += tGetI64v(p + n, &pDelIdx->offset); - n += tGetI64v(p + n, &pDelIdx->size); + n += tGetI64(p + n, &pDelIdx->suid, true); + n += tGetI64(p + n, &pDelIdx->uid, true); + n += tGetI64v(p + n, &pDelIdx->offset, true); + n += tGetI64v(p + n, &pDelIdx->size, true); return n; } @@ -497,9 +497,9 @@ int32_t tPutDelData(uint8_t *p, void *ph) { SDelData *pDelData = (SDelData *)ph; int32_t n = 0; - n += tPutI64v(p ? p + n : p, pDelData->version); - n += tPutI64(p ? p + n : p, pDelData->sKey); - n += tPutI64(p ? p + n : p, pDelData->eKey); + n += tPutI64v(p ? p + n : p, pDelData->version, true); + n += tPutI64(p ? p + n : p, pDelData->sKey, true); + n += tPutI64(p ? p + n : p, pDelData->eKey, true); return n; } @@ -508,9 +508,9 @@ int32_t tGetDelData(uint8_t *p, void *ph) { SDelData *pDelData = (SDelData *)ph; int32_t n = 0; - n += tGetI64v(p + n, &pDelData->version); - n += tGetI64(p + n, &pDelData->sKey); - n += tGetI64(p + n, &pDelData->eKey); + n += tGetI64v(p + n, &pDelData->version, true); + n += tGetI64(p + n, &pDelData->sKey, true); + n += tGetI64(p + n, &pDelData->eKey, true); return n; } @@ -575,9 +575,7 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * if (pRow->type == TSDBROW_ROW_FMT) { tRowGet(pRow->pTSRow, pTSchema, iCol, pColVal); } else if (pRow->type == TSDBROW_COL_FMT) { - SColData *pColData; - - tBlockDataGetColData(pRow->pBlockData, pTColumn->colId, &pColData); + SColData *pColData = tBlockDataGetColData(pRow->pBlockData, pTColumn->colId); if (pColData) { tColDataGetValue(pColData, pRow->iRow, pColVal); @@ -589,8 +587,50 @@ void tsdbRowGetColVal(TSDBROW *pRow, STSchema *pTSchema, int32_t iCol, SColVal * } } +static void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key) { + if (row->type == TSDBROW_ROW_FMT) { + key->version = row->version; + tRowGetKey(row->pTSRow, &key->rowkey); + } else { + key->version = row->pBlockData->aVersion[row->iRow]; + key->rowkey.ts = row->pBlockData->aTSKEY[row->iRow]; + key->rowkey.numOfKeys = 0; + for (int32_t i = 0; i < row->pBlockData->nColData; i++) { + SColData *pColData = &row->pBlockData->aColData[i]; + if (pColData->cflag & COL_IS_KEY) { + SColVal cv; + tColDataGetValue(pColData, row->iRow, &cv); + key->rowkey.keys[key->rowkey.numOfKeys].type = pColData->type; + key->rowkey.keys[key->rowkey.numOfKeys].value = cv.value; + key->rowkey.numOfKeys++; + } else { + break; + } + } + } +} + +int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2) { + int32_t c = tRowKeyCmpr(&key1->rowkey, &key2->rowkey); + + if (c) { + return c; + } + + if (key1->version < key2->version) { + return -1; + } else if (key1->version > key2->version) { + return 1; + } + return 0; +} + int32_t tsdbRowCmprFn(const void *p1, const void *p2) { - return tsdbKeyCmprFn(&TSDBROW_KEY((TSDBROW *)p1), &TSDBROW_KEY((TSDBROW *)p2)); + STsdbRowKey key1, key2; + + tsdbRowGetKey((TSDBROW *)p1, &key1); + tsdbRowGetKey((TSDBROW *)p2, &key2); + return tsdbRowKeyCmpr(&key1, &key2); } // STSDBRowIter ====================================================== @@ -1126,8 +1166,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, continue; } - tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, - (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + tColDataInit(&pBlockData->aColData[iCid], pTColumn->colId, pTColumn->type, pTColumn->flags); iColumn++; pTColumn = (iColumn < pTSchema->numOfCols) ? &pTSchema->columns[iColumn] : NULL; @@ -1138,8 +1177,7 @@ int32_t tBlockDataInit(SBlockData *pBlockData, TABLEID *pId, STSchema *pTSchema, for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { STColumn *pTColumn = &pTSchema->columns[iColData + 1]; - tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, - (pTColumn->flags & COL_SMA_ON) ? 1 : 0); + tColDataInit(&pBlockData->aColData[iColData], pTColumn->colId, pTColumn->type, pTColumn->flags); } } @@ -1167,6 +1205,23 @@ void tBlockDataClear(SBlockData *pBlockData) { } } +int32_t tBlockDataAddColData(SBlockData *pBlockData, int16_t cid, int8_t type, int8_t cflag, SColData **ppColData) { + ASSERT(pBlockData->nColData == 0 || pBlockData->aColData[pBlockData->nColData - 1].cid < cid); + + SColData *newColData = taosMemoryRealloc(pBlockData->aColData, sizeof(SColData) * (pBlockData->nColData + 1)); + if (newColData == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pBlockData->aColData = newColData; + pBlockData->nColData++; + + *ppColData = &pBlockData->aColData[pBlockData->nColData - 1]; + tColDataInit(*ppColData, cid, type, cflag); + + return 0; +} + /* flag > 0: forward update * flag == 0: insert * flag < 0: backward update @@ -1287,7 +1342,7 @@ int32_t tBlockDataUpsertRow(SBlockData *pBlockData, TSDBROW *pRow, STSchema *pTS } } -void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColData) { +SColData *tBlockDataGetColData(SBlockData *pBlockData, int16_t cid) { ASSERT(cid != PRIMARYKEY_TIMESTAMP_COL_ID); int32_t lidx = 0; int32_t ridx = pBlockData->nColData - 1; @@ -1298,8 +1353,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); if (c == 0) { - *ppColData = pColData; - return; + return pColData; } else if (c < 0) { lidx = midx + 1; } else { @@ -1307,42 +1361,49 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD } } - *ppColData = NULL; + return NULL; } int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, int32_t *szOut, uint8_t *aBuf[], - int32_t aBufN[]) { - int32_t code = 0; - - SDiskDataHdr hdr = {.delimiter = TSDB_FILE_DLMT, - .fmtVer = 0, - .suid = pBlockData->suid, - .uid = pBlockData->uid, - .nRow = pBlockData->nRow, - .cmprAlg = cmprAlg}; + int32_t aBufSize[]) { + int32_t code = 0; + SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL]; + SDiskDataHdr hdr = { + .delimiter = TSDB_FILE_DLMT, + .fmtVer = 0, + .suid = pBlockData->suid, + .uid = pBlockData->uid, + .nRow = pBlockData->nRow, + .cmprAlg = cmprAlg, + }; // encode ================= // columns AND SBlockCol - aBufN[0] = 0; + aBufSize[0] = 0; for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); ASSERT(pColData->flag); if (pColData->flag == HAS_NONE) continue; + if (pColData->cflag & COL_IS_KEY) { + ASSERT(pColData->flag == HAS_VALUE); + hdr.fmtVer = 1; + continue; + } SBlockCol blockCol = {.cid = pColData->cid, .type = pColData->type, - .smaOn = pColData->smaOn, + .cflag = pColData->cflag, .flag = pColData->flag, .szOrigin = pColData->nData}; if (pColData->flag != HAS_NULL) { - code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], aBufN[0], &aBuf[2]); + code = tsdbCmprColData(pColData, cmprAlg, &blockCol, &aBuf[0], aBufSize[0], &aBuf[2]); if (code) goto _exit; - blockCol.offset = aBufN[0]; - aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; + blockCol.offset = aBufSize[0]; + aBufSize[0] = aBufSize[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; } code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); @@ -1351,46 +1412,74 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, } // SBlockCol - aBufN[1] = hdr.szBlkCol; + aBufSize[1] = hdr.szBlkCol; - // uid + version + tskey - aBufN[2] = 0; + aBufSize[2] = 0; + // uid if (pBlockData->uid == 0) { code = tsdbCmprData((uint8_t *)pBlockData->aUid, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, cmprAlg, - &aBuf[2], aBufN[2], &hdr.szUid, &aBuf[3]); + &aBuf[2], aBufSize[2], &hdr.szUid, &aBuf[3]); if (code) goto _exit; } - aBufN[2] += hdr.szUid; + aBufSize[2] += hdr.szUid; + // version code = tsdbCmprData((uint8_t *)pBlockData->aVersion, sizeof(int64_t) * pBlockData->nRow, TSDB_DATA_TYPE_BIGINT, - cmprAlg, &aBuf[2], aBufN[2], &hdr.szVer, &aBuf[3]); + cmprAlg, &aBuf[2], aBufSize[2], &hdr.szVer, &aBuf[3]); if (code) goto _exit; - aBufN[2] += hdr.szVer; + aBufSize[2] += hdr.szVer; + // ts code = tsdbCmprData((uint8_t *)pBlockData->aTSKEY, sizeof(TSKEY) * pBlockData->nRow, TSDB_DATA_TYPE_TIMESTAMP, - cmprAlg, &aBuf[2], aBufN[2], &hdr.szKey, &aBuf[3]); + cmprAlg, &aBuf[2], aBufSize[2], &hdr.szKey, &aBuf[3]); if (code) goto _exit; - aBufN[2] += hdr.szKey; + aBufSize[2] += hdr.szKey; + + // primary key columns + if (hdr.fmtVer == 1) { + for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); + if ((pColData->cflag & COL_IS_KEY) == 0) break; + + SBlockCol *pBlockCol = &primaryKeyBlockCols[hdr.numPrimaryKeyCols]; + pBlockCol->cid = pColData->cid; + pBlockCol->type = pColData->type; + pBlockCol->cflag = pColData->cflag; + pBlockCol->flag = pColData->flag; + pBlockCol->szOrigin = pColData->nData; + + code = tsdbCmprColData(pColData, cmprAlg, pBlockCol, &aBuf[2], aBufSize[2], &aBuf[3]); + if (code) goto _exit; + aBufSize[2] = aBufSize[2] + pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue; + + hdr.numPrimaryKeyCols++; + } + } // hdr - aBufN[3] = tPutDiskDataHdr(NULL, &hdr); - code = tRealloc(&aBuf[3], aBufN[3]); + aBufSize[3] = tPutDiskDataHdr(NULL, &hdr); + code = tRealloc(&aBuf[3], aBufSize[3]); if (code) goto _exit; tPutDiskDataHdr(aBuf[3], &hdr); + for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) { + code = tRealloc(&aBuf[3], aBufSize[3] + tPutBlockCol(NULL, &primaryKeyBlockCols[i])); + if (code) goto _exit; + aBufSize[3] += tPutBlockCol(aBuf[3] + aBufSize[3], &primaryKeyBlockCols[i]); + } // aggragate if (ppOut) { - *szOut = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; + *szOut = aBufSize[0] + aBufSize[1] + aBufSize[2] + aBufSize[3]; code = tRealloc(ppOut, *szOut); if (code) goto _exit; - memcpy(*ppOut, aBuf[3], aBufN[3]); - memcpy(*ppOut + aBufN[3], aBuf[2], aBufN[2]); - if (aBufN[1]) { - memcpy(*ppOut + aBufN[3] + aBufN[2], aBuf[1], aBufN[1]); + memcpy(*ppOut, aBuf[3], aBufSize[3]); + memcpy(*ppOut + aBufSize[3], aBuf[2], aBufSize[2]); + if (aBufSize[1]) { + memcpy(*ppOut + aBufSize[3] + aBufSize[2], aBuf[1], aBufSize[1]); } - if (aBufN[0]) { - memcpy(*ppOut + aBufN[3] + aBufN[2] + aBufN[1], aBuf[0], aBufN[0]); + if (aBufSize[0]) { + memcpy(*ppOut + aBufSize[3] + aBufSize[2] + aBufSize[1], aBuf[0], aBufSize[0]); } } @@ -1405,6 +1494,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin int32_t n = 0; SDiskDataHdr hdr = {0}; + SBlockCol primaryKeyBlockCols[TD_MAX_PRIMARY_KEY_COL]; // SDiskDataHdr n += tGetDiskDataHdr(pIn + n, &hdr); @@ -1414,6 +1504,11 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin pBlockData->uid = hdr.uid; pBlockData->nRow = hdr.nRow; + // primary SBlockCol + for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) { + n += tGetBlockCol(pIn + n, &primaryKeyBlockCols[i]); + } + // uid if (hdr.uid == 0) { ASSERT(hdr.szUid); @@ -1437,39 +1532,41 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin if (code) goto _exit; n += hdr.szKey; - // loop to decode each column data - if (hdr.szBlkCol == 0) goto _exit; + // Primary key columns + for (int32_t i = 0; i < hdr.numPrimaryKeyCols; i++) { + SColData *pColData; - int32_t nColData = 0; - int32_t nt = 0; - while (nt < hdr.szBlkCol) { - SBlockCol blockCol = {0}; - nt += tGetBlockCol(pIn + n + nt, &blockCol); - ++nColData; + code = tBlockDataAddColData(pBlockData, primaryKeyBlockCols[i].cid, primaryKeyBlockCols[i].type, + primaryKeyBlockCols[i].cflag, &pColData); + if (code) goto _exit; + + code = tsdbDecmprColData(pIn + n, &primaryKeyBlockCols[i], hdr.cmprAlg, hdr.nRow, pColData, &aBuf[0]); + if (code) goto _exit; + + n = n + primaryKeyBlockCols[i].szBitmap + primaryKeyBlockCols[i].szOffset + primaryKeyBlockCols[i].szValue; } - ASSERT(nt == hdr.szBlkCol); - code = tBlockDataAdjustColData(pBlockData, nColData); - if (code) goto _exit; + // regular key columns + if (hdr.szBlkCol > 0) { + int32_t nt = 0; + while (nt < hdr.szBlkCol) { + SColData *pColData; + SBlockCol blockCol = {0}; + nt += tGetBlockCol(pIn + n + nt, &blockCol); - nt = 0; - int32_t iColData = 0; - while (nt < hdr.szBlkCol) { - SBlockCol blockCol = {0}; - nt += tGetBlockCol(pIn + n + nt, &blockCol); + code = tBlockDataAddColData(pBlockData, blockCol.cid, blockCol.type, blockCol.cflag, &pColData); + if (code) goto _exit; - SColData *pColData = &pBlockData->aColData[iColData++]; - - tColDataInit(pColData, blockCol.cid, blockCol.type, blockCol.smaOn); - if (blockCol.flag == HAS_NULL) { - for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { - code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + if (blockCol.flag == HAS_NULL) { + for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) { + code = tColDataAppendValue(pColData, &COL_VAL_NULL(blockCol.cid, blockCol.type)); + if (code) goto _exit; + } + } else { + code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, + &aBuf[0]); if (code) goto _exit; } - } else { - code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, - &aBuf[0]); - if (code) goto _exit; } } @@ -1481,16 +1578,19 @@ _exit: int32_t tPutDiskDataHdr(uint8_t *p, const SDiskDataHdr *pHdr) { int32_t n = 0; - n += tPutU32(p ? p + n : p, pHdr->delimiter); - n += tPutU32v(p ? p + n : p, pHdr->fmtVer); - n += tPutI64(p ? p + n : p, pHdr->suid); - n += tPutI64(p ? p + n : p, pHdr->uid); - n += tPutI32v(p ? p + n : p, pHdr->szUid); - n += tPutI32v(p ? p + n : p, pHdr->szVer); - n += tPutI32v(p ? p + n : p, pHdr->szKey); - n += tPutI32v(p ? p + n : p, pHdr->szBlkCol); - n += tPutI32v(p ? p + n : p, pHdr->nRow); - n += tPutI8(p ? p + n : p, pHdr->cmprAlg); + n += tPutU32(p ? p + n : p, pHdr->delimiter, true); + n += tPutU32v(p ? p + n : p, pHdr->fmtVer, true); + n += tPutI64(p ? p + n : p, pHdr->suid, true); + n += tPutI64(p ? p + n : p, pHdr->uid, true); + n += tPutI32v(p ? p + n : p, pHdr->szUid, true); + n += tPutI32v(p ? p + n : p, pHdr->szVer, true); + n += tPutI32v(p ? p + n : p, pHdr->szKey, true); + n += tPutI32v(p ? p + n : p, pHdr->szBlkCol, true); + n += tPutI32v(p ? p + n : p, pHdr->nRow, true); + n += tPutI8(p ? p + n : p, pHdr->cmprAlg, true); + if (pHdr->fmtVer == 1) { + n += tPutI8(p ? p + n : p, pHdr->numPrimaryKeyCols, true); + } return n; } @@ -1499,16 +1599,21 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { int32_t n = 0; SDiskDataHdr *pHdr = (SDiskDataHdr *)ph; - n += tGetU32(p + n, &pHdr->delimiter); - n += tGetU32v(p + n, &pHdr->fmtVer); - n += tGetI64(p + n, &pHdr->suid); - n += tGetI64(p + n, &pHdr->uid); - n += tGetI32v(p + n, &pHdr->szUid); - n += tGetI32v(p + n, &pHdr->szVer); - n += tGetI32v(p + n, &pHdr->szKey); - n += tGetI32v(p + n, &pHdr->szBlkCol); - n += tGetI32v(p + n, &pHdr->nRow); - n += tGetI8(p + n, &pHdr->cmprAlg); + n += tGetU32(p + n, &pHdr->delimiter, true); + n += tGetU32v(p + n, &pHdr->fmtVer, true); + n += tGetI64(p + n, &pHdr->suid, true); + n += tGetI64(p + n, &pHdr->uid, true); + n += tGetI32v(p + n, &pHdr->szUid, true); + n += tGetI32v(p + n, &pHdr->szVer, true); + n += tGetI32v(p + n, &pHdr->szKey, true); + n += tGetI32v(p + n, &pHdr->szBlkCol, true); + n += tGetI32v(p + n, &pHdr->nRow, true); + n += tGetI8(p + n, &pHdr->cmprAlg, true); + if (pHdr->fmtVer == 1) { + n += tGetI8(p + n, &pHdr->numPrimaryKeyCols, true); + } else { + pHdr->numPrimaryKeyCols = 0; + } return n; } @@ -1517,11 +1622,11 @@ int32_t tGetDiskDataHdr(uint8_t *p, void *ph) { int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { int32_t n = 0; - n += tPutI16v(p ? p + n : p, pColAgg->colId); - n += tPutI16v(p ? p + n : p, pColAgg->numOfNull); - n += tPutI64(p ? p + n : p, pColAgg->sum); - n += tPutI64(p ? p + n : p, pColAgg->max); - n += tPutI64(p ? p + n : p, pColAgg->min); + n += tPutI16v(p ? p + n : p, pColAgg->colId, true); + n += tPutI16v(p ? p + n : p, pColAgg->numOfNull, true); + n += tPutI64(p ? p + n : p, pColAgg->sum, true); + n += tPutI64(p ? p + n : p, pColAgg->max, true); + n += tPutI64(p ? p + n : p, pColAgg->min, true); return n; } @@ -1529,11 +1634,11 @@ int32_t tPutColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { int32_t tGetColumnDataAgg(uint8_t *p, SColumnDataAgg *pColAgg) { int32_t n = 0; - n += tGetI16v(p + n, &pColAgg->colId); - n += tGetI16v(p + n, &pColAgg->numOfNull); - n += tGetI64(p + n, &pColAgg->sum); - n += tGetI64(p + n, &pColAgg->max); - n += tGetI64(p + n, &pColAgg->min); + n += tGetI16v(p + n, &pColAgg->colId, true); + n += tGetI16v(p + n, &pColAgg->numOfNull, true); + n += tGetI64(p + n, &pColAgg->sum, true); + n += tGetI64(p + n, &pColAgg->max, true); + n += tGetI64(p + n, &pColAgg->min, true); return n; } @@ -1656,7 +1761,7 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in ASSERT(pColData->cid == pBlockCol->cid); ASSERT(pColData->type == pBlockCol->type); - pColData->smaOn = pBlockCol->smaOn; + pColData->cflag = pBlockCol->cflag; pColData->flag = pBlockCol->flag; pColData->nVal = nVal; pColData->nData = pBlockCol->szOrigin; diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index 53fdc96c37..52146d2135 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -17,13 +17,13 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE #include "tcompare.h" +#include "osString.h" #include "regex.h" #include "tdef.h" #include "thash.h" #include "tlog.h" #include "tutil.h" #include "types.h" -#include "osString.h" int32_t setChkInBytes1(const void *pLeft, const void *pRight) { return NULL != taosHashGet((SHashObj *)pRight, pLeft, 1) ? 1 : 0; @@ -209,11 +209,11 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { int32_t len1 = varDataLen(pLeft); int32_t len2 = varDataLen(pRight); - int32_t ret = tasoUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1>len2 ? len2:len1); + int32_t ret = tasoUcs4Compare((TdUcs4 *)varDataVal(pLeft), (TdUcs4 *)varDataVal(pRight), len1 > len2 ? len2 : len1); if (ret == 0) { if (len1 > len2) return 1; - else if(len1 < len2) + else if (len1 < len2) return -1; else return 0; @@ -242,9 +242,7 @@ int32_t compareLenBinaryVal(const void *pLeft, const void *pRight) { } } -int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight) { - return compareLenBinaryVal(pRight, pLeft); -} +int32_t compareLenBinaryValDesc(const void *pLeft, const void *pRight) { return compareLenBinaryVal(pRight, pLeft); } // string > number > bool > null // ref: https://dev.mysql.com/doc/refman/8.0/en/json.html#json-comparison @@ -328,7 +326,7 @@ int32_t compareInt8Uint16(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint32(const void *pLeft, const void *pRight) { - int8_t left = GET_INT8_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); if (left < 0) return -1; uint32_t right = GET_UINT32_VAL(pRight); if ((uint32_t)left > right) return 1; @@ -337,7 +335,7 @@ int32_t compareInt8Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt8Uint64(const void *pLeft, const void *pRight) { - int8_t left = GET_INT8_VAL(pLeft); + int8_t left = GET_INT8_VAL(pLeft); if (left < 0) return -1; uint64_t right = GET_UINT64_VAL(pRight); if ((uint64_t)left > right) return 1; @@ -402,7 +400,7 @@ int32_t compareInt16Uint16(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint32(const void *pLeft, const void *pRight) { - int16_t left = GET_INT16_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); if (left < 0) return -1; uint32_t right = GET_UINT32_VAL(pRight); if ((uint32_t)left > right) return 1; @@ -411,7 +409,7 @@ int32_t compareInt16Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt16Uint64(const void *pLeft, const void *pRight) { - int16_t left = GET_INT16_VAL(pLeft); + int16_t left = GET_INT16_VAL(pLeft); if (left < 0) return -1; uint64_t right = GET_UINT64_VAL(pRight); if ((uint64_t)left > right) return 1; @@ -476,7 +474,7 @@ int32_t compareInt32Uint16(const void *pLeft, const void *pRight) { } int32_t compareInt32Uint32(const void *pLeft, const void *pRight) { - int32_t left = GET_INT32_VAL(pLeft); + int32_t left = GET_INT32_VAL(pLeft); if (left < 0) return -1; uint32_t right = GET_UINT32_VAL(pRight); if ((uint32_t)left > right) return 1; @@ -485,7 +483,7 @@ int32_t compareInt32Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt32Uint64(const void *pLeft, const void *pRight) { - int32_t left = GET_INT32_VAL(pLeft); + int32_t left = GET_INT32_VAL(pLeft); if (left < 0) return -1; uint64_t right = GET_UINT64_VAL(pRight); if ((uint64_t)left > right) return 1; @@ -558,7 +556,7 @@ int32_t compareInt64Uint32(const void *pLeft, const void *pRight) { } int32_t compareInt64Uint64(const void *pLeft, const void *pRight) { - int64_t left = GET_INT64_VAL(pLeft); + int64_t left = GET_INT64_VAL(pLeft); if (left < 0) return -1; uint64_t right = GET_UINT64_VAL(pRight); if ((uint64_t)left > right) return 1;