From cccd5dbd8bda0ad56d736add278419754a30aefa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 12 May 2022 05:28:08 +0000 Subject: [PATCH] refact data --- include/common/tdataformat.h | 3 + include/common/tmsg.h | 13 ++- source/common/src/tdataformat.c | 165 ++++++++++++++++++++++++-------- 3 files changed, 138 insertions(+), 43 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2e8a214a9d..b01b7ac240 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -38,6 +38,8 @@ typedef struct SKVIdx SKVIdx; // STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow); int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow); +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, + int8_t *flags); // STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); @@ -86,6 +88,7 @@ struct STSRowBuilder { uint8_t *pKVBuf; int32_t szTPBuf; uint8_t *pTPBuf; + uint8_t *pBitBuf; int32_t nCols; int32_t kvVLen; int32_t tpVLen; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 7f17c1673b..6294e14adb 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -271,12 +271,13 @@ typedef struct { int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp); int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp); -void tFreeSSubmitRsp(SSubmitRsp *pRsp); - -#define COL_SMA_ON ((int8_t)0x1) -#define COL_IDX_ON ((int8_t)0x2) -#define COL_VAL_SET ((int8_t)0x4) +void tFreeSSubmitRsp(SSubmitRsp* pRsp); +#define COL_SMA_ON ((int8_t)0x1) +#define COL_IDX_ON ((int8_t)0x2) +#define COL_SET_VAL ((int8_t)0x10) +#define COL_SET_NONE ((int8_t)0x20) +#define COL_SET_NULL ((int8_t)0x40) typedef struct SSchema { int8_t type; int8_t flags; @@ -285,6 +286,8 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; +#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0) + #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) #define SSCHMEA_TYPE(s) ((s)->type) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index ad020fe7d9..d492d98b63 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,13 +19,17 @@ #include "tdatablock.h" #include "tlog.h" -#define TD_KV_ROW 0x1U +#define TD_HAS_NONE 0x1U +#define TD_HAS_NULL 0x2U +#define TD_HAS_VAL 0x4U +#define TD_KV_ROW 0x10U struct SKVIdx { int32_t cid; int32_t offset; }; +// STSRow2 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) { if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1; if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1; @@ -50,6 +54,59 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) { return 0; } +int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData, + int8_t *flags) { + if (cid == 0) { + *ppData = (uint8_t *)&pRow->ts; + *nData = sizeof(TSKEY); + *flags = 0; + } else { + uint32_t tflags = pRow->flags & 0xf; + *ppData = NULL; + *nData = 0; + + switch (tflags) { + case TD_HAS_NONE: + *flags = -1; + break; + case TD_HAS_NULL: + *flags = 1; + break; + case TD_HAS_VAL: + *flags = 0; + // find the row + break; + case TD_HAS_NULL | TD_HAS_NONE: + // read bit map (todo) + if (0) { + *flags = 1; + } else { + *flags = -1; + } + break; + case TD_HAS_VAL | TD_HAS_NONE: + case TD_HAS_VAL | TD_HAS_NULL: + // read bitmap (todo) + if (0) { + if (tflags & TD_HAS_NONE) { + *flags = -1; + } else { + *flags = 1; + } + } else { + // get value (todo) + } + break; + case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: + break; + default: + return -1; + } + } + return 0; +} + +// STSchema int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); if (*ppTSchema == NULL) { @@ -87,6 +144,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema * void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); } +// STSRowBuilder int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) { int32_t kvBufLen; int32_t tpBufLen; @@ -131,7 +189,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) { for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) { pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - pBuilder->pTColumn->flags &= (~COL_VAL_SET); + pBuilder->pTColumn->flags &= 0xf; } pBuilder->nCols = 0; @@ -144,7 +202,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD int32_t iCol; uint8_t *p; - // search column + // search column (todo: make here faster) if (pBuilder->pTColumn->colId < cid) { iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1; for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) { @@ -160,7 +218,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD } // check - if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) { + if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) { return -1; } @@ -168,9 +226,12 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD if (cid == 0) { ASSERT(pData && nData == sizeof(TSKEY)); pBuilder->row.ts = *(TSKEY *)pData; + + pBuilder->pTColumn->flags |= COL_SET_VAL; } else { if (pData) { - // ASSERT(!IS_NULL(pData)); + pBuilder->row.flags |= TD_HAS_VAL; + pBuilder->pTColumn->flags |= COL_SET_VAL; // set tuple data p = pBuilder->pTPBuf + pBuilder->pTColumn->offset; @@ -197,51 +258,79 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD pBuilder->kvVLen += nData; } } else { - // set NULL val + pBuilder->row.flags |= TD_HAS_NULL; + pBuilder->pTColumn->flags |= COL_SET_NULL; + + p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols; + ((SKVIdx *)p)->cid = cid; + ((SKVIdx *)p)->offset = -1; } + + pBuilder->nCols++; } - pBuilder->pTColumn->flags |= COL_VAL_SET; - pBuilder->nCols++; return 0; } int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { - if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) { + int32_t tpDataLen, kvDataLen; + uint32_t flags; + + // error not set ts + if (!COL_IS_SET(pBuilder->pTSchema->columns)) { return -1; } - if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) { - // encode as TD_KV_ROW - pBuilder->row.flags |= TD_KV_ROW; - pBuilder->row.ncols = pBuilder->nCols; - pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen; - pBuilder->row.pData = pBuilder->pKVBuf; + if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) { + pBuilder->row.flags |= TD_HAS_NONE; + } - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { - memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols, - pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen); - } - } else { - // encode as TD_TUPLE_ROW - pBuilder->row.flags &= (~TD_KV_ROW); - pBuilder->row.sver = pBuilder->pTSchema->version; - pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; - pBuilder->row.pData = pBuilder->pTPBuf; - - if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) { - // set non-set cols as None - for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) { - pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol]; - if (pBuilder->pTColumn->flags & COL_VAL_SET) continue; - - { - // set None (todo) - } - - pBuilder->pTColumn->flags |= COL_VAL_SET; + flags = pBuilder->row.flags & 0xf; + switch (flags) { + case TD_HAS_NONE: + case TD_HAS_NULL: + pBuilder->row.sver = pBuilder->pTSchema->version; + pBuilder->row.nData = 0; + pBuilder->row.pData = NULL; + break; + case TD_HAS_VAL: + pBuilder->row.sver = pBuilder->pTSchema->version; + pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen; + pBuilder->row.pData = pBuilder->pTPBuf; + break; + case TD_HAS_NULL | TD_HAS_NONE: + pBuilder->row.sver = pBuilder->pTSchema->version; + // set bitmap (todo) + pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1; + pBuilder->row.pData = pBuilder->pBitBuf; + break; + case TD_HAS_VAL | TD_HAS_NONE: + case TD_HAS_VAL | TD_HAS_NULL: + case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE: + if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) { + tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; + } else { + tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen; } - } + kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen; + + if (kvDataLen < tpDataLen) { + pBuilder->row.flags |= TD_KV_ROW; + pBuilder->row.ncols = pBuilder->nCols; + pBuilder->row.nData = kvDataLen; + pBuilder->row.pData = pBuilder->pKVBuf; + // memmove(); todo + // qsort + } else { + pBuilder->row.sver = pBuilder->pTSchema->numOfCols; + // set bitmap etc (todo) + pBuilder->row.nData = tpDataLen; + pBuilder->row.pData = pBuilder->pTPBuf; + } + break; + default: + ASSERT(0); + return -1; } *ppRow = &pBuilder->row;