xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refact data

This commit is contained in:
Hongze Cheng 2022-05-12 05:28:08 +00:00
parent b4f772a66b
commit cccd5dbd8b
3 changed files with 138 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/common/tdataformat.h
View File

@ -38,6 +38,8 @@ typedef struct SKVIdx SKVIdx;
// STSRow2
int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
int8_t *flags);
// STSchema
int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
@ -86,6 +88,7 @@ struct STSRowBuilder {
uint8_t *pKVBuf;
int32_t szTPBuf;
uint8_t *pTPBuf;
uint8_t *pBitBuf;
int32_t nCols;
int32_t kvVLen;
int32_t tpVLen;

13
include/common/tmsg.h
View File

@ -271,12 +271,13 @@ typedef struct {
int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
void tFreeSSubmitRsp(SSubmitRsp *pRsp);
#define COL_SMA_ON ((int8_t)0x1)
#define COL_IDX_ON ((int8_t)0x2)
#define COL_VAL_SET ((int8_t)0x4)
void tFreeSSubmitRsp(SSubmitRsp* pRsp);
#define COL_SMA_ON ((int8_t)0x1)
#define COL_IDX_ON ((int8_t)0x2)
#define COL_SET_VAL ((int8_t)0x10)
#define COL_SET_NONE ((int8_t)0x20)
#define COL_SET_NULL ((int8_t)0x40)
typedef struct SSchema {
int8_t type;
int8_t flags;
@ -285,6 +286,8 @@ typedef struct SSchema {
char name[TSDB_COL_NAME_LEN];
} SSchema;
#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0)
#define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
#define SSCHMEA_TYPE(s) ((s)->type)

165
source/common/src/tdataformat.c
View File

@ -19,13 +19,17 @@
#include "tdatablock.h"
#include "tlog.h"
#define TD_KV_ROW 0x1U
#define TD_HAS_NONE 0x1U
#define TD_HAS_NULL 0x2U
#define TD_HAS_VAL 0x4U
#define TD_KV_ROW 0x10U
struct SKVIdx {
int32_t cid;
int32_t offset;
};
// STSRow2
int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1;
@ -50,6 +54,59 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
return 0;
}
int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
int8_t *flags) {
if (cid == 0) {
*ppData = (uint8_t *)&pRow->ts;
*nData = sizeof(TSKEY);
*flags = 0;
} else {
uint32_t tflags = pRow->flags & 0xf;
*ppData = NULL;
*nData = 0;
switch (tflags) {
case TD_HAS_NONE:
*flags = -1;
break;
case TD_HAS_NULL:
*flags = 1;
break;
case TD_HAS_VAL:
*flags = 0;
// find the row
break;
case TD_HAS_NULL | TD_HAS_NONE:
// read bit map (todo)
if (0) {
*flags = 1;
} else {
*flags = -1;
}
break;
case TD_HAS_VAL | TD_HAS_NONE:
case TD_HAS_VAL | TD_HAS_NULL:
// read bitmap (todo)
if (0) {
if (tflags & TD_HAS_NONE) {
*flags = -1;
} else {
*flags = 1;
}
} else {
// get value (todo)
}
break;
case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
break;
default:
return -1;
}
}
return 0;
}
// STSchema
int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) {
*ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols);
if (*ppTSchema == NULL) {
@ -87,6 +144,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema *
void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); }
// STSRowBuilder
int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
int32_t kvBufLen;
int32_t tpBufLen;
@ -131,7 +189,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) {
pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
pBuilder->pTColumn->flags &= (~COL_VAL_SET);
pBuilder->pTColumn->flags &= 0xf;
}
pBuilder->nCols = 0;
@ -144,7 +202,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
int32_t iCol;
uint8_t *p;
// search column
// search column (todo: make here faster)
if (pBuilder->pTColumn->colId < cid) {
iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
@ -160,7 +218,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
}
// check
if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) {
if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) {
return -1;
}
@ -168,9 +226,12 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
if (cid == 0) {
ASSERT(pData && nData == sizeof(TSKEY));
pBuilder->row.ts = *(TSKEY *)pData;
pBuilder->pTColumn->flags |= COL_SET_VAL;
} else {
if (pData) {
// ASSERT(!IS_NULL(pData));
pBuilder->row.flags |= TD_HAS_VAL;
pBuilder->pTColumn->flags |= COL_SET_VAL;
// set tuple data
p = pBuilder->pTPBuf + pBuilder->pTColumn->offset;
@ -197,51 +258,79 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
pBuilder->kvVLen += nData;
}
} else {
// set NULL val
pBuilder->row.flags |= TD_HAS_NULL;
pBuilder->pTColumn->flags |= COL_SET_NULL;
p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
((SKVIdx *)p)->cid = cid;
((SKVIdx *)p)->offset = -1;
}
pBuilder->nCols++;
}
pBuilder->pTColumn->flags |= COL_VAL_SET;
pBuilder->nCols++;
return 0;
}
int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) {
int32_t tpDataLen, kvDataLen;
uint32_t flags;
// error not set ts
if (!COL_IS_SET(pBuilder->pTSchema->columns)) {
return -1;
}
if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) {
// encode as TD_KV_ROW
pBuilder->row.flags |= TD_KV_ROW;
pBuilder->row.ncols = pBuilder->nCols;
pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen;
pBuilder->row.pData = pBuilder->pKVBuf;
if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
pBuilder->row.flags |= TD_HAS_NONE;
}
if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen);
}
} else {
// encode as TD_TUPLE_ROW
pBuilder->row.flags &= (~TD_KV_ROW);
pBuilder->row.sver = pBuilder->pTSchema->version;
pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
pBuilder->row.pData = pBuilder->pTPBuf;
if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
// set non-set cols as None
for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
if (pBuilder->pTColumn->flags & COL_VAL_SET) continue;
{
// set None (todo)
}
pBuilder->pTColumn->flags |= COL_VAL_SET;
flags = pBuilder->row.flags & 0xf;
switch (flags) {
case TD_HAS_NONE:
case TD_HAS_NULL:
pBuilder->row.sver = pBuilder->pTSchema->version;
pBuilder->row.nData = 0;
pBuilder->row.pData = NULL;
break;
case TD_HAS_VAL:
pBuilder->row.sver = pBuilder->pTSchema->version;
pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
pBuilder->row.pData = pBuilder->pTPBuf;
break;
case TD_HAS_NULL | TD_HAS_NONE:
pBuilder->row.sver = pBuilder->pTSchema->version;
// set bitmap (todo)
pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1;
pBuilder->row.pData = pBuilder->pBitBuf;
break;
case TD_HAS_VAL | TD_HAS_NONE:
case TD_HAS_VAL | TD_HAS_NULL:
case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) {
tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
} else {
tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
}
}
kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
if (kvDataLen < tpDataLen) {
pBuilder->row.flags |= TD_KV_ROW;
pBuilder->row.ncols = pBuilder->nCols;
pBuilder->row.nData = kvDataLen;
pBuilder->row.pData = pBuilder->pKVBuf;
// memmove(); todo
// qsort
} else {
pBuilder->row.sver = pBuilder->pTSchema->numOfCols;
// set bitmap etc (todo)
pBuilder->row.nData = tpDataLen;
pBuilder->row.pData = pBuilder->pTPBuf;
}
break;
default:
ASSERT(0);
return -1;
}
*ppRow = &pBuilder->row;