From 65bdd34007db8516efe408776aa7e900b48f4fd2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 May 2022 13:59:53 +0000 Subject: [PATCH 01/22] feat: tag refact --- include/common/tdataformat.h | 11 +- include/util/tencode.h | 5 + source/common/src/tdataformat.c | 242 +++++++++++++++++++------------- 3 files changed, 155 insertions(+), 103 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index ef931ed3b1..350dc9367e 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -59,12 +59,12 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, u int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); // STag -int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag); +int32_t tTagNew(STagVal *pTagVals, int16_t nTag, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); -int32_t tTagSet(STag *pTag, SSchema *pSchema, int32_t nCols, int iCol, uint8_t *pData, uint32_t nData, STag **ppTag); -void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, uint32_t *nData); +void tTagGet(STag *pTag, STagVal *pTagVal); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); +int32_t tTagToValArray(STag *pTag, STagVal **ppTagVals, int16_t *nTag); // STRUCT ================= struct STColumn { @@ -118,7 +118,10 @@ struct SColVal { }; struct STagVal { - int16_t cid; + union { + int16_t cid; + char *pKey; + }; int8_t type; uint32_t nData; uint8_t *pData; diff --git a/include/util/tencode.h b/include/util/tencode.h index cbacd59fa7..914091ad51 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -642,6 +642,11 @@ static FORCE_INLINE int32_t tGetBinary(uint8_t* p, uint8_t** ppData, uint32_t* n return n; } +static FORCE_INLINE int32_t tPutCStr(uint8_t* p, char* pData) { + return tPutBinary(p, (uint8_t*)pData, strlen(pData) + 1); +} +static FORCE_INLINE int32_t tGetCStr(uint8_t* p, char** ppData) { return tGetBinary(p, (uint8_t**)ppData, NULL); } + #ifdef __cplusplus } #endif diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index e8d7e3ac09..b2316ce4e1 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -31,16 +31,13 @@ typedef struct { } STSKVRow; #pragma pack(pop) -typedef struct STagIdx { - int16_t cid; - uint16_t offset; -} STagIdx; - #pragma pack(push, 1) struct STag { - uint16_t len; - uint16_t nTag; - STagIdx idx[]; + int8_t isJson; + int16_t len; + int16_t nTag; + int32_t ver; + int16_t idx[]; }; #pragma pack(pop) @@ -521,123 +518,149 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { return 0; } -static FORCE_INLINE int tTagIdxCmprFn(const void *p1, const void *p2) { - STagIdx *pTagIdx1 = (STagIdx *)p1; - STagIdx *pTagIdx2 = (STagIdx *)p2; - if (pTagIdx1->cid < pTagIdx1->cid) { +static int tTagValCmprFn(const void *p1, const void *p2) { + if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { return -1; - } else if (pTagIdx1->cid > pTagIdx1->cid) { + } else if (((STagVal *)p1)->cid > ((STagVal *)p2)->cid) { return 1; } + + ASSERT(0); return 0; } -int32_t tTagNew(STagVal *pTagVals, int16_t nTag, STag **ppTag) { - STagVal *pTagVal; - uint8_t *p; - int32_t n; - uint16_t tsize = sizeof(STag) + sizeof(STagIdx) * nTag; +static int tTagValJsonCmprFn(const void *p1, const void *p2) { + return strcmp(((STagVal *)p1)[0].pKey, ((STagVal *)p2)[0].pKey); +} +static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { + int32_t n = 0; + // key + if (isJson) { + n += tPutCStr(p ? p + n : p, pTagVal->pKey); + } else { + n += tPutI16v(p ? p + n : p, pTagVal->cid); + } + + // type + n += tPutI8(p ? p + n : p, pTagVal->type); + + // value + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); + } else { + ASSERT(pTagVal->nData == TYPE_BYTES[pTagVal->type]); + if (p) memcpy(p + n, pTagVal->pData, pTagVal->nData); + n += pTagVal->nData; + } + + return n; +} +static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { + int32_t n = 0; + + // key + if (isJson) { + n += tGetCStr(p + n, &pTagVal->pKey); + } else { + n += tGetI16v(p + n, &pTagVal->cid); + } + + // type + n += tGetI8(p + n, &pTagVal->type); + + // value + if (IS_VAR_DATA_TYPE(pTagVal->type)) { + n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); + } else { + pTagVal->pData = p + n; + pTagVal->nData = TYPE_BYTES[pTagVal->type]; + n += pTagVal->nData; + } + + return n; +} +int32_t tTagNew(STagVal *pTagVals, int16_t nTag, int32_t version, int8_t isJson, STag **ppTag) { + int32_t code = 0; + uint8_t *p = NULL; + int16_t n = 0; + int32_t szTag = sizeof(STag) + sizeof(int16_t) * nTag; + + // sort + if (isJson) { + qsort(pTagVals, nTag, sizeof(STagVal), tTagValJsonCmprFn); + } else { + qsort(pTagVals, nTag, sizeof(STagVal), tTagValCmprFn); + } + + // get size for (int16_t iTag = 0; iTag < nTag; iTag++) { - pTagVal = &pTagVals[iTag]; - - if (IS_VAR_DATA_TYPE(pTagVal->type)) { - tsize += tPutBinary(NULL, pTagVal->pData, pTagVal->nData); - } else { - ASSERT(pTagVal->nData == TYPE_BYTES[pTagVal->type]); - tsize += pTagVal->nData; - } + szTag += tPutTagVal(NULL, &pTagVals[iTag], isJson); } - (*ppTag) = (STag *)taosMemoryMalloc(tsize); - if (*ppTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + // TODO + // if (szTag >= 16 * 1024) { + // code = TSDB_CODE_IVLD_TAG; + // goto _err; + // } + + // build tag + (*ppTag) = (STag *)taosMemoryMalloc(szTag); + if ((*ppTag) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } - - p = (uint8_t *)&((*ppTag)->idx[nTag]); - n = 0; - - (*ppTag)->len = tsize; + (*ppTag)->isJson = isJson ? 1 : 0; + (*ppTag)->len = szTag; (*ppTag)->nTag = nTag; + (*ppTag)->ver = version; + + p = (uint8_t *)&(*ppTag)->idx[nTag]; + n = 0; for (int16_t iTag = 0; iTag < nTag; iTag++) { - pTagVal = &pTagVals[iTag]; - - (*ppTag)->idx[iTag].cid = pTagVal->cid; - (*ppTag)->idx[iTag].offset = n; - - if (IS_VAR_DATA_TYPE(pTagVal->type)) { - n += tPutBinary(p + n, pTagVal->pData, pTagVal->nData); - } else { - memcpy(p + n, pTagVal->pData, pTagVal->nData); - n += pTagVal->nData; - } + (*ppTag)->idx[iTag] = n; + n += tPutTagVal(p + n, &pTagVals[iTag], isJson); } - qsort((*ppTag)->idx, (*ppTag)->nTag, sizeof(STagIdx), tTagIdxCmprFn); - return 0; + return code; + +_err: + return code; } void tTagFree(STag *pTag) { if (pTag) taosMemoryFree(pTag); } -int32_t tTagSet(STag *pTag, SSchema *pSchema, int32_t nCols, int iCol, uint8_t *pData, uint32_t nData, STag **ppTag) { - STagVal *pTagVals; - int16_t nTags = 0; - SSchema *pColumn; - uint8_t *p; - uint32_t n; +void tTagGet(STag *pTag, STagVal *pTagVal) { + int16_t lidx = 0; + int16_t ridx = pTag->nTag - 1; + int16_t midx; + uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag]; + STagVal tv; + int c; - pTagVals = (STagVal *)taosMemoryMalloc(sizeof(*pTagVals) * nCols); - if (pTagVals == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } + pTagVal->type = TSDB_DATA_TYPE_NULL; + pTagVal->pData = NULL; + pTagVal->nData = 0; + while (lidx <= ridx) { + midx = (lidx + ridx) / 2; - for (int32_t i = 0; i < nCols; i++) { - pColumn = &pSchema[i]; - - if (i == iCol) { - p = pData; - n = nData; + tGetTagVal(p + pTag->idx[midx], &tv, pTag->isJson); + if (pTag->isJson) { + c = tTagValJsonCmprFn(pTagVal, &tv); } else { - tTagGet(pTag, pColumn->colId, pColumn->type, &p, &n); + c = tTagValCmprFn(pTagVal, &tv); } - if (p == NULL) continue; - - ASSERT(IS_VAR_DATA_TYPE(pColumn->type) || n == pColumn->bytes); - - pTagVals[nTags].cid = pColumn->colId; - pTagVals[nTags].type = pColumn->type; - pTagVals[nTags].nData = n; - pTagVals[nTags].pData = p; - - nTags++; - } - - // create new tag - if (tTagNew(pTagVals, nTags, ppTag) < 0) { - taosMemoryFree(pTagVals); - return -1; - } - - taosMemoryFree(pTagVals); - return 0; -} - -void tTagGet(STag *pTag, int16_t cid, int8_t type, uint8_t **ppData, uint32_t *nData) { - STagIdx *pTagIdx = bsearch(&((STagIdx){.cid = cid}), pTag->idx, pTag->nTag, sizeof(STagIdx), tTagIdxCmprFn); - if (pTagIdx == NULL) { - *ppData = NULL; - *nData = 0; - } else { - uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag] + pTagIdx->offset; - if (IS_VAR_DATA_TYPE(type)) { - tGetBinary(p, ppData, nData); + if (c < 0) { + ridx = midx - 1; + } else if (c > 0) { + lidx = midx + 1; } else { - *ppData = p; - *nData = TYPE_BYTES[type]; + pTagVal->type = tv.type; + pTagVal->nData = tv.nData; + pTagVal->pData = tv.pData; + break; } } } @@ -648,6 +671,27 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag) { int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { return tDecodeBinary(pDecoder, (uint8_t **)ppTag, NULL); } +int32_t tTagToValArray(STag *pTag, STagVal **ppTagVals, int16_t *nTag) { + int32_t code = 0; + uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag]; + + *nTag = pTag->nTag; + (*ppTagVals) = (STagVal *)taosMemoryMalloc(sizeof(STagVal) * pTag->nTag); + if (*ppTagVals == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { + tGetTagVal(p + pTag->idx[iTag], &(*ppTagVals)[iTag], pTag->isJson); + } + + return code; + +_err: + return code; +} + #if 1 // =================================================================================================================== static void dataColSetNEleNull(SDataCol *pCol, int nEle); int tdAllocMemForCol(SDataCol *pCol, int maxPoints) { From 61e21ebf2aa0cab6098305cd0e8fc4ae10258ded Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 May 2022 14:05:14 +0000 Subject: [PATCH 02/22] feat: tag refact --- include/common/tdataformat.h | 3 ++- source/common/src/tdataformat.c | 11 ++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 350dc9367e..3f6ff863ba 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -18,6 +18,7 @@ #include "os.h" #include "talgo.h" +#include "tarray.h" #include "tencode.h" #include "ttypes.h" #include "tutil.h" @@ -64,7 +65,7 @@ void tTagFree(STag *pTag); void tTagGet(STag *pTag, STagVal *pTagVal); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); -int32_t tTagToValArray(STag *pTag, STagVal **ppTagVals, int16_t *nTag); +int32_t tTagToValArray(STag *pTag, SArray **ppArray); // STRUCT ================= struct STColumn { diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b2316ce4e1..87b01623e8 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -671,19 +671,20 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag) { int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { return tDecodeBinary(pDecoder, (uint8_t **)ppTag, NULL); } -int32_t tTagToValArray(STag *pTag, STagVal **ppTagVals, int16_t *nTag) { +int32_t tTagToValArray(STag *pTag, SArray **ppArray) { int32_t code = 0; uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag]; + STagVal tv; - *nTag = pTag->nTag; - (*ppTagVals) = (STagVal *)taosMemoryMalloc(sizeof(STagVal) * pTag->nTag); - if (*ppTagVals == NULL) { + (*ppArray) = taosArrayInit(pTag->nTag, sizeof(STagVal)); + if (*ppArray == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { - tGetTagVal(p + pTag->idx[iTag], &(*ppTagVals)[iTag], pTag->isJson); + tGetTagVal(p + pTag->idx[iTag], &tv, pTag->isJson); + taosArrayPush(*ppArray, &tv); } return code; From 806ba2cdcee2b5c9b9dff127e8203241e8faaca0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 May 2022 14:18:31 +0000 Subject: [PATCH 03/22] feat: refact tag api --- include/common/tdataformat.h | 2 +- source/common/src/tdataformat.c | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 3f6ff863ba..974dfc2264 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -60,7 +60,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, u int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); // STag -int32_t tTagNew(STagVal *pTagVals, int16_t nTag, int32_t version, int8_t isJson, STag **ppTag); +int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); void tTagGet(STag *pTag, STagVal *pTagVal); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 87b01623e8..cb92e68ea4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -579,22 +579,23 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { return n; } -int32_t tTagNew(STagVal *pTagVals, int16_t nTag, int32_t version, int8_t isJson, STag **ppTag) { +int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { int32_t code = 0; uint8_t *p = NULL; int16_t n = 0; + int16_t nTag = taosArrayGetSize(pArray); int32_t szTag = sizeof(STag) + sizeof(int16_t) * nTag; // sort if (isJson) { - qsort(pTagVals, nTag, sizeof(STagVal), tTagValJsonCmprFn); + qsort(pArray->pData, nTag, sizeof(STagVal), tTagValJsonCmprFn); } else { - qsort(pTagVals, nTag, sizeof(STagVal), tTagValCmprFn); + qsort(pArray->pData, nTag, sizeof(STagVal), tTagValCmprFn); } // get size for (int16_t iTag = 0; iTag < nTag; iTag++) { - szTag += tPutTagVal(NULL, &pTagVals[iTag], isJson); + szTag += tPutTagVal(NULL, (STagVal *)taosArrayGet(pArray, iTag), isJson); } // TODO @@ -618,7 +619,7 @@ int32_t tTagNew(STagVal *pTagVals, int16_t nTag, int32_t version, int8_t isJson, n = 0; for (int16_t iTag = 0; iTag < nTag; iTag++) { (*ppTag)->idx[iTag] = n; - n += tPutTagVal(p + n, &pTagVals[iTag], isJson); + n += tPutTagVal(p + n, (STagVal *)taosArrayGet(pArray, iTag), isJson); } return code; From b55331ca9882ab67c0a0c816bd4b83db837c7bd6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 May 2022 14:18:59 +0000 Subject: [PATCH 04/22] more --- source/common/src/tdataformat.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index cb92e68ea4..0d74800d4e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -677,7 +677,7 @@ int32_t tTagToValArray(STag *pTag, SArray **ppArray) { uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag]; STagVal tv; - (*ppArray) = taosArrayInit(pTag->nTag, sizeof(STagVal)); + (*ppArray) = taosArrayInit(pTag->nTag + 1, sizeof(STagVal)); if (*ppArray == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; From 9b4c9e4802ecd9920b8750600808240a496793fd Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 28 May 2022 17:16:04 +0800 Subject: [PATCH 05/22] enh: tag refactor for normal tags --- include/common/tdataformat.h | 148 +++-------- include/common/tmsg.h | 9 + source/client/src/clientImpl.c | 18 +- source/common/src/tdatablock.c | 53 ++-- source/common/src/tdataformat.c | 251 ++++++------------ source/common/src/tmsg.c | 6 +- source/common/src/trow.c | 4 +- source/dnode/vnode/src/meta/metaEntry.c | 5 +- source/dnode/vnode/src/meta/metaQuery.c | 4 +- source/dnode/vnode/src/meta/metaTable.c | 39 ++- source/libs/executor/src/scanoperator.c | 20 +- source/libs/parser/inc/parUtil.h | 2 + source/libs/parser/src/parInsert.c | 129 +++++---- source/libs/parser/src/parTranslater.c | 149 ++++++++--- source/libs/parser/src/parUtil.c | 2 + source/libs/scalar/src/sclvector.c | 10 +- .../libs/scalar/test/scalar/scalarTests.cpp | 8 +- 17 files changed, 439 insertions(+), 418 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 3f6ff863ba..912ffb7876 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -59,13 +59,18 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); +// STagVal +static FORCE_INLINE void tTagValSet(STagVal *pTagVal, void *key, int8_t type, uint8_t *pData, uint32_t nData, + bool isJson); + // STag int32_t tTagNew(STagVal *pTagVals, int16_t nTag, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); -void tTagGet(STag *pTag, STagVal *pTagVal); +bool tTagGet(const STag *pTag, STagVal *pTagVal); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); -int32_t tTagToValArray(STag *pTag, SArray **ppArray); +int32_t tTagToValArray(const STag *pTag, SArray **ppArray); +void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // STRUCT ================= struct STColumn { @@ -128,6 +133,29 @@ struct STagVal { uint8_t *pData; }; +static FORCE_INLINE void tTagValSet(STagVal *pTagVal, void *key, int8_t type, uint8_t *pData, uint32_t nData, + bool isJson) { + if (isJson) { + pTagVal->pKey = (char *)key; + } else { + pTagVal->cid = *(int16_t *)key; + } + + pTagVal->type = type; + pTagVal->pData = pData; + pTagVal->nData = nData; +} + +#pragma pack(push, 1) +struct STag { + int8_t isJson; + int16_t len; + int16_t nTag; + int32_t ver; + int16_t idx[]; +}; +#pragma pack(pop) + #if 1 //================================================================================================================================================ // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. #define TD_SUPPORT_BITMAP @@ -186,7 +214,8 @@ struct STagVal { #define schemaColAt(s, i) ((s)->columns + i) #define tdFreeSchema(s) taosMemoryFreeClear((s)) -STSchema *tdDupSchema(const STSchema *pSchema); + STSchema * + tdDupSchema(const STSchema *pSchema); int32_t tdEncodeSchema(void **buf, STSchema *pSchema); void *tdDecodeSchema(void *buf, STSchema **pRSchema); @@ -370,109 +399,8 @@ SDataCols *tdFreeDataCols(SDataCols *pCols); int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer); -// ----------------- K-V data row structure -/* |<-------------------------------------- len -------------------------------------------->| - * |<----- header ----->|<--------------------------- body -------------------------------->| - * +----------+----------+---------------------------------+---------------------------------+ - * | uint16_t | int16_t | | | - * +----------+----------+---------------------------------+---------------------------------+ - * | len | ncols | cols index | data part | - * +----------+----------+---------------------------------+---------------------------------+ - */ -typedef void *SKVRow; -typedef struct { - int16_t colId; - uint16_t offset; -} SColIdx; -#define TD_KV_ROW_HEAD_SIZE (sizeof(uint16_t) + sizeof(int16_t)) - -#define kvRowLen(r) (*(uint16_t *)(r)) -#define kvRowNCols(r) (*(int16_t *)POINTER_SHIFT(r, sizeof(uint16_t))) -#define kvRowSetLen(r, len) kvRowLen(r) = (len) -#define kvRowSetNCols(r, n) kvRowNCols(r) = (n) -#define kvRowColIdx(r) (SColIdx *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE) -#define kvRowValues(r) POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) -#define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) -#define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) taosMemoryFreeClear(r) -#define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) -#define kvRowValLen(r) (kvRowLen(r) - TD_KV_ROW_HEAD_SIZE - sizeof(SColIdx) * kvRowNCols(r)) -#define kvRowTKey(r) (*(TKEY *)(kvRowValues(r))) -#define kvRowKey(r) tdGetKey(kvRowTKey(r)) -#define kvRowKeys(r) POINTER_SHIFT(r, *(uint16_t *)POINTER_SHIFT(r, TD_KV_ROW_HEAD_SIZE + sizeof(int16_t))) -#define kvRowDeleted(r) TKEY_IS_DELETED(kvRowTKey(r)) - -SKVRow tdKVRowDup(SKVRow row); -int32_t tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); -int32_t tdEncodeKVRow(void **buf, SKVRow row); -void *tdDecodeKVRow(void *buf, SKVRow *row); -void tdSortKVRowByColIdx(SKVRow row); - -static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) { - if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { - return 1; - } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { - return -1; - } else { - return 0; - } -} - -static FORCE_INLINE void *tdGetKVRowValOfCol(const SKVRow row, int16_t colId) { - void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ); - if (ret == NULL) return NULL; - return kvRowColVal(row, (SColIdx *)ret); -} - -static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) { - return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ); -} - -// ----------------- K-V data row builder -typedef struct { - int16_t tCols; - int16_t nCols; - SColIdx *pColIdx; - uint16_t alloc; - uint16_t size; - void *buf; -} SKVRowBuilder; - -int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder); -void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); -void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); -SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); - -static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) { - if (pBuilder->nCols >= pBuilder->tCols) { - pBuilder->tCols *= 2; - SColIdx *pColIdx = (SColIdx *)taosMemoryRealloc((void *)(pBuilder->pColIdx), sizeof(SColIdx) * pBuilder->tCols); - if (pColIdx == NULL) return -1; - pBuilder->pColIdx = pColIdx; - } - - pBuilder->pColIdx[pBuilder->nCols].colId = colId; - pBuilder->pColIdx[pBuilder->nCols].offset = pBuilder->size; - - pBuilder->nCols++; - - if (tlen > pBuilder->alloc - pBuilder->size) { - while (tlen > pBuilder->alloc - pBuilder->size) { - pBuilder->alloc *= 2; - } - void *buf = taosMemoryRealloc(pBuilder->buf, pBuilder->alloc); - if (buf == NULL) return -1; - pBuilder->buf = buf; - } - - memcpy(POINTER_SHIFT(pBuilder->buf, pBuilder->size), value, tlen); - pBuilder->size += tlen; - - return 0; -} #endif #ifdef __cplusplus @@ -480,3 +408,15 @@ static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t co #endif #endif /*_TD_COMMON_DATA_FORMAT_H_*/ + +// SKVRowBuilder; + +// int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder); +// void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); +// void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); +// SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); + +// static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) + +// #ifdef JSON_TAG_REFACTOR +// TODO: JSON_TAG_TODO diff --git a/include/common/tmsg.h b/include/common/tmsg.h index faf4addb4b..37e4cc5e8c 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1747,6 +1747,15 @@ typedef struct SVCreateTbReq { int tEncodeSVCreateTbReq(SEncoder* pCoder, const SVCreateTbReq* pReq); int tDecodeSVCreateTbReq(SDecoder* pCoder, SVCreateTbReq* pReq); +static FORCE_INLINE void tdDestroySVCreateTbReq(SVCreateTbReq* req) { + taosMemoryFreeClear(req->name); + if (req->type == TSDB_CHILD_TABLE) { + taosMemoryFreeClear(req->ctb.pTag); + } else if (req->type == TSDB_NORMAL_TABLE) { + taosMemoryFreeClear(req->ntb.schemaRow.pSchema); + } +} + typedef struct { int32_t nReqs; union { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 362df7b447..fc899b9b83 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -949,11 +949,23 @@ static char* parseTagDatatoJson(void* p) { goto end; } - int16_t nCols = kvRowNCols(p); + SArray* pTagVals = NULL; + if (tTagToValArray((const STag*)p, &pTagVals) != 0) { + goto end; + } + + int16_t nCols = taosArrayGetSize(pTagVals); char tagJsonKey[256] = {0}; for (int j = 0; j < nCols; ++j) { - SColIdx* pColIdx = kvRowColIdxAt(p, j); - char* val = (char*)(kvRowColVal(p, pColIdx)); + STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); + + // JSON_TAG_REFACTOR + pTagVal->nData; + pTagVal->pData; // for varChar, len not included + // TODO: adapt below code; + char* val = (char*)pTagVal->pData; + // TODO: adapt below code; + if (j == 0) { if (*val == TSDB_DATA_TYPE_NULL) { string = taosMemoryCalloc(1, 8); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 0f34d52d35..34ec5a1251 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -127,7 +127,7 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; } else if (*pData == TSDB_DATA_TYPE_JSON) { - dataLen = kvRowLen(pData + CHAR_BYTES); + dataLen = ((STag*)(pData + CHAR_BYTES))->len; } else { ASSERT(0); } @@ -1652,18 +1652,24 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - ASSERT(0); + STagVal tagVal = {.cid = 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .pData = (uint8_t*)&pDataBlock->info.groupId, + .nData = sizeof(uint64_t)}; + STag* pTag = NULL; + tTagNew(&tagVal, 1, 1, false, &pTag); + if (!pTag) { + tdDestroySVCreateTbReq(&createTbReq); + return NULL; } - tdAddColToKVRow(&kvRowBuilder, 1, &pDataBlock->info.groupId, sizeof(uint64_t)); - createTbReq.ctb.pTag = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); + createTbReq.ctb.pTag = (uint8_t*)pTag; int32_t code; tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); + + tdDestroySVCreateTbReq(&createTbReq); + if (code < 0) return NULL; - taosMemoryFree(cname); } cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; @@ -1706,22 +1712,37 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { - ASSERT(0); + STagVal tagVal = {.cid = 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .pData = (uint8_t*)&pDataBlock->info.groupId, + .nData = sizeof(uint64_t)}; + STag* pTag = NULL; + tTagNew(&tagVal, 1, 1, false, &pTag); + if (!pTag) { + tdDestroySVCreateTbReq(&createTbReq); + taosMemoryFreeClear(ret); + return NULL; } - tdAddColToKVRow(&kvRowBuilder, 1, &pDataBlock->info.groupId, sizeof(uint64_t)); - createTbReq.ctb.pTag = tdGetKVRowFromBuilder(&kvRowBuilder); - tdDestroyKVRowBuilder(&kvRowBuilder); + createTbReq.ctb.pTag = (uint8_t*)pTag; int32_t code; tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) return NULL; + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosMemoryFreeClear(ret); + return NULL; + } SEncoder encoder = {0}; tEncoderInit(&encoder, blockData, schemaLen); - if (tEncodeSVCreateTbReq(&encoder, &createTbReq) < 0) return NULL; + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + + if (code < 0) { + taosMemoryFreeClear(ret); + return NULL; + } } blkHead->schemaLen = htonl(schemaLen); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 87b01623e8..c98317e385 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -19,6 +19,8 @@ #include "tdatablock.h" #include "tlog.h" +static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson); + typedef struct SKVIdx { int32_t cid; int32_t offset; @@ -31,16 +33,6 @@ typedef struct { } STSKVRow; #pragma pack(pop) -#pragma pack(push, 1) -struct STag { - int8_t isJson; - int16_t len; - int16_t nTag; - int32_t ver; - int16_t idx[]; -}; -#pragma pack(pop) - #define TSROW_IS_KV_ROW(r) ((r)->flags & TSROW_KV_ROW) #define BIT1_SIZE(n) (((n)-1) / 8 + 1) #define BIT2_SIZE(n) (((n)-1) / 4 + 1) @@ -531,6 +523,77 @@ static int tTagValCmprFn(const void *p1, const void *p2) { static int tTagValJsonCmprFn(const void *p1, const void *p2) { return strcmp(((STagVal *)p1)[0].pKey, ((STagVal *)p2)[0].pKey); } + +static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const char *tag, int32_t ln) { + switch (type) { + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_NCHAR: { + char tmpVal[32] = {0}; + memcpy(tmpVal, val, 32); + printf("%s:%d type:%d vlen:%d, val:\"%s\"\n", tag, ln, (int32_t)type, vlen, tmpVal); + } break; + case TSDB_DATA_TYPE_FLOAT: + printf("%s:%d type:%d vlen:%d, val:%f\n", tag, ln, (int32_t)type, vlen, *(float *)val); + break; + case TSDB_DATA_TYPE_DOUBLE: + printf("%s:%d type:%d vlen:%d, val:%lf\n", tag, ln, (int32_t)type, vlen, *(double *)val); + break; + case TSDB_DATA_TYPE_BOOL: + printf("%s:%d type:%d vlen:%d, val:%" PRIu8 "\n", tag, ln, (int32_t)type, vlen, *(uint8_t *)val); + break; + case TSDB_DATA_TYPE_TINYINT: + printf("%s:%d type:%d vlen:%d, val:%" PRIi8 "\n", tag, ln, (int32_t)type, vlen, *(int8_t *)val); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf("%s:%d type:%d vlen:%d, val:%" PRIi16 "\n", tag, ln, (int32_t)type, vlen, *(int16_t *)val); + break; + case TSDB_DATA_TYPE_INT: + printf("%s:%d type:%d vlen:%d, val:%" PRIi32 "\n", tag, ln, (int32_t)type, vlen, *(int32_t *)val); + break; + case TSDB_DATA_TYPE_BIGINT: + printf("%s:%d type:%d vlen:%d, val:%" PRIi64 "\n", tag, ln, (int32_t)type, vlen, *(int64_t *)val); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + printf("%s:%d type:%d vlen:%d, val:%" PRIi64 "\n", tag, ln, (int32_t)type, vlen, *(int64_t *)val); + break; + case TSDB_DATA_TYPE_UTINYINT: + printf("%s:%d type:%d vlen:%d, val:%" PRIu8 "\n", tag, ln, (int32_t)type, vlen, *(uint8_t *)val); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf("%s:%d type:%d vlen:%d, val:%" PRIu16 "\n", tag, ln, (int32_t)type, vlen, *(uint16_t *)val); + break; + case TSDB_DATA_TYPE_UINT: + printf("%s:%d type:%d vlen:%d, val:%" PRIu32 "\n", tag, ln, (int32_t)type, vlen, *(uint32_t *)val); + break; + case TSDB_DATA_TYPE_UBIGINT: + printf("%s:%d type:%d vlen:%d, val:%" PRIu64 "\n", tag, ln, (int32_t)type, vlen, *(uint64_t *)val); + break; + default: + ASSERT(0); + break; + } +} + +void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { + printf("%s:%d >>> STAG === isJson:%s, len: %d, nTag: %d, sver:%d\n", tag, ln, pTag->isJson ? "true" : "false", + (int32_t)pTag->len, (int32_t)pTag->nTag, pTag->ver); + char *p = (char *)&pTag->idx[pTag->nTag]; + for (uint16_t n = 0; n < pTag->nTag; ++n) { + int16_t *pIdx = pTag->idx + n; + STagVal tagVal = {0}; + if (pTag->isJson) { + tagVal.pKey = (char *)POINTER_SHIFT(p, *pIdx); + } else { + tagVal.cid = *(int16_t *)POINTER_SHIFT(p, *pIdx); + } + printf("%s:%d loop[%d-%d] offset=%d\n", __func__, __LINE__, (int32_t)pTag->nTag, (int32_t)n, *pIdx); + tGetTagVal(p, &tagVal, pTag->isJson); + debugPrintTagVal(tagVal.type, tagVal.pData, tagVal.nData, __func__, __LINE__); + } + printf("\n"); +} + static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { int32_t n = 0; @@ -544,6 +607,7 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { // type n += tPutI8(p ? p + n : p, pTagVal->type); + debugPrintTagVal(pTagVal->type, pTagVal->pData, pTagVal->nData, __func__, __LINE__); // value if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); @@ -621,6 +685,8 @@ int32_t tTagNew(STagVal *pTagVals, int16_t nTag, int32_t version, int8_t isJson, n += tPutTagVal(p + n, &pTagVals[iTag], isJson); } + debugPrintSTag(*ppTag, __func__, __LINE__); + return code; _err: @@ -631,7 +697,7 @@ void tTagFree(STag *pTag) { if (pTag) taosMemoryFree(pTag); } -void tTagGet(STag *pTag, STagVal *pTagVal) { +bool tTagGet(const STag *pTag, STagVal *pTagVal) { int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; int16_t midx; @@ -660,9 +726,10 @@ void tTagGet(STag *pTag, STagVal *pTagVal) { pTagVal->type = tv.type; pTagVal->nData = tv.nData; pTagVal->pData = tv.pData; - break; + return true; } } + return false; } int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag) { @@ -671,7 +738,7 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag) { int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { return tDecodeBinary(pDecoder, (uint8_t **)ppTag, NULL); } -int32_t tTagToValArray(STag *pTag, SArray **ppArray) { +int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { int32_t code = 0; uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag]; STagVal tv; @@ -1019,162 +1086,4 @@ void tdResetDataCols(SDataCols *pCols) { } } -SKVRow tdKVRowDup(SKVRow row) { - SKVRow trow = taosMemoryMalloc(kvRowLen(row)); - if (trow == NULL) return NULL; - - kvRowCpy(trow, row); - return trow; -} - -static int compareColIdx(const void *a, const void *b) { - const SColIdx *x = (const SColIdx *)a; - const SColIdx *y = (const SColIdx *)b; - if (x->colId > y->colId) { - return 1; - } - if (x->colId < y->colId) { - return -1; - } - return 0; -} - -void tdSortKVRowByColIdx(SKVRow row) { qsort(kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), compareColIdx); } - -int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { - SColIdx *pColIdx = NULL; - SKVRow row = *orow; - SKVRow nrow = NULL; - void *ptr = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_GE); - - if (ptr == NULL || ((SColIdx *)ptr)->colId > colId) { // need to add a column value to the row - int diff = IS_VAR_DATA_TYPE(type) ? varDataTLen(value) : TYPE_BYTES[type]; - int nRowLen = kvRowLen(row) + sizeof(SColIdx) + diff; - int oRowCols = kvRowNCols(row); - - ASSERT(diff > 0); - nrow = taosMemoryMalloc(nRowLen); - if (nrow == NULL) return -1; - - kvRowSetLen(nrow, nRowLen); - kvRowSetNCols(nrow, oRowCols + 1); - - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), sizeof(SColIdx) * oRowCols); - memcpy(kvRowValues(nrow), kvRowValues(row), kvRowValLen(row)); - - pColIdx = kvRowColIdxAt(nrow, oRowCols); - pColIdx->colId = colId; - pColIdx->offset = kvRowValLen(row); - - memcpy(kvRowColVal(nrow, pColIdx), value, diff); // copy new value - - tdSortKVRowByColIdx(nrow); - - *orow = nrow; - taosMemoryFree(row); - } else { - ASSERT(((SColIdx *)ptr)->colId == colId); - if (IS_VAR_DATA_TYPE(type)) { - void *pOldVal = kvRowColVal(row, (SColIdx *)ptr); - - if (varDataTLen(value) == varDataTLen(pOldVal)) { // just update the column value in place - memcpy(pOldVal, value, varDataTLen(value)); - } else { // need to reallocate the memory - int16_t nlen = kvRowLen(row) + (varDataTLen(value) - varDataTLen(pOldVal)); - ASSERT(nlen > 0); - nrow = taosMemoryMalloc(nlen); - if (nrow == NULL) return -1; - - kvRowSetLen(nrow, nlen); - kvRowSetNCols(nrow, kvRowNCols(row)); - - int zsize = sizeof(SColIdx) * kvRowNCols(row) + ((SColIdx *)ptr)->offset; - memcpy(kvRowColIdx(nrow), kvRowColIdx(row), zsize); - memcpy(kvRowColVal(nrow, ((SColIdx *)ptr)), value, varDataTLen(value)); - // Copy left value part - int lsize = kvRowLen(row) - TD_KV_ROW_HEAD_SIZE - zsize - varDataTLen(pOldVal); - if (lsize > 0) { - memcpy(POINTER_SHIFT(nrow, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(value)), - POINTER_SHIFT(row, TD_KV_ROW_HEAD_SIZE + zsize + varDataTLen(pOldVal)), lsize); - } - - for (int i = 0; i < kvRowNCols(nrow); i++) { - pColIdx = kvRowColIdxAt(nrow, i); - - if (pColIdx->offset > ((SColIdx *)ptr)->offset) { - pColIdx->offset = pColIdx->offset - varDataTLen(pOldVal) + varDataTLen(value); - } - } - - *orow = nrow; - taosMemoryFree(row); - } - } else { - memcpy(kvRowColVal(row, (SColIdx *)ptr), value, TYPE_BYTES[type]); - } - } - - return 0; -} - -int tdEncodeKVRow(void **buf, SKVRow row) { - // May change the encode purpose - if (buf != NULL) { - kvRowCpy(*buf, row); - *buf = POINTER_SHIFT(*buf, kvRowLen(row)); - } - - return kvRowLen(row); -} - -void *tdDecodeKVRow(void *buf, SKVRow *row) { - *row = tdKVRowDup(buf); - if (*row == NULL) return NULL; - return POINTER_SHIFT(buf, kvRowLen(*row)); -} - -int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) { - pBuilder->tCols = 128; - pBuilder->nCols = 0; - pBuilder->pColIdx = (SColIdx *)taosMemoryMalloc(sizeof(SColIdx) * pBuilder->tCols); - if (pBuilder->pColIdx == NULL) return -1; - pBuilder->alloc = 1024; - pBuilder->size = 0; - pBuilder->buf = taosMemoryMalloc(pBuilder->alloc); - if (pBuilder->buf == NULL) { - taosMemoryFree(pBuilder->pColIdx); - return -1; - } - return 0; -} - -void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) { - taosMemoryFreeClear(pBuilder->pColIdx); - taosMemoryFreeClear(pBuilder->buf); -} - -void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { - pBuilder->nCols = 0; - pBuilder->size = 0; -} - -SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) { - int tlen = sizeof(SColIdx) * pBuilder->nCols + pBuilder->size; - // if (tlen == 0) return NULL; // nCols == 0 means no tags - - tlen += TD_KV_ROW_HEAD_SIZE; - - SKVRow row = taosMemoryMalloc(tlen); - if (row == NULL) return NULL; - - kvRowSetNCols(row, pBuilder->nCols); - kvRowSetLen(row, tlen); - - if (pBuilder->nCols > 0) { - memcpy(kvRowColIdx(row), pBuilder->pColIdx, sizeof(SColIdx) * pBuilder->nCols); - memcpy(kvRowValues(row), pBuilder->buf, pBuilder->size); - } - - return row; -} #endif \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7615f7b070..721eb9587b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3846,7 +3846,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (pReq->type == TSDB_CHILD_TABLE) { if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1; - if (tEncodeBinary(pCoder, pReq->ctb.pTag, kvRowLen(pReq->ctb.pTag)) < 0) return -1; + if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1; } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { @@ -3858,8 +3858,6 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { } int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { - uint32_t len; - if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; @@ -3871,7 +3869,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { if (pReq->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1; - if (tDecodeBinary(pCoder, &pReq->ctb.pTag, &len) < 0) return -1; + if (tDecodeTag(pCoder, (STag **)&pReq->ctb.pTag) < 0) return -1; } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 4d0846f6c2..cc18240325 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -1191,9 +1191,9 @@ bool tdGetTpRowDataOfCol(STSRowIter *pIter, col_type_t colType, int32_t offset, } static FORCE_INLINE int32_t compareKvRowColId(const void *key1, const void *key2) { - if (*(int16_t *)key1 > ((SColIdx *)key2)->colId) { + if (*(col_id_t *)key1 > ((SKvRowIdx *)key2)->colId) { return 1; - } else if (*(int16_t *)key1 < ((SColIdx *)key2)->colId) { + } else if (*(col_id_t *)key1 < ((SKvRowIdx *)key2)->colId) { return -1; } else { return 0; diff --git a/source/dnode/vnode/src/meta/metaEntry.c b/source/dnode/vnode/src/meta/metaEntry.c index be2ddfc32f..a003494457 100644 --- a/source/dnode/vnode/src/meta/metaEntry.c +++ b/source/dnode/vnode/src/meta/metaEntry.c @@ -30,7 +30,7 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { if (tEncodeI64(pCoder, pME->ctbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ctbEntry.ttlDays) < 0) return -1; if (tEncodeI64(pCoder, pME->ctbEntry.suid) < 0) return -1; - if (tEncodeBinary(pCoder, pME->ctbEntry.pTags, kvRowLen(pME->ctbEntry.pTags)) < 0) return -1; + if (tEncodeTag(pCoder, (const STag *)pME->ctbEntry.pTags) < 0) return -1; } else if (pME->type == TSDB_NORMAL_TABLE) { if (tEncodeI64(pCoder, pME->ntbEntry.ctime) < 0) return -1; if (tEncodeI32(pCoder, pME->ntbEntry.ttlDays) < 0) return -1; @@ -47,7 +47,6 @@ int metaEncodeEntry(SEncoder *pCoder, const SMetaEntry *pME) { } int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { - uint32_t len; if (tStartDecode(pCoder) < 0) return -1; if (tDecodeI64(pCoder, &pME->version) < 0) return -1; @@ -62,7 +61,7 @@ int metaDecodeEntry(SDecoder *pCoder, SMetaEntry *pME) { if (tDecodeI64(pCoder, &pME->ctbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ctbEntry.ttlDays) < 0) return -1; if (tDecodeI64(pCoder, &pME->ctbEntry.suid) < 0) return -1; - if (tDecodeBinary(pCoder, &pME->ctbEntry.pTags, &len) < 0) return -1; // (TODO) + if (tDecodeTag(pCoder, (STag **)&pME->ctbEntry.pTags) < 0) return -1; // (TODO) } else if (pME->type == TSDB_NORMAL_TABLE) { if (tDecodeI64(pCoder, &pME->ntbEntry.ctime) < 0) return -1; if (tDecodeI32(pCoder, &pME->ntbEntry.ttlDays) < 0) return -1; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 184b640bdd..b59b6a6fd0 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -566,5 +566,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); - return tdGetKVRowValOfCol((const SKVRow)pEntry->ctbEntry.pTags, cid); + STagVal tagVal = {.cid = cid}; + tTagGet((const STag *)pEntry->ctbEntry.pTags, &tagVal); + return tagVal.pData; } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7182f496c4..e1f3915053 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -563,29 +563,39 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } memcpy((void *)ctbEntry.ctbEntry.pTags, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); } else { - SKVRowBuilder kvrb = {0}; - const SKVRow pOldTag = (const SKVRow)ctbEntry.ctbEntry.pTags; - SKVRow pNewTag = NULL; + const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; + STag *pNewTag = NULL; + STagVal *pTagVals = taosMemoryCalloc(pTagSchema->nCols, sizeof(STagVal)); - tdInitKVRowBuilder(&kvrb); + if (!pTagVals) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + int16_t nTags = 0; for (int32_t i = 0; i < pTagSchema->nCols; i++) { SSchema *pCol = &pTagSchema->pSchema[i]; + STagVal *pTagVal = pTagVals + nTags; if (iCol == i) { - tdAddColToKVRow(&kvrb, pCol->colId, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); + tTagValSet(pTagVal, &pCol->colId, pCol->type, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal, false); + ++nTags; } else { - void *p = tdGetKVRowValOfCol(pOldTag, pCol->colId); - if (p) { + STagVal tagVal = {.cid = pCol->colId}; + if (tTagGet(pOldTag, &tagVal) && tagVal.pData) { if (IS_VAR_DATA_TYPE(pCol->type)) { - tdAddColToKVRow(&kvrb, pCol->colId, p, varDataTLen(p)); + tTagValSet(pTagVal, &pCol->colId, pCol->type, tagVal.pData, varDataTLen(tagVal.pData), false); } else { - tdAddColToKVRow(&kvrb, pCol->colId, p, pCol->bytes); + tTagValSet(pTagVal, &pCol->colId, pCol->type, tagVal.pData, pCol->bytes, false); } + ++nTags; } } } - - ctbEntry.ctbEntry.pTags = tdGetKVRowFromBuilder(&kvrb); - tdDestroyKVRowBuilder(&kvrb); + if ((terrno = tTagNew(pTagVals, nTags, pTagSchema->version, false, &pNewTag)) < 0) { + taosMemoryFreeClear(pTagVals); + goto _err; + } + ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; + taosMemoryFreeClear(pTagVals); } // save to table.db @@ -775,7 +785,10 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { metaDecodeEntry(&dc, &stbEntry); pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - pTagData = tdGetKVRowValOfCol((const SKVRow)pCtbEntry->ctbEntry.pTags, pTagColumn->colId); + + STagVal tagVal = {.cid = pTagColumn->colId}; + tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); + pTagData = tagVal.pData; // update tag index #ifdef USE_INVERTED_INDEX diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 613fa26c2d..4a275660bc 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -305,17 +305,17 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) } else { // these are tags const char* p = NULL; if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - const uint8_t* tmp = mr.me.ctbEntry.pTags; + const STag* tmp = (const STag*)mr.me.ctbEntry.pTags; - char* data = taosMemoryCalloc(kvRowLen(tmp) + 1, 1); + char* data = taosMemoryCalloc(tmp->len + 1, 1); if (data == NULL) { metaReaderClear(&mr); - qError("doTagScan calloc error:%d", kvRowLen(tmp) + 1); + qError("doTagScan calloc error:%d", tmp->len + 1); return; } *data = TSDB_DATA_TYPE_JSON; - memcpy(data + 1, tmp, kvRowLen(tmp)); + memcpy(data + 1, tmp, tmp->len); p = data; } else { p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId); @@ -1633,16 +1633,16 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { colDataAppend(pDst, count, str, false); } else { // it is a tag value if (pDst->info.type == TSDB_DATA_TYPE_JSON) { - const uint8_t* tmp = mr.me.ctbEntry.pTags; + const STag* tmp = (const STag*)mr.me.ctbEntry.pTags; // TODO opt perf by realloc memory - char* data = taosMemoryCalloc(kvRowLen(tmp) + 1, 1); + char* data = taosMemoryCalloc(tmp->len + 1, 1); if (data == NULL) { - qError("%s failed to malloc memory, size:%d", GET_TASKID(pTaskInfo), kvRowLen(tmp) + 1); + qError("%s failed to malloc memory, size:%d", GET_TASKID(pTaskInfo), tmp->len + 1); longjmp(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } *data = TSDB_DATA_TYPE_JSON; - memcpy(data + 1, tmp, kvRowLen(tmp)); + memcpy(data + 1, tmp, tmp->len); colDataAppend(pDst, count, data, false); taosMemoryFree(data); } else { @@ -1677,8 +1677,8 @@ static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { } SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, SExprInfo* pExpr, int32_t numOfOutput, - SSDataBlock* pResBlock, SArray* pColMatchInfo, - STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { + SSDataBlock* pResBlock, SArray* pColMatchInfo, STableListInfo* pTableListInfo, + SExecTaskInfo* pTaskInfo) { STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 7ad5a7ecab..5098ea9232 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -55,7 +55,9 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); +#ifdef JSON_TAG_REFACTOR int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId); +#endif int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 58a6d1483f..ebac38943f 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -54,7 +54,7 @@ typedef struct SInsertParseContext { SMsgBuf msg; // input STableMeta* pTableMeta; // each table SParsedDataColInfo tags; // each table - SKVRowBuilder tagsBuilder; // each table + STagVal* pTagVals; // each table SVCreateTbReq createTblReq; // each table SHashObj* pVgroupsHashObj; // global SHashObj* pTableBlockHashObj; // global @@ -72,9 +72,11 @@ static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; typedef struct SKvParam { - SKVRowBuilder* builder; - SSchema* schema; - char buf[TSDB_MAX_TAGS_LEN]; + int16_t nTag; + int16_t pos; + STagVal* pTagVals; + SSchema* schema; + char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; typedef struct SMemParam { @@ -212,7 +214,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con return buildInvalidOperationMsg(pMsgBuf, msg4); } - char tbname[TSDB_TABLE_FNAME_LEN] = {0}; + char tbname[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(tbname, p + 1, tbLen); /*tbLen = */ strdequote(tbname); @@ -619,14 +621,14 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_NCHAR: { return func(pMsgBuf, pToken->z, pToken->n, param); } - +#ifdef JSON_TAG_REFACTOR case TSDB_DATA_TYPE_JSON: { if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); } return func(pMsgBuf, pToken->z, pToken->n, param); } - +#endif case TSDB_DATA_TYPE_TIMESTAMP: { int64_t tmpVal; if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -757,9 +759,11 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi int8_t type = pa->schema->type; int16_t colId = pa->schema->colId; +#ifdef JSON_TAG_REFACTOR if (TSDB_DATA_TYPE_JSON == type) { return parseJsontoTagData(value, pa->builder, pMsgBuf, colId); } +#endif if (value == NULL) { // it is a null data // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, @@ -768,55 +772,63 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi } if (TSDB_DATA_TYPE_BINARY == type) { - STR_WITH_SIZE_TO_VARSTR(pa->buf, value, len); - tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf)); + memcpy(pa->buf + pa->pos, value, len); + tTagValSet(pa->pTagVals + pa->nTag++, &colId, type, (uint8_t*)(pa->buf + pa->pos), len, false); + pa->pos += len; } else if (TSDB_DATA_TYPE_NCHAR == type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' + + ASSERT((pa->pos + pa->schema->bytes - VARSTR_HEADER_SIZE) <= TSDB_MAX_TAGS_LEN); + int32_t output = 0; - if (!taosMbsToUcs4(value, len, (TdUcs4*)varDataVal(pa->buf), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(value, len, (TdUcs4*)(pa->buf + pa->pos), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { if (errno == E2BIG) { return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name); } - char buf[512] = {0}; snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); return buildSyntaxErrMsg(pMsgBuf, buf, value); } - - varDataSetLen(pa->buf, output); - tdAddColToKVRow(pa->builder, colId, pa->buf, varDataTLen(pa->buf)); + tTagValSet(pa->pTagVals + pa->nTag++, &colId, type, (uint8_t*)(pa->buf + pa->pos), output, false); + pa->pos += output; } else { - tdAddColToKVRow(pa->builder, colId, value, TYPE_BYTES[type]); + memcpy(pa->buf + pa->pos, value, TYPE_BYTES[type]); + tTagValSet(pa->pTagVals + pa->nTag++, &colId, type, (uint8_t*)(pa->buf + pa->pos), TYPE_BYTES[type], false); + pa->pos + TYPE_BYTES[type]; } + ASSERT(pa->pos <= TSDB_MAX_TAGS_LEN); return TSDB_CODE_SUCCESS; } -static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, SKVRow row, int64_t suid) { +static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid) { pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; - pTbReq->ctb.pTag = row; + pTbReq->ctb.pTag = (uint8_t*)pTag; return TSDB_CODE_SUCCESS; } // pSql -> tag1_value, ...) static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { - if (tdInitKVRowBuilder(&pCxt->tagsBuilder) < 0) { + ASSERT(!pCxt->pTagVals); + if (!(pCxt->pTagVals = taosMemoryCalloc(pCxt->tags.numOfBound, sizeof(STagVal)))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.builder = &pCxt->tagsBuilder}; + SKvParam param = {.pTagVals = pCxt->pTagVals, .nTag = 0, .pos = 0}; SToken sToken; bool isParseBindParam = false; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" + // TODO: JSON_TAG_REFACTOR => here would have json tag? for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); if (sToken.type == TK_NK_QUESTION) { isParseBindParam = true; if (NULL == pCxt->pStmtCb) { + taosMemoryFreeClear(pCxt->pTagVals); return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); } @@ -824,6 +836,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } if (isParseBindParam) { + taosMemoryFreeClear(pCxt->pTagVals); return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); } @@ -834,16 +847,19 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } if (isParseBindParam) { + taosMemoryFreeClear(pCxt->pTagVals); return TSDB_CODE_SUCCESS; } - SKVRow row = tdGetKVRowFromBuilder(&pCxt->tagsBuilder); - if (NULL == row) { + // TODO: JSON_TAG_REFACTOR (would be JSON tag or normal tag) + STag* pTag = NULL; + if (tTagNew(param.pTagVals, param.nTag, 1, false, &pTag) != 0) { + taosMemoryFreeClear(pCxt->pTagVals); return buildInvalidOperationMsg(&pCxt->msg, "out of memory"); } - tdSortKVRowByColIdx(row); - return buildCreateTbReq(&pCxt->createTblReq, tName, row, pCxt->pTableMeta->suid); + taosMemoryFreeClear(pCxt->pTagVals); + return buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid); } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { @@ -1062,7 +1078,7 @@ void destroyCreateSubTbReq(SVCreateTbReq* pReq) { static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); - tdDestroyKVRowBuilder(&pCxt->tagsBuilder); + taosMemoryFreeClear(pCxt->pTagVals); destroyCreateSubTbReq(&pCxt->createTblReq); } @@ -1082,9 +1098,9 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { // VALUES (field1_value, ...) [(field1_value2, ...) ...] | FILE csv_file_path // [...]; static int32_t parseInsertBody(SInsertParseContext* pCxt) { - int32_t tbNum = 0; - char tbFName[TSDB_TABLE_FNAME_LEN]; - bool autoCreateTbl = false; + int32_t tbNum = 0; + char tbFName[TSDB_TABLE_FNAME_LEN]; + bool autoCreateTbl = false; // for each table while (1) { @@ -1186,8 +1202,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } memcpy(tags, &pCxt->tags, sizeof(pCxt->tags)); - (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, - pCxt->pTableBlockHashObj); + (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, + pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj); memset(&pCxt->tags, 0, sizeof(pCxt->tags)); pCxt->pVgroupsHashObj = NULL; @@ -1219,6 +1235,7 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .totalNum = 0, + .pTagVals = NULL, .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), .pStmtCb = pContext->pStmtCb}; @@ -1332,13 +1349,13 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN return TSDB_CODE_QRY_APP_ERROR; } - SKVRowBuilder tagBuilder; - if (tdInitKVRowBuilder(&tagBuilder) < 0) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; + STagVal* pTagVals = taosMemoryCalloc(tags->numOfBound, sizeof(STagVal)); + if (!pTagVals) { + return buildInvalidOperationMsg(&pBuf, "out of memory"); } SSchema* pSchema = pDataBlock->pTableMeta->schema; - SKvParam param = {.builder = &tagBuilder}; + SKvParam param = {.pTagVals = pTagVals, .nTag = 0, .pos = 0}; for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { @@ -1357,19 +1374,19 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN CHECK_CODE(KvRowAppend(&pBuf, (char*)bind[c].buffer, colLen, ¶m)); } - SKVRow row = tdGetKVRowFromBuilder(&tagBuilder); - if (NULL == row) { - tdDestroyKVRowBuilder(&tagBuilder); + STag* pTag = NULL; + + // TODO: JSON_TAG_REFACTOR (if is json or not)? + if (0 != tTagNew(pTagVals, param.nTag, 1, false, &pTag)) { return buildInvalidOperationMsg(&pBuf, "out of memory"); } - tdSortKVRowByColIdx(row); SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, tName, row, suid)); + CHECK_CODE(buildCreateTbReq(&tbReq, tName, pTag, suid)); CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); destroyCreateSubTbReq(&tbReq); - tdDestroyKVRowBuilder(&tagBuilder); + taosMemoryFreeClear(pTagVals); return TSDB_CODE_SUCCESS; } @@ -1601,7 +1618,6 @@ int32_t qBuildStmtColFields(void* pBlock, int32_t* fieldNum, TAOS_FIELD** fields typedef struct SmlExecTableHandle { SParsedDataColInfo tags; // each table - SKVRowBuilder tagsBuilder; // each table SVCreateTbReq createTblReq; // each table } SmlExecTableHandle; @@ -1613,7 +1629,6 @@ typedef struct SmlExecHandle { static void smlDestroyTableHandle(void* pHandle) { SmlExecTableHandle* handle = (SmlExecTableHandle*)pHandle; - tdDestroyKVRowBuilder(&handle->tagsBuilder); destroyBoundColumnInfo(&handle->tags); destroyCreateSubTbReq(&handle->createTblReq); } @@ -1689,13 +1704,23 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS return TSDB_CODE_SUCCESS; } -static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedDataColInfo* tags, SSchema* pSchema, - SKVRow* row, SMsgBuf* msg) { - if (tdInitKVRowBuilder(tagsBuilder) < 0) { +/** + * @brief No json tag for schemaless + * + * @param cols + * @param tags + * @param pSchema + * @param ppTag + * @param msg + * @return int32_t + */ +static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SMsgBuf* msg) { + STagVal* pTagVals = taosMemoryCalloc(tags->numOfBound, sizeof(STagVal)); + if (!pTagVals) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.builder = tagsBuilder}; + SKvParam param = {.pTagVals = pTagVals, .nTag = 0, .pos = 0}; for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; param.schema = pTagSchema; @@ -1707,11 +1732,12 @@ static int32_t smlBuildTagRow(SArray* cols, SKVRowBuilder* tagsBuilder, SParsedD } } - *row = tdGetKVRowFromBuilder(tagsBuilder); - if (*row == NULL) { + if (tTagNew(pTagVals, param.nTag, 1, false, ppTag) != 0) { + taosMemoryFree(pTagVals); return TSDB_CODE_OUT_OF_MEMORY; } - tdSortKVRowByColIdx(*row); + + taosMemoryFree(pTagVals); return TSDB_CODE_SUCCESS; } @@ -1728,14 +1754,13 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols buildInvalidOperationMsg(&pBuf, "bound tags error"); return ret; } - SKVRow row = NULL; - ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tagsBuilder, &smlHandle->tableExecHandle.tags, pTagsSchema, - &row, &pBuf); + STag* pTag = NULL; + ret = smlBuildTagRow(tags, &smlHandle->tableExecHandle.tags, pTagsSchema, &pTag, &pBuf); if (ret != TSDB_CODE_SUCCESS) { return ret; } - buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, row, pTableMeta->suid); + buildCreateTbReq(&smlHandle->tableExecHandle.createTblReq, tableName, pTag, pTableMeta->suid); STableDataBlocks* pDataBlock = NULL; ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d142d89373..c22f9fe211 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4113,7 +4113,7 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { return code; } -static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, SKVRow row, +static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, const STag *pTag, uint64_t suid, SVgroupInfo* pVgInfo) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; @@ -4124,7 +4124,7 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S req.type = TD_CHILD_TABLE; req.name = strdup(pStmt->tableName); req.ctb.suid = suid; - req.ctb.pTag = row; + req.ctb.pTag = (uint8_t*)pTag; if (pStmt->ignoreExists) { req.flags |= TD_CREATE_IF_NOT_EXISTS; } @@ -4144,8 +4144,9 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S } } -static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, - SKVRowBuilder* pBuilder) { +// static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, +// SKVRowBuilder* pBuilder) { +#ifdef JSON_TAG_REFACTOR if (pSchema->type == TSDB_DATA_TYPE_JSON) { if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); @@ -4153,14 +4154,15 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); } +#endif - if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal), - IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); - } +// if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { +// tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal), +// IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); +// } - return TSDB_CODE_SUCCESS; -} +// return TSDB_CODE_SUCCESS; +// } static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) { int32_t code = getFuncInfo(pCxt, pFunc); @@ -4189,18 +4191,28 @@ static int32_t translateTagVal(STranslateContext* pCxt, uint8_t precision, SSche } static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, - SKVRowBuilder* pBuilder) { + STag** ppTag) { int32_t numOfTags = getNumOfTags(pSuperTableMeta); if (LIST_LENGTH(pStmt->pValsOfTags) != LIST_LENGTH(pStmt->pSpecificTags) || numOfTags < LIST_LENGTH(pStmt->pValsOfTags)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TAGS_NOT_MATCHED); } + STagVal* pTagVals = (STagVal*)taosMemoryCalloc(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); + char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); + if (!pTagVals || !pTagBuf) { + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(pTagBuf); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); + } + int32_t code = 0; + int16_t nTags = 0, nBufPos = 0; SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); SNode * pTag, *pNode; FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) { SColumnNode* pCol = (SColumnNode*)pTag; SSchema* pSchema = NULL; + STagVal* pTagVal = pTagVals + nTags; for (int32_t i = 0; i < numOfTags; ++i) { if (0 == strcmp(pCol->colName, pTagSchema[i].name)) { pSchema = pTagSchema + i; @@ -4208,10 +4220,12 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } if (NULL == pSchema) { + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(pTagBuf); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); } SValueNode* pVal = NULL; - int32_t code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal); + code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal); if (TSDB_CODE_SUCCESS == code) { if (NULL == pVal) { pVal = (SValueNode*)pNode; @@ -4219,29 +4233,74 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla REPLACE_LIST2_NODE(pVal); } } +#ifdef JSON_TAG_REFACTOR if (TSDB_CODE_SUCCESS == code) { code = addValToKVRow(pCxt, pVal, pSchema, pBuilder); } +#endif + + if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + // TODO: JSON_TAG_TODO: is copy is a must? + void* nodeVal = nodesGetValueFromNode(pVal); + if (IS_VAR_DATA_TYPE(pSchema->type)) { + memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), varDataLen(nodeVal)); + tTagValSet(pTagVal, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, varDataLen(nodeVal), false); + nBufPos += varDataLen(pVal->datum.p); + } else { + memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), TYPE_BYTES[pSchema->type]); + tTagValSet(pTagVal, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, TYPE_BYTES[pSchema->type], + false); + nBufPos += TYPE_BYTES[pSchema->type]; + } + } + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(pTagBuf); return code; } } + // TODO: JSON_TAG_TODO: version + code = tTagNew(pTagVals, nTags, 1, false, ppTag); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(pTagBuf); + return code; + } + + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(pTagBuf); return TSDB_CODE_SUCCESS; } static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClause* pStmt, STableMeta* pSuperTableMeta, - SKVRowBuilder* pBuilder) { + STag** ppTag) { if (getNumOfTags(pSuperTableMeta) != LIST_LENGTH(pStmt->pValsOfTags)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TAGS_NOT_MATCHED); } - SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); + SSchema* pTagSchemas = getTableTagSchema(pSuperTableMeta); SNode* pNode; + int32_t code = 0; int32_t index = 0; + int16_t nTag = 0; + STagVal* pTagVals = taosMemoryCalloc(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); + char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); + + const char* qTagBuf = pTagBuf; + + if (!pTagVals || !pTagBuf) { + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(qTagBuf); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); + } + FOREACH(pNode, pStmt->pValsOfTags) { SValueNode* pVal = NULL; - int32_t code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema + index, pNode, &pVal); + STagVal* pTagVal = pTagVals + nTag; + SSchema* pTagSchema = pTagSchemas + index; + code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema, pNode, &pVal); if (TSDB_CODE_SUCCESS == code) { if (NULL == pVal) { pVal = (SValueNode*)pNode; @@ -4249,14 +4308,48 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau REPLACE_NODE(pVal); } } +#ifdef JSON_TAG_REFACTOR if (TSDB_CODE_SUCCESS == code) { code = addValToKVRow(pCxt, pVal, pTagSchema + index++, pBuilder); } +#endif + if (TSDB_CODE_SUCCESS == code) { + if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + char* tmpVal = nodesGetValueFromNode(pVal); + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + memcpy(pTagBuf, varDataVal(tmpVal), varDataLen(tmpVal)); + tTagValSet(pTagVal, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, varDataLen(tmpVal), + false); + pTagBuf += varDataLen(tmpVal); + } else { + memcpy(pTagBuf, tmpVal, TYPE_BYTES[pTagSchema->type]); + tTagValSet(pTagVal, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, + TYPE_BYTES[pTagSchema->type], false); + pTagBuf += TYPE_BYTES[pTagSchema->type]; + } + ++nTag; + } + ++index; + } + // TODO: buf is need to store the tags + + // TODO: JSON_TAG_TODO remove below codes if code is 0 all the time. if (TSDB_CODE_SUCCESS != code) { - return code; + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(qTagBuf); + return generateSyntaxErrMsg(&pCxt->msgBuf, code); } } + // TODO: JSON_TAG_TODO: version? + // TODO: JSON_TAG_REFACTOR: json or not + if (TSDB_CODE_SUCCESS != (code = tTagNew(pTagVals, nTag, 1, false, ppTag))) { + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(qTagBuf); + return generateSyntaxErrMsg(&pCxt->msgBuf, code); + } + taosMemoryFreeClear(pTagVals); + taosMemoryFreeClear(qTagBuf); return TSDB_CODE_SUCCESS; } @@ -4274,26 +4367,13 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableMeta(pCxt, pStmt->useDbName, pStmt->useTableName, &pSuperTableMeta); } - SKVRowBuilder kvRowBuilder = {0}; - if (TSDB_CODE_SUCCESS == code) { - code = tdInitKVRowBuilder(&kvRowBuilder); - } + STag* pTag = NULL; if (TSDB_CODE_SUCCESS == code) { if (NULL != pStmt->pSpecificTags) { - code = buildKVRowForBindTags(pCxt, pStmt, pSuperTableMeta, &kvRowBuilder); + code = buildKVRowForBindTags(pCxt, pStmt, pSuperTableMeta, &pTag); } else { - code = buildKVRowForAllTags(pCxt, pStmt, pSuperTableMeta, &kvRowBuilder); - } - } - - SKVRow row = NULL; - if (TSDB_CODE_SUCCESS == code) { - row = tdGetKVRowFromBuilder(&kvRowBuilder); - if (NULL == row) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - tdSortKVRowByColIdx(row); + code = buildKVRowForAllTags(pCxt, pStmt, pSuperTableMeta, &pTag); } } @@ -4302,11 +4382,10 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info); } if (TSDB_CODE_SUCCESS == code) { - addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, row, pSuperTableMeta->uid, &info); + addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, pTag, pSuperTableMeta->uid, &info); } taosMemoryFreeClear(pSuperTableMeta); - tdDestroyKVRowBuilder(&kvRowBuilder); return code; } @@ -4528,6 +4607,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) { +#ifdef JSON_TAG_REFACTOR SKVRowBuilder kvRowBuilder = {0}; int32_t code = tdInitKVRowBuilder(&kvRowBuilder); @@ -4553,6 +4633,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->pTagVal = row; pStmt->pVal->datum.p = row; // for free tdDestroyKVRowBuilder(&kvRowBuilder); +#endif } else { pReq->nTagVal = pStmt->pVal->node.resType.bytes; if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 9de43b8cd3..d070ffaf5c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -328,6 +328,7 @@ static bool isValidateTag(char* input) { return true; } +#ifdef JSON_TAG_REFACTOR int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId) { // set json NULL data uint8_t jsonNULL = TSDB_DATA_TYPE_NULL; @@ -448,6 +449,7 @@ end: cJSON_Delete(root); return retCode; } +#endif static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { if (NULL != pTablesHash) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 0fb3712c30..b68c292feb 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -922,8 +922,13 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -char *getJsonValue(char *json, char *key){ //todo - json++; // jump type +char *getJsonValue(char *json, char *key) { // todo + json++; // jump type + + STagVal tagVal = {.pKey = key}; + tTagGet(((const STag *)json), &tagVal); + return (char *)tagVal.pData; +#if 0 int16_t cols = kvRowNCols(json); for (int i = 0; i < cols; ++i) { SColIdx *pColIdx = kvRowColIdxAt(json, i); @@ -939,6 +944,7 @@ char *getJsonValue(char *json, char *key){ //todo } } return NULL; +#endif } void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 3fafc83b18..3a16e0a969 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1035,7 +1035,7 @@ void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ SNode *pLeft = NULL, *pRight = NULL; scltMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, keyVar); - scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, kvRowLen(json), 1, json); + scltMakeColumnNode(&pLeft, src, TSDB_DATA_TYPE_JSON, ((STag*)json)->len, 1, json); scltMakeOpNode(opNode, OP_TYPE_JSON_GET_VALUE, TSDB_DATA_TYPE_JSON, pLeft, pRight); } @@ -1105,6 +1105,7 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do nodesDestroyNode(opNode); } +#if 0 TEST(columnTest, json_column_arith_op) { scltInitLogFile(); char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; @@ -1178,7 +1179,7 @@ TEST(columnTest, json_column_arith_op) { tdDestroyKVRowBuilder(&kvRowBuilder); taosMemoryFree(row); } - +#endif void *prepareNchar(char* rightData){ int32_t len = 0; int32_t inputLen = strlen(rightData); @@ -1188,7 +1189,7 @@ void *prepareNchar(char* rightData){ varDataSetLen(t, len); return t; } - +#if 0 TEST(columnTest, json_column_logic_op) { scltInitLogFile(); char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}"; @@ -1308,6 +1309,7 @@ TEST(columnTest, json_column_logic_op) { tdDestroyKVRowBuilder(&kvRowBuilder); taosMemoryFree(row); } +#endif TEST(columnTest, smallint_value_add_int_column) { scltInitLogFile(); From a32dd76bf8e1fe03fcfb60f6c37fac9040461917 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 28 May 2022 09:42:23 +0000 Subject: [PATCH 06/22] feat: optimize tag --- source/common/src/tdataformat.c | 64 ++++++++++++++++++++++++--------- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 0d74800d4e..f42b5e465b 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -32,12 +32,14 @@ typedef struct { #pragma pack(pop) #pragma pack(push, 1) +#define TD_TAG_JSON ((int8_t)0x1) +#define TD_TAG_LARGE ((int8_t)0x2) struct STag { - int8_t isJson; + int8_t flags; int16_t len; int16_t nTag; int32_t ver; - int16_t idx[]; + int8_t idx[]; }; #pragma pack(pop) @@ -525,7 +527,6 @@ static int tTagValCmprFn(const void *p1, const void *p2) { return 1; } - ASSERT(0); return 0; } static int tTagValJsonCmprFn(const void *p1, const void *p2) { @@ -584,7 +585,8 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { uint8_t *p = NULL; int16_t n = 0; int16_t nTag = taosArrayGetSize(pArray); - int32_t szTag = sizeof(STag) + sizeof(int16_t) * nTag; + int32_t szTag = 0; + int8_t isLarge = 0; // sort if (isJson) { @@ -597,12 +599,14 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { for (int16_t iTag = 0; iTag < nTag; iTag++) { szTag += tPutTagVal(NULL, (STagVal *)taosArrayGet(pArray, iTag), isJson); } + if (szTag <= INT8_MAX) { + szTag = szTag + sizeof(STag) + sizeof(int8_t) * nTag; + } else { + szTag = szTag + sizeof(STag) + sizeof(int16_t) * nTag; + isLarge = 1; + } - // TODO - // if (szTag >= 16 * 1024) { - // code = TSDB_CODE_IVLD_TAG; - // goto _err; - // } + ASSERT(szTag <= INT16_MAX); // build tag (*ppTag) = (STag *)taosMemoryMalloc(szTag); @@ -610,15 +614,29 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - (*ppTag)->isJson = isJson ? 1 : 0; + (*ppTag)->flags = 0; + if (isJson) { + (*ppTag)->flags |= TD_TAG_JSON; + } + if (isLarge) { + (*ppTag)->flags |= TD_TAG_LARGE; + } (*ppTag)->len = szTag; (*ppTag)->nTag = nTag; (*ppTag)->ver = version; - p = (uint8_t *)&(*ppTag)->idx[nTag]; + if (isLarge) { + p = (uint8_t *)&((int16_t *)(*ppTag)->idx)[nTag]; + } else { + p = (uint8_t *)&(*ppTag)->idx[nTag]; + } n = 0; for (int16_t iTag = 0; iTag < nTag; iTag++) { - (*ppTag)->idx[iTag] = n; + if (isLarge) { + ((int16_t *)(*ppTag)->idx)[iTag] = n; + } else { + (*ppTag)->idx[iTag] = n; + } n += tPutTagVal(p + n, (STagVal *)taosArrayGet(pArray, iTag), isJson); } @@ -636,18 +654,32 @@ void tTagGet(STag *pTag, STagVal *pTagVal) { int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; int16_t midx; - uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag]; + uint8_t *p; + int8_t isJson = pTag->flags & TD_TAG_JSON; + int8_t isLarge = pTag->flags & TD_TAG_LARGE; + int16_t offset; STagVal tv; int c; + if (isLarge) { + p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; + } else { + p = (uint8_t *)&pTag->idx[pTag->nTag]; + } + pTagVal->type = TSDB_DATA_TYPE_NULL; pTagVal->pData = NULL; pTagVal->nData = 0; while (lidx <= ridx) { midx = (lidx + ridx) / 2; + if (isLarge) { + offset = ((int16_t *)pTag->idx)[midx]; + } else { + offset = pTag->idx[midx]; + } - tGetTagVal(p + pTag->idx[midx], &tv, pTag->isJson); - if (pTag->isJson) { + tGetTagVal(p + offset, &tv, isJson); + if (isJson) { c = tTagValJsonCmprFn(pTagVal, &tv); } else { c = tTagValCmprFn(pTagVal, &tv); @@ -684,7 +716,7 @@ int32_t tTagToValArray(STag *pTag, SArray **ppArray) { } for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { - tGetTagVal(p + pTag->idx[iTag], &tv, pTag->isJson); + tGetTagVal(p + pTag->idx[iTag], &tv, pTag->flags & TD_TAG_JSON); taosArrayPush(*ppArray, &tv); } From 292d57993258ce8f15a249f577a09284e0b1cea8 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 28 May 2022 18:05:50 +0800 Subject: [PATCH 07/22] feat: adapt for interface --- include/common/tdataformat.h | 20 ++++---- source/common/src/tdatablock.c | 28 +++++++++-- source/dnode/vnode/src/meta/metaTable.c | 21 +++----- source/libs/parser/src/parInsert.c | 46 ++++++++--------- source/libs/parser/src/parTranslater.c | 67 ++++++++++++------------- 5 files changed, 94 insertions(+), 88 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 6b74549b62..9cc7a3049d 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -60,8 +60,8 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, u int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); // STagVal -static FORCE_INLINE void tTagValSet(STagVal *pTagVal, void *key, int8_t type, uint8_t *pData, uint32_t nData, - bool isJson); +static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, + bool isJson); // STag int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); @@ -133,17 +133,19 @@ struct STagVal { uint8_t *pData; }; -static FORCE_INLINE void tTagValSet(STagVal *pTagVal, void *key, int8_t type, uint8_t *pData, uint32_t nData, - bool isJson) { +static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, + bool isJson) { + STagVal tagVal = {0}; if (isJson) { - pTagVal->pKey = (char *)key; + tagVal.pKey = (char *)key; } else { - pTagVal->cid = *(int16_t *)key; + tagVal.cid = *(int16_t *)key; } - pTagVal->type = type; - pTagVal->pData = pData; - pTagVal->nData = nData; + tagVal.type = type; + tagVal.pData = pData; + tagVal.nData = nData; + taosArrayPush(pTagArray, &tagVal); } #pragma pack(push, 1) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 34ec5a1251..1918061837 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1631,6 +1631,11 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, const char* stbFullName, int32_t vgId) { SSubmitReq* ret = NULL; + SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); + if(!tagArray) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } // cal size int32_t cap = sizeof(SSubmitReq); @@ -1652,14 +1657,19 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.type = TSDB_CHILD_TABLE; createTbReq.ctb.suid = suid; + + STagVal tagVal = {.cid = 1, .type = TSDB_DATA_TYPE_UBIGINT, .pData = (uint8_t*)&pDataBlock->info.groupId, .nData = sizeof(uint64_t)}; STag* pTag = NULL; - tTagNew(&tagVal, 1, 1, false, &pTag); + taosArrayClear(tagArray); + taosArrayPush(tagArray, &tagVal); + tTagNew(tagArray, 1, false, &pTag); if (!pTag) { tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); return NULL; } createTbReq.ctb.pTag = (uint8_t*)pTag; @@ -1669,7 +1679,11 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo tdDestroySVCreateTbReq(&createTbReq); - if (code < 0) return NULL; + if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); + return NULL; + } } cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; @@ -1716,10 +1730,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo .type = TSDB_DATA_TYPE_UBIGINT, .pData = (uint8_t*)&pDataBlock->info.groupId, .nData = sizeof(uint64_t)}; - STag* pTag = NULL; - tTagNew(&tagVal, 1, 1, false, &pTag); + taosArrayClear(tagArray); + taosArrayPush(tagArray, &tagVal); + STag* pTag = NULL; + tTagNew(tagArray, 1, false, &pTag); if (!pTag) { tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); taosMemoryFreeClear(ret); return NULL; } @@ -1729,6 +1746,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); if (code < 0) { tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(tagArray); taosMemoryFreeClear(ret); return NULL; } @@ -1740,6 +1758,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo tdDestroySVCreateTbReq(&createTbReq); if (code < 0) { + taosArrayDestroy(tagArray); taosMemoryFreeClear(ret); return NULL; } @@ -1777,5 +1796,6 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } ret->length = htonl(ret->length); + taosArrayDestroy(tagArray); return ret; } diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index e1f3915053..753bd8bce7 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -565,37 +565,32 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA } else { const STag *pOldTag = (const STag *)ctbEntry.ctbEntry.pTags; STag *pNewTag = NULL; - STagVal *pTagVals = taosMemoryCalloc(pTagSchema->nCols, sizeof(STagVal)); - - if (!pTagVals) { + SArray *pTagArray = taosArrayInit(pTagSchema->nCols, sizeof(STagVal)); + if (!pTagArray) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - int16_t nTags = 0; for (int32_t i = 0; i < pTagSchema->nCols; i++) { SSchema *pCol = &pTagSchema->pSchema[i]; - STagVal *pTagVal = pTagVals + nTags; if (iCol == i) { - tTagValSet(pTagVal, &pCol->colId, pCol->type, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal, false); - ++nTags; + tTagValPush(pTagArray, &pCol->colId, pCol->type, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal, false); } else { STagVal tagVal = {.cid = pCol->colId}; if (tTagGet(pOldTag, &tagVal) && tagVal.pData) { if (IS_VAR_DATA_TYPE(pCol->type)) { - tTagValSet(pTagVal, &pCol->colId, pCol->type, tagVal.pData, varDataTLen(tagVal.pData), false); + tTagValPush(pTagArray, &pCol->colId, pCol->type, tagVal.pData, varDataTLen(tagVal.pData), false); } else { - tTagValSet(pTagVal, &pCol->colId, pCol->type, tagVal.pData, pCol->bytes, false); + tTagValPush(pTagArray, &pCol->colId, pCol->type, tagVal.pData, pCol->bytes, false); } - ++nTags; } } } - if ((terrno = tTagNew(pTagVals, nTags, pTagSchema->version, false, &pNewTag)) < 0) { - taosMemoryFreeClear(pTagVals); + if ((terrno = tTagNew(pTagArray, pTagSchema->version, false, &pNewTag)) < 0) { + taosArrayDestroy(pTagArray); goto _err; } ctbEntry.ctbEntry.pTags = (uint8_t *)pNewTag; - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); } // save to table.db diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index ebac38943f..37ee6df682 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -54,7 +54,7 @@ typedef struct SInsertParseContext { SMsgBuf msg; // input STableMeta* pTableMeta; // each table SParsedDataColInfo tags; // each table - STagVal* pTagVals; // each table + SArray* pTagVals; // each table SVCreateTbReq createTblReq; // each table SHashObj* pVgroupsHashObj; // global SHashObj* pTableBlockHashObj; // global @@ -72,9 +72,8 @@ static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE; static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE; typedef struct SKvParam { - int16_t nTag; int16_t pos; - STagVal* pTagVals; + SArray* pTagVals; SSchema* schema; char buf[TSDB_MAX_TAGS_LEN]; } SKvParam; @@ -773,7 +772,7 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi if (TSDB_DATA_TYPE_BINARY == type) { memcpy(pa->buf + pa->pos, value, len); - tTagValSet(pa->pTagVals + pa->nTag++, &colId, type, (uint8_t*)(pa->buf + pa->pos), len, false); + tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), len, false); pa->pos += len; } else if (TSDB_DATA_TYPE_NCHAR == type) { // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' @@ -789,11 +788,11 @@ static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, voi snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); return buildSyntaxErrMsg(pMsgBuf, buf, value); } - tTagValSet(pa->pTagVals + pa->nTag++, &colId, type, (uint8_t*)(pa->buf + pa->pos), output, false); + tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), output, false); pa->pos += output; } else { memcpy(pa->buf + pa->pos, value, TYPE_BYTES[type]); - tTagValSet(pa->pTagVals + pa->nTag++, &colId, type, (uint8_t*)(pa->buf + pa->pos), TYPE_BYTES[type], false); + tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), TYPE_BYTES[type], false); pa->pos + TYPE_BYTES[type]; } ASSERT(pa->pos <= TSDB_MAX_TAGS_LEN); @@ -813,11 +812,11 @@ static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* // pSql -> tag1_value, ...) static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { ASSERT(!pCxt->pTagVals); - if (!(pCxt->pTagVals = taosMemoryCalloc(pCxt->tags.numOfBound, sizeof(STagVal)))) { + if (!(pCxt->pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)))) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.pTagVals = pCxt->pTagVals, .nTag = 0, .pos = 0}; + SKvParam param = {.pTagVals = pCxt->pTagVals, .pos = 0}; SToken sToken; bool isParseBindParam = false; char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" @@ -828,7 +827,6 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint if (sToken.type == TK_NK_QUESTION) { isParseBindParam = true; if (NULL == pCxt->pStmtCb) { - taosMemoryFreeClear(pCxt->pTagVals); return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); } @@ -836,7 +834,6 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } if (isParseBindParam) { - taosMemoryFreeClear(pCxt->pTagVals); return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); } @@ -847,18 +844,15 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } if (isParseBindParam) { - taosMemoryFreeClear(pCxt->pTagVals); return TSDB_CODE_SUCCESS; } // TODO: JSON_TAG_REFACTOR (would be JSON tag or normal tag) STag* pTag = NULL; - if (tTagNew(param.pTagVals, param.nTag, 1, false, &pTag) != 0) { - taosMemoryFreeClear(pCxt->pTagVals); + if (tTagNew(param.pTagVals, 1, false, &pTag) != 0) { return buildInvalidOperationMsg(&pCxt->msg, "out of memory"); } - taosMemoryFreeClear(pCxt->pTagVals); return buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid); } @@ -1078,7 +1072,7 @@ void destroyCreateSubTbReq(SVCreateTbReq* pReq) { static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); - taosMemoryFreeClear(pCxt->pTagVals); + taosArrayDestroy(pCxt->pTagVals); destroyCreateSubTbReq(&pCxt->createTblReq); } @@ -1349,13 +1343,13 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN return TSDB_CODE_QRY_APP_ERROR; } - STagVal* pTagVals = taosMemoryCalloc(tags->numOfBound, sizeof(STagVal)); - if (!pTagVals) { + SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); + if (!pTagArray) { return buildInvalidOperationMsg(&pBuf, "out of memory"); } SSchema* pSchema = pDataBlock->pTableMeta->schema; - SKvParam param = {.pTagVals = pTagVals, .nTag = 0, .pos = 0}; + SKvParam param = {.pTagVals = pTagArray, .pos = 0}; for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { @@ -1377,7 +1371,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN STag* pTag = NULL; // TODO: JSON_TAG_REFACTOR (if is json or not)? - if (0 != tTagNew(pTagVals, param.nTag, 1, false, &pTag)) { + if (0 != tTagNew(pTagArray, 1, false, &pTag)) { return buildInvalidOperationMsg(&pBuf, "out of memory"); } @@ -1386,7 +1380,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); destroyCreateSubTbReq(&tbReq); - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); return TSDB_CODE_SUCCESS; } @@ -1715,12 +1709,12 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS * @return int32_t */ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* pSchema, STag** ppTag, SMsgBuf* msg) { - STagVal* pTagVals = taosMemoryCalloc(tags->numOfBound, sizeof(STagVal)); - if (!pTagVals) { + SArray* pTagArray = taosArrayInit(tags->numOfBound, sizeof(STagVal)); + if (!pTagArray) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.pTagVals = pTagVals, .nTag = 0, .pos = 0}; + SKvParam param = {.pTagVals = pTagArray, .pos = 0}; for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; param.schema = pTagSchema; @@ -1732,12 +1726,12 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p } } - if (tTagNew(pTagVals, param.nTag, 1, false, ppTag) != 0) { - taosMemoryFree(pTagVals); + if (tTagNew(pTagArray, 1, false, ppTag) != 0) { + taosArrayDestroy(pTagArray); return TSDB_CODE_OUT_OF_MEMORY; } - taosMemoryFree(pTagVals); + taosArrayDestroy(pTagArray); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index c22f9fe211..22357cab44 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4113,8 +4113,8 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) { return code; } -static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, const STag *pTag, - uint64_t suid, SVgroupInfo* pVgInfo) { +static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, + const STag* pTag, uint64_t suid, SVgroupInfo* pVgInfo) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId}; strcpy(name.dbname, pStmt->dbName); @@ -4147,13 +4147,13 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S // static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, // SKVRowBuilder* pBuilder) { #ifdef JSON_TAG_REFACTOR - if (pSchema->type == TSDB_DATA_TYPE_JSON) { - if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); - } - - return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); +if (pSchema->type == TSDB_DATA_TYPE_JSON) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); } + + return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); +} #endif // if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { @@ -4198,10 +4198,10 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TAGS_NOT_MATCHED); } - STagVal* pTagVals = (STagVal*)taosMemoryCalloc(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); - char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); - if (!pTagVals || !pTagBuf) { - taosMemoryFreeClear(pTagVals); + SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); + char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); + if (!pTagArray || !pTagBuf) { + taosArrayDestroy(pTagArray); taosMemoryFreeClear(pTagBuf); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); } @@ -4212,7 +4212,6 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) { SColumnNode* pCol = (SColumnNode*)pTag; SSchema* pSchema = NULL; - STagVal* pTagVal = pTagVals + nTags; for (int32_t i = 0; i < numOfTags; ++i) { if (0 == strcmp(pCol->colName, pTagSchema[i].name)) { pSchema = pTagSchema + i; @@ -4220,7 +4219,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } if (NULL == pSchema) { - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); taosMemoryFreeClear(pTagBuf); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); } @@ -4233,7 +4232,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla REPLACE_LIST2_NODE(pVal); } } -#ifdef JSON_TAG_REFACTOR +#ifdef JSON_TAG_REFACTOR if (TSDB_CODE_SUCCESS == code) { code = addValToKVRow(pCxt, pVal, pSchema, pBuilder); } @@ -4244,32 +4243,32 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla void* nodeVal = nodesGetValueFromNode(pVal); if (IS_VAR_DATA_TYPE(pSchema->type)) { memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), varDataLen(nodeVal)); - tTagValSet(pTagVal, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, varDataLen(nodeVal), false); + tTagValPush(pTagArray, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, varDataLen(nodeVal), false); nBufPos += varDataLen(pVal->datum.p); } else { memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), TYPE_BYTES[pSchema->type]); - tTagValSet(pTagVal, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, TYPE_BYTES[pSchema->type], - false); + tTagValPush(pTagArray, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, TYPE_BYTES[pSchema->type], + false); nBufPos += TYPE_BYTES[pSchema->type]; } } if (TSDB_CODE_SUCCESS != code) { - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); taosMemoryFreeClear(pTagBuf); return code; } } // TODO: JSON_TAG_TODO: version - code = tTagNew(pTagVals, nTags, 1, false, ppTag); + code = tTagNew(pTagArray, 1, false, ppTag); if (TSDB_CODE_SUCCESS != code) { - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); taosMemoryFreeClear(pTagBuf); return code; } - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); taosMemoryFreeClear(pTagBuf); return TSDB_CODE_SUCCESS; } @@ -4284,21 +4283,19 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau SNode* pNode; int32_t code = 0; int32_t index = 0; - int16_t nTag = 0; - STagVal* pTagVals = taosMemoryCalloc(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); + SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); const char* qTagBuf = pTagBuf; - if (!pTagVals || !pTagBuf) { - taosMemoryFreeClear(pTagVals); + if (!pTagArray || !pTagBuf) { + taosArrayDestroy(pTagArray); taosMemoryFreeClear(qTagBuf); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); } FOREACH(pNode, pStmt->pValsOfTags) { SValueNode* pVal = NULL; - STagVal* pTagVal = pTagVals + nTag; SSchema* pTagSchema = pTagSchemas + index; code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema, pNode, &pVal); if (TSDB_CODE_SUCCESS == code) { @@ -4318,16 +4315,14 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau char* tmpVal = nodesGetValueFromNode(pVal); if (IS_VAR_DATA_TYPE(pTagSchema->type)) { memcpy(pTagBuf, varDataVal(tmpVal), varDataLen(tmpVal)); - tTagValSet(pTagVal, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, varDataLen(tmpVal), - false); + tTagValPush(pTagArray, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, varDataLen(tmpVal), false); pTagBuf += varDataLen(tmpVal); } else { memcpy(pTagBuf, tmpVal, TYPE_BYTES[pTagSchema->type]); - tTagValSet(pTagVal, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, - TYPE_BYTES[pTagSchema->type], false); + tTagValPush(pTagArray, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, TYPE_BYTES[pTagSchema->type], + false); pTagBuf += TYPE_BYTES[pTagSchema->type]; } - ++nTag; } ++index; } @@ -4335,20 +4330,20 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau // TODO: JSON_TAG_TODO remove below codes if code is 0 all the time. if (TSDB_CODE_SUCCESS != code) { - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); taosMemoryFreeClear(qTagBuf); return generateSyntaxErrMsg(&pCxt->msgBuf, code); } } // TODO: JSON_TAG_TODO: version? // TODO: JSON_TAG_REFACTOR: json or not - if (TSDB_CODE_SUCCESS != (code = tTagNew(pTagVals, nTag, 1, false, ppTag))) { - taosMemoryFreeClear(pTagVals); + if (0 != (code = tTagNew(pTagArray, 1, false, ppTag))) { + taosArrayDestroy(pTagArray); taosMemoryFreeClear(qTagBuf); return generateSyntaxErrMsg(&pCxt->msgBuf, code); } - taosMemoryFreeClear(pTagVals); + taosArrayDestroy(pTagArray); taosMemoryFreeClear(qTagBuf); return TSDB_CODE_SUCCESS; } From b73cf23cc6f90f92863c3b4ae3495f3cb1cb5223 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 28 May 2022 18:24:57 +0800 Subject: [PATCH 08/22] fix: adapt the code according to large or small offset --- source/common/src/tdataformat.c | 71 +++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 13 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 7f58159e30..1a7f5fa3fb 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -574,20 +574,52 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c } } + // if (isLarge) { + // p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; + // } else { + // p = (uint8_t *)&pTag->idx[pTag->nTag]; + // } + + // (*ppArray) = taosArrayInit(pTag->nTag + 1, sizeof(STagVal)); + // if (*ppArray == NULL) { + // code = TSDB_CODE_OUT_OF_MEMORY; + // goto _err; + // } + + // for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { + // if (isLarge) { + // offset = ((int16_t *)pTag->idx)[iTag]; + // } else { + // offset = pTag->idx[iTag]; + // } + void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { - printf("%s:%d >>> STAG === isJson:%s, len: %d, nTag: %d, sver:%d\n", tag, ln, pTag->isJson ? "true" : "false", - (int32_t)pTag->len, (int32_t)pTag->nTag, pTag->ver); - char *p = (char *)&pTag->idx[pTag->nTag]; + int8_t isJson = pTag->flags & TD_TAG_JSON; + int8_t isLarge = pTag->flags & TD_TAG_LARGE; + uint8_t *p = NULL; + int16_t offset = 0; + + if (isLarge) { + p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; + } else { + p = (uint8_t *)&pTag->idx[pTag->nTag]; + } + printf("%s:%d >>> STAG === %s:%s, len: %d, nTag: %d, sver:%d\n", tag, ln, isJson ? "json" : "normal", + isLarge ? "large" : "small", (int32_t)pTag->len, (int32_t)pTag->nTag, pTag->ver); for (uint16_t n = 0; n < pTag->nTag; ++n) { - int16_t *pIdx = pTag->idx + n; - STagVal tagVal = {0}; - if (pTag->isJson) { - tagVal.pKey = (char *)POINTER_SHIFT(p, *pIdx); + if (isLarge) { + offset = ((int16_t *)pTag->idx)[n]; } else { - tagVal.cid = *(int16_t *)POINTER_SHIFT(p, *pIdx); + offset = pTag->idx[n]; } - printf("%s:%d loop[%d-%d] offset=%d\n", __func__, __LINE__, (int32_t)pTag->nTag, (int32_t)n, *pIdx); - tGetTagVal(p, &tagVal, pTag->isJson); + STagVal tagVal = {0}; + if (isJson) { + tagVal.pKey = (char *)POINTER_SHIFT(p, offset); + } else { + tagVal.cid = *(int16_t *)POINTER_SHIFT(p, offset); + } + printf("%s:%d loop[%d-%d] offset=%d\n", __func__, __LINE__, (int32_t)pTag->nTag, (int32_t)n, (int32_t)offset); + tGetTagVal(p, &tagVal, isJson); debugPrintTagVal(tagVal.type, tagVal.pData, tagVal.nData, __func__, __LINE__); } printf("\n"); @@ -771,8 +803,16 @@ int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { return tDecodeBinary(pDec int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { int32_t code = 0; - uint8_t *p = (uint8_t *)&pTag->idx[pTag->nTag]; - STagVal tv; + uint8_t *p = NULL; + STagVal tv = {0}; + int8_t isLarge = pTag->flags & TD_TAG_LARGE; + int16_t offset = 0; + + if (isLarge) { + p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; + } else { + p = (uint8_t *)&pTag->idx[pTag->nTag]; + } (*ppArray) = taosArrayInit(pTag->nTag + 1, sizeof(STagVal)); if (*ppArray == NULL) { @@ -781,7 +821,12 @@ int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { } for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { - tGetTagVal(p + pTag->idx[iTag], &tv, pTag->flags & TD_TAG_JSON); + if (isLarge) { + offset = ((int16_t *)pTag->idx)[iTag]; + } else { + offset = pTag->idx[iTag]; + } + tGetTagVal(p + offset, &tv, pTag->flags & TD_TAG_JSON); taosArrayPush(*ppArray, &tv); } From 174eb2d8625f3eea04407e5529ea9f4bc51f1694 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sat, 28 May 2022 20:13:59 +0800 Subject: [PATCH 09/22] fix: add more debug logs --- source/common/src/tdataformat.c | 4 ++-- source/common/src/tmsg.c | 2 ++ source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/parser/src/parTranslater.c | 13 +++++++++++++ 4 files changed, 18 insertions(+), 3 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1a7f5fa3fb..dce7c1739e 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -619,7 +619,7 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { tagVal.cid = *(int16_t *)POINTER_SHIFT(p, offset); } printf("%s:%d loop[%d-%d] offset=%d\n", __func__, __LINE__, (int32_t)pTag->nTag, (int32_t)n, (int32_t)offset); - tGetTagVal(p, &tagVal, isJson); + tGetTagVal(p + offset, &tagVal, isJson); debugPrintTagVal(tagVal.type, tagVal.pData, tagVal.nData, __func__, __LINE__); } printf("\n"); @@ -703,7 +703,7 @@ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { ASSERT(szTag <= INT16_MAX); // build tag - (*ppTag) = (STag *)taosMemoryMalloc(szTag); + (*ppTag) = (STag *)taosMemoryCalloc(szTag, 1); if ((*ppTag) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 721eb9587b..e10dfc2996 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3847,6 +3847,7 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (pReq->type == TSDB_CHILD_TABLE) { if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1; if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1; + debugPrintSTag((STag*)pReq->ctb.pTag, __func__, __LINE__); } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { @@ -3870,6 +3871,7 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { if (pReq->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1; if (tDecodeTag(pCoder, (STag **)&pReq->ctb.pTag) < 0) return -1; + debugPrintSTag((STag*)pReq->ctb.pTag, __func__, __LINE__); } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 753bd8bce7..aa235c1cc9 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -578,7 +578,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA STagVal tagVal = {.cid = pCol->colId}; if (tTagGet(pOldTag, &tagVal) && tagVal.pData) { if (IS_VAR_DATA_TYPE(pCol->type)) { - tTagValPush(pTagArray, &pCol->colId, pCol->type, tagVal.pData, varDataTLen(tagVal.pData), false); + tTagValPush(pTagArray, &pCol->colId, pCol->type, varDataVal(tagVal.pData), varDataLen(tagVal.pData), false); } else { tTagValPush(pTagArray, &pCol->colId, pCol->type, tagVal.pData, pCol->bytes, false); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 22357cab44..fd0377f998 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4030,6 +4030,19 @@ static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch pVgData->numOfTables = (int32_t)taosArrayGetSize(pTbBatch->req.pArray); taosArrayPush(pBufArray, &pVgData); +#if 1 // debug print + SDecoder decoder = {0}; + SVCreateTbBatchReq req = {0}; + // decode + tDecoderInit(&decoder, pBuf, tlen - sizeof(SMsgHead)); + if (tDecodeSVCreateTbBatchReq(&decoder, &req) < 0) { + ASSERT(0); + } + tDecoderClear(&decoder); +#endif + printf("%s:%d: OK in send \n", __func__, __LINE__); + ASSERT(0); + return TSDB_CODE_SUCCESS; } From fdf7d7785686e5ce9ecd1dc3206d281043a326c7 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 29 May 2022 20:28:54 +0800 Subject: [PATCH 10/22] fix: bug fix for tag refactor --- source/common/src/tdataformat.c | 6 ++++-- source/common/src/tmsg.c | 2 -- source/dnode/vnode/src/meta/metaTable.c | 22 ++++++++++++---------- source/libs/parser/src/parTranslater.c | 13 ------------- 4 files changed, 16 insertions(+), 27 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index dce7c1739e..8959a63860 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -638,7 +638,6 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { // type n += tPutI8(p ? p + n : p, pTagVal->type); - debugPrintTagVal(pTagVal->type, pTagVal->pData, pTagVal->nData, __func__, __LINE__); // value if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); @@ -799,7 +798,10 @@ int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag) { return tEncodeBinary(pEncoder, (const uint8_t *)pTag, pTag->len); } -int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { return tDecodeBinary(pDecoder, (uint8_t **)ppTag, NULL); } +int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag) { + uint32_t len = 0; + return tDecodeBinary(pDecoder, (uint8_t **)ppTag, &len); +} int32_t tTagToValArray(const STag *pTag, SArray **ppArray) { int32_t code = 0; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e10dfc2996..721eb9587b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3847,7 +3847,6 @@ int tEncodeSVCreateTbReq(SEncoder *pCoder, const SVCreateTbReq *pReq) { if (pReq->type == TSDB_CHILD_TABLE) { if (tEncodeI64(pCoder, pReq->ctb.suid) < 0) return -1; if (tEncodeTag(pCoder, (const STag *)pReq->ctb.pTag) < 0) return -1; - debugPrintSTag((STag*)pReq->ctb.pTag, __func__, __LINE__); } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tEncodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { @@ -3871,7 +3870,6 @@ int tDecodeSVCreateTbReq(SDecoder *pCoder, SVCreateTbReq *pReq) { if (pReq->type == TSDB_CHILD_TABLE) { if (tDecodeI64(pCoder, &pReq->ctb.suid) < 0) return -1; if (tDecodeTag(pCoder, (STag **)&pReq->ctb.pTag) < 0) return -1; - debugPrintSTag((STag*)pReq->ctb.pTag, __func__, __LINE__); } else if (pReq->type == TSDB_NORMAL_TABLE) { if (tDecodeSSchemaWrapper(pCoder, &pReq->ntb.schemaRow) < 0) return -1; } else { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index aa235c1cc9..def024667b 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -726,17 +726,17 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } -static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int8_t type, tb_uid_t uid, +static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { - int32_t nTagData = 0; + // int32_t nTagData = 0; - if (pTagData) { - if (IS_VAR_DATA_TYPE(type)) { - nTagData = varDataTLen(pTagData); - } else { - nTagData = tDataTypes[type].bytes; - } - } + // if (pTagData) { + // if (IS_VAR_DATA_TYPE(type)) { + // nTagData = varDataTLen(pTagData); + // } else { + // nTagData = tDataTypes[type].bytes; + // } + // } *nTagIdxKey = sizeof(STagIdxKey) + nTagData + sizeof(tb_uid_t); *ppTagIdxKey = (STagIdxKey *)taosMemoryMalloc(*nTagIdxKey); @@ -768,6 +768,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { int32_t nTagIdxKey; const SSchema *pTagColumn; // = &stbEntry.stbEntry.schema.pSchema[0]; const void *pTagData = NULL; // + int32_t nTagData = 0; SDecoder dc = {0}; // get super table @@ -784,6 +785,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { STagVal tagVal = {.cid = pTagColumn->colId}; tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; // update tag index #ifdef USE_INVERTED_INDEX @@ -798,7 +800,7 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { int ret = indexPut((SIndex *)pMeta->pTagIvtIdx, tmGroup, tuid); indexMultiTermDestroy(tmGroup); #else - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, pTagColumn->type, pCtbEntry->uid, + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { return -1; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fd0377f998..22357cab44 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4030,19 +4030,6 @@ static int32_t serializeVgroupCreateTableBatch(SVgroupCreateTableBatch* pTbBatch pVgData->numOfTables = (int32_t)taosArrayGetSize(pTbBatch->req.pArray); taosArrayPush(pBufArray, &pVgData); -#if 1 // debug print - SDecoder decoder = {0}; - SVCreateTbBatchReq req = {0}; - // decode - tDecoderInit(&decoder, pBuf, tlen - sizeof(SMsgHead)); - if (tDecodeSVCreateTbBatchReq(&decoder, &req) < 0) { - ASSERT(0); - } - tDecoderClear(&decoder); -#endif - printf("%s:%d: OK in send \n", __func__, __LINE__); - ASSERT(0); - return TSDB_CODE_SUCCESS; } From 5b6f9bdb9cd0f1d09b602c6357136a4dfa2cd3e6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 30 May 2022 03:51:36 +0000 Subject: [PATCH 11/22] refact: tag api --- include/common/tdataformat.h | 25 +++++-- include/util/tencode.h | 46 ++++++++++++ source/common/src/tdataformat.c | 121 +++++++++++++++++++++++++------- 3 files changed, 158 insertions(+), 34 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index eea1661ca9..957b93d359 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -128,9 +128,23 @@ struct STagVal { int16_t cid; char *pKey; }; - int8_t type; - uint32_t nData; - uint8_t *pData; + int8_t type; + union { + int8_t i8; + uint8_t u8; + int16_t i16; + uint16_t u16; + int32_t i32; + uint32_t u32; + int64_t i64; + uint64_t u64; + float f; + double d; + struct { + uint32_t nData; + uint8_t *pData; + }; + }; }; static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, @@ -218,8 +232,7 @@ struct STag { #define schemaColAt(s, i) ((s)->columns + i) #define tdFreeSchema(s) taosMemoryFreeClear((s)) - STSchema * - tdDupSchema(const STSchema *pSchema); +STSchema *tdDupSchema(const STSchema *pSchema); int32_t tdEncodeSchema(void **buf, STSchema *pSchema); void *tdDecodeSchema(void *buf, STSchema **pRSchema); @@ -403,8 +416,6 @@ SDataCols *tdFreeDataCols(SDataCols *pCols); int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToMerge, int32_t *pOffset, bool update, TDRowVerT maxVer); - - #endif #ifdef __cplusplus diff --git a/include/util/tencode.h b/include/util/tencode.h index 914091ad51..c1c5c1150d 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -530,6 +530,24 @@ static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { return sizeof(int64_t); } +static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { + union { + uint32_t ui; + float f; + } v = {.f = f}; + + return tPutU32(p, v.ui); +} + +static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { + union { + uint64_t ui; + double d; + } v = {.d = d}; + + return tPutU64(p, v.ui); +} + static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } @@ -619,6 +637,34 @@ static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { return n; } +static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) { + int32_t n = 0; + + union { + uint32_t ui; + float f; + } v; + + n = tGetU32(p, &v.ui); + + *f = v.f; + return n; +} + +static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) { + int32_t n = 0; + + union { + uint64_t ui; + double d; + } v; + + n = tGetU64(p, &v.ui); + + *d = v.d; + return n; +} + // ===================== static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { int n = 0; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 8959a63860..34e78c21db 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -574,28 +574,28 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c } } - // if (isLarge) { - // p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; - // } else { - // p = (uint8_t *)&pTag->idx[pTag->nTag]; - // } +// if (isLarge) { +// p = (uint8_t *)&((int16_t *)pTag->idx)[pTag->nTag]; +// } else { +// p = (uint8_t *)&pTag->idx[pTag->nTag]; +// } - // (*ppArray) = taosArrayInit(pTag->nTag + 1, sizeof(STagVal)); - // if (*ppArray == NULL) { - // code = TSDB_CODE_OUT_OF_MEMORY; - // goto _err; - // } +// (*ppArray) = taosArrayInit(pTag->nTag + 1, sizeof(STagVal)); +// if (*ppArray == NULL) { +// code = TSDB_CODE_OUT_OF_MEMORY; +// goto _err; +// } - // for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { - // if (isLarge) { - // offset = ((int16_t *)pTag->idx)[iTag]; - // } else { - // offset = pTag->idx[iTag]; - // } +// for (int16_t iTag = 0; iTag < pTag->nTag; iTag++) { +// if (isLarge) { +// offset = ((int16_t *)pTag->idx)[iTag]; +// } else { +// offset = pTag->idx[iTag]; +// } void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { - int8_t isJson = pTag->flags & TD_TAG_JSON; - int8_t isLarge = pTag->flags & TD_TAG_LARGE; + int8_t isJson = pTag->flags & TD_TAG_JSON; + int8_t isLarge = pTag->flags & TD_TAG_LARGE; uint8_t *p = NULL; int16_t offset = 0; @@ -642,9 +642,45 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); } else { - ASSERT(pTagVal->nData == TYPE_BYTES[pTagVal->type]); - if (p) memcpy(p + n, pTagVal->pData, pTagVal->nData); - n += pTagVal->nData; + p = p ? p + n : p; + switch (pTagVal->type) { + case TSDB_DATA_TYPE_BOOL: + n += tPutI8(p, pTagVal->i8 ? 1 : 0); + break; + case TSDB_DATA_TYPE_TINYINT: + n += tPutI8(p, pTagVal->i8); + break; + case TSDB_DATA_TYPE_SMALLINT: + n += tPutI16(p, pTagVal->i16); + break; + case TSDB_DATA_TYPE_INT: + n += tPutI32(p, pTagVal->i32); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + n += tPutI64(p, pTagVal->i64); + break; + case TSDB_DATA_TYPE_FLOAT: + n += tPutFloat(p, pTagVal->f); + break; + case TSDB_DATA_TYPE_DOUBLE: + n += tPutDouble(p, pTagVal->d); + break; + case TSDB_DATA_TYPE_UTINYINT: + n += tPutU8(p, pTagVal->u8); + break; + case TSDB_DATA_TYPE_USMALLINT: + n += tPutU16(p, pTagVal->u16); + break; + case TSDB_DATA_TYPE_UINT: + n += tPutU32(p, pTagVal->u32); + break; + case TSDB_DATA_TYPE_UBIGINT: + n += tPutU64(p, pTagVal->u64); + break; + default: + ASSERT(0); + } } return n; @@ -666,9 +702,42 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - pTagVal->pData = p + n; - pTagVal->nData = TYPE_BYTES[pTagVal->type]; - n += pTagVal->nData; + switch (pTagVal->type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + n += tGetI8(p + n, &pTagVal->i8); + break; + case TSDB_DATA_TYPE_SMALLINT: + n += tGetI16(p, &pTagVal->i16); + break; + case TSDB_DATA_TYPE_INT: + n += tGetI32(p, &pTagVal->i32); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + n += tGetI64(p, &pTagVal->i64); + break; + case TSDB_DATA_TYPE_FLOAT: + n += tGetFloat(p, &pTagVal->f); + break; + case TSDB_DATA_TYPE_DOUBLE: + n += tGetDouble(p, &pTagVal->d); + break; + case TSDB_DATA_TYPE_UTINYINT: + n += tGetU8(p, &pTagVal->u8); + break; + case TSDB_DATA_TYPE_USMALLINT: + n += tGetU16(p, &pTagVal->u16); + break; + case TSDB_DATA_TYPE_UINT: + n += tGetU32(p, &pTagVal->u32); + break; + case TSDB_DATA_TYPE_UBIGINT: + n += tGetU64(p, &pTagVal->u64); + break; + default: + ASSERT(0); + } } return n; @@ -785,9 +854,7 @@ bool tTagGet(const STag *pTag, STagVal *pTagVal) { } else if (c > 0) { lidx = midx + 1; } else { - pTagVal->type = tv.type; - pTagVal->nData = tv.nData; - pTagVal->pData = tv.pData; + memcpy(pTagVal, &tv, sizeof(tv)); return true; } } From 816cea07f99d583849a7396bbfdacd3f038b7635 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 17:49:33 +0800 Subject: [PATCH 12/22] feat:add new logic for new tag format --- include/common/tdataformat.h | 44 +-- include/common/tmsg.h | 2 +- source/client/src/clientImpl.c | 51 +-- source/common/src/tdatablock.c | 21 +- source/common/src/tdataformat.c | 115 ++----- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 16 +- source/dnode/vnode/src/meta/metaTable.c | 21 +- source/libs/executor/src/scanoperator.c | 55 ++- source/libs/parser/inc/parUtil.h | 4 +- source/libs/parser/src/parInsert.c | 430 ++++++++++++++++++------ source/libs/parser/src/parTranslater.c | 255 +++++++------- source/libs/parser/src/parUtil.c | 74 ++-- source/libs/scalar/src/sclvector.c | 39 +-- 14 files changed, 587 insertions(+), 542 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 957b93d359..38bbee1f1d 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -59,14 +59,11 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder); int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, uint8_t *pData, uint32_t nData); int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow); -// STagVal -static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, - bool isJson); - // STag int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); bool tTagGet(const STag *pTag, STagVal *pTagVal); +char* tTagValToData(const STagVal *pTagVal, bool isJson); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); int32_t tTagToValArray(const STag *pTag, SArray **ppArray); @@ -130,16 +127,7 @@ struct STagVal { }; int8_t type; union { - int8_t i8; - uint8_t u8; - int16_t i16; - uint16_t u16; - int32_t i32; - uint32_t u32; int64_t i64; - uint64_t u64; - float f; - double d; struct { uint32_t nData; uint8_t *pData; @@ -147,24 +135,9 @@ struct STagVal { }; }; -static FORCE_INLINE void tTagValPush(SArray *pTagArray, void *key, int8_t type, uint8_t *pData, uint32_t nData, - bool isJson) { - STagVal tagVal = {0}; - if (isJson) { - tagVal.pKey = (char *)key; - } else { - tagVal.cid = *(int16_t *)key; - } - - tagVal.type = type; - tagVal.pData = pData; - tagVal.nData = nData; - taosArrayPush(pTagArray, &tagVal); -} - #pragma pack(push, 1) -#define TD_TAG_JSON ((int8_t)0x1) -#define TD_TAG_LARGE ((int8_t)0x2) +#define TD_TAG_JSON ((int8_t)0x80) // distinguish JSON string and JSON value with the highest bit +#define TD_TAG_LARGE ((int8_t)0x40) struct STag { int8_t flags; int16_t len; @@ -424,14 +397,3 @@ int32_t tdMergeDataCols(SDataCols *target, SDataCols *source, int32_t rowsToM #endif /*_TD_COMMON_DATA_FORMAT_H_*/ -// SKVRowBuilder; - -// int32_t tdInitKVRowBuilder(SKVRowBuilder *pBuilder); -// void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder); -// void tdResetKVRowBuilder(SKVRowBuilder *pBuilder); -// SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder); - -// static FORCE_INLINE int32_t tdAddColToKVRow(SKVRowBuilder *pBuilder, col_id_t colId, const void *value, int32_t tlen) - -// #ifdef JSON_TAG_REFACTOR -// TODO: JSON_TAG_TODO diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 37e4cc5e8c..4d69cf5a32 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -287,7 +287,7 @@ typedef struct SSchema { char name[TSDB_COL_NAME_LEN]; } SSchema; -#define COL_IS_SET(FLG) ((FLG) & (COL_SET_VAL | COL_SET_NULL) != 0) +#define COL_IS_SET(FLG) (((FLG) & (COL_SET_VAL | COL_SET_NULL)) != 0) #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index fc899b9b83..db5e6ab4c0 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -958,30 +958,11 @@ static char* parseTagDatatoJson(void* p) { char tagJsonKey[256] = {0}; for (int j = 0; j < nCols; ++j) { STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); - - // JSON_TAG_REFACTOR - pTagVal->nData; - pTagVal->pData; // for varChar, len not included - // TODO: adapt below code; - char* val = (char*)pTagVal->pData; - // TODO: adapt below code; - - if (j == 0) { - if (*val == TSDB_DATA_TYPE_NULL) { - string = taosMemoryCalloc(1, 8); - sprintf(string, "%s", TSDB_DATA_NULL_STR_L); - goto end; - } - continue; - } - // json key encode by binary memset(tagJsonKey, 0, sizeof(tagJsonKey)); - memcpy(tagJsonKey, varDataVal(val), varDataLen(val)); + memcpy(tagJsonKey, pTagVal->pKey, strlen(pTagVal->pKey)); // json value - val += varDataTLen(val); - char* realData = POINTER_SHIFT(val, CHAR_BYTES); - char type = *val; + char type = pTagVal->type; if (type == TSDB_DATA_TYPE_NULL) { cJSON* value = cJSON_CreateNull(); if (value == NULL) { @@ -990,11 +971,11 @@ static char* parseTagDatatoJson(void* p) { cJSON_AddItemToObject(json, tagJsonKey, value); } else if (type == TSDB_DATA_TYPE_NCHAR) { cJSON* value = NULL; - if (varDataLen(realData) > 0) { - char* tagJsonValue = taosMemoryCalloc(varDataLen(realData), 1); - int32_t length = taosUcs4ToMbs((TdUcs4*)varDataVal(realData), varDataLen(realData), tagJsonValue); + if (pTagVal->nData > 0) { + char* tagJsonValue = taosMemoryCalloc(pTagVal->nData, 1); + int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); if (length < 0) { - tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, val); + tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, pTagVal->pData); taosMemoryFree(tagJsonValue); goto end; } @@ -1003,7 +984,7 @@ static char* parseTagDatatoJson(void* p) { if (value == NULL) { goto end; } - } else if (varDataLen(realData) == 0) { + } else if (pTagVal->nData == 0) { value = cJSON_CreateString(""); } else { ASSERT(0); @@ -1011,22 +992,14 @@ static char* parseTagDatatoJson(void* p) { cJSON_AddItemToObject(json, tagJsonKey, value); } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double jsonVd = *(double*)(realData); + double jsonVd = *(double*)(&pTagVal->i64); cJSON* value = cJSON_CreateNumber(jsonVd); if (value == NULL) { goto end; } cJSON_AddItemToObject(json, tagJsonKey, value); - // }else if(type == TSDB_DATA_TYPE_BIGINT){ - // int64_t jsonVd = *(int64_t*)(realData); - // cJSON* value = cJSON_CreateNumber((double)jsonVd); - // if (value == NULL) - // { - // goto end; - // } - // cJSON_AddItemToObject(json, tagJsonKey, value); } else if (type == TSDB_DATA_TYPE_BOOL) { - char jsonVd = *(char*)(realData); + char jsonVd = *(char*)(&pTagVal->i64); cJSON* value = cJSON_CreateBool(jsonVd); if (value == NULL) { goto end; @@ -1091,7 +1064,7 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int if (jsonInnerType == TSDB_DATA_TYPE_NULL) { sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_JSON) { + } else if (jsonInnerType == TD_TAG_JSON) { char* jsonString = parseTagDatatoJson(jsonInnerData); STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); @@ -1110,10 +1083,6 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int double jsonVd = *(double*)(jsonInnerData); sprintf(varDataVal(dst), "%.9lf", jsonVd); varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType == TSDB_DATA_TYPE_BIGINT) { - int64_t jsonVd = *(int64_t*)(jsonInnerData); - sprintf(varDataVal(dst), "%" PRId64, jsonVd); - varDataSetLen(dst, strlen(varDataVal(dst))); } else if (jsonInnerType == TSDB_DATA_TYPE_BOOL) { sprintf(varDataVal(dst), "%s", (*((char*)jsonInnerData) == 1) ? "true" : "false"); varDataSetLen(dst, strlen(varDataVal(dst))); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1918061837..e6d532f2dc 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -116,22 +116,23 @@ int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, con int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { - int32_t dataLen = varDataTLen(pData); + int32_t dataLen = 0; if (type == TSDB_DATA_TYPE_JSON) { if (*pData == TSDB_DATA_TYPE_NULL) { - dataLen = 0; - } else if (*pData == TSDB_DATA_TYPE_NCHAR) { - dataLen = varDataTLen(pData + CHAR_BYTES); - } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { - dataLen = DOUBLE_BYTES; - } else if (*pData == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES; - } else if (*pData == TSDB_DATA_TYPE_JSON) { - dataLen = ((STag*)(pData + CHAR_BYTES))->len; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES) + CHAR_BYTES; + } else if (*pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = DOUBLE_BYTES + CHAR_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES + CHAR_BYTES; + } else if (*pData == TD_TAG_JSON) { // json string + dataLen = ((STag*)(pData))->len; } else { ASSERT(0); } - dataLen += CHAR_BYTES; + }else { + dataLen = varDataTLen(pData); } SVarColAttr* pAttr = &pColumnInfoData->varmeta; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 34e78c21db..722fa57bad 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -123,7 +123,7 @@ int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t iCol, SColVal ASSERT(iCol != 0); ASSERT(pTColumn->colId != 0); - ASSERT(pRow->flags & 0xf != 0); + ASSERT((pRow->flags & 0xf) != 0); switch (pRow->flags & 0xf) { case TSROW_HAS_NONE: *pColVal = ColValNONE; @@ -432,7 +432,6 @@ static void setBitMap(uint8_t *p, STSchema *pTSchema, uint8_t flags) { } int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { int32_t nDataTP, nDataKV; - uint32_t flags; STSKVRow *pTSKVRow = (STSKVRow *)pBuilder->pKVBuf; int32_t nCols = pBuilder->pTSchema->numOfCols; @@ -446,7 +445,7 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { pBuilder->row.flags |= TSROW_HAS_NONE; } - ASSERT(pBuilder->row.flags & 0xf != 0); + ASSERT((pBuilder->row.flags & 0xf) != 0); *(ppRow) = &pBuilder->row; switch (pBuilder->row.flags & 0xf) { case TSROW_HAS_NONE: @@ -476,7 +475,7 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { if (nDataKV < nDataTP) { // generate KV row - ASSERT(pBuilder->row.flags & 0xf != TSROW_HAS_VAL); + ASSERT((pBuilder->row.flags & 0xf) != TSROW_HAS_VAL); pBuilder->row.flags |= TSROW_KV_ROW; pBuilder->row.nData = nDataKV; @@ -492,12 +491,12 @@ int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) { pBuilder->row.nData = nDataTP; uint8_t *p; - uint8_t flags = pBuilder->row.flags & 0xf; + uint8_t flags = (pBuilder->row.flags & 0xf); if (flags == TSROW_HAS_VAL) { pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2; } else { - if (flags == TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE) { + if (flags == (TSROW_HAS_VAL | TSROW_HAS_NULL | TSROW_HAS_NONE)) { pBuilder->row.pData = pBuilder->pTPBuf; } else { pBuilder->row.pData = pBuilder->pTPBuf + pBuilder->szBitMap2 - pBuilder->szBitMap1; @@ -620,7 +619,11 @@ void debugPrintSTag(STag *pTag, const char *tag, int32_t ln) { } printf("%s:%d loop[%d-%d] offset=%d\n", __func__, __LINE__, (int32_t)pTag->nTag, (int32_t)n, (int32_t)offset); tGetTagVal(p + offset, &tagVal, isJson); - debugPrintTagVal(tagVal.type, tagVal.pData, tagVal.nData, __func__, __LINE__); + if(IS_VAR_DATA_TYPE(tagVal.type)){ + debugPrintTagVal(tagVal.type, tagVal.pData, tagVal.nData, __func__, __LINE__); + }else{ + debugPrintTagVal(tagVal.type, &tagVal.i64, tDataTypes[tagVal.type].bytes, __func__, __LINE__); + } } printf("\n"); } @@ -643,44 +646,8 @@ static int32_t tPutTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { n += tPutBinary(p ? p + n : p, pTagVal->pData, pTagVal->nData); } else { p = p ? p + n : p; - switch (pTagVal->type) { - case TSDB_DATA_TYPE_BOOL: - n += tPutI8(p, pTagVal->i8 ? 1 : 0); - break; - case TSDB_DATA_TYPE_TINYINT: - n += tPutI8(p, pTagVal->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tPutI16(p, pTagVal->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tPutI32(p, pTagVal->i32); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - n += tPutI64(p, pTagVal->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tPutFloat(p, pTagVal->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tPutDouble(p, pTagVal->d); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tPutU8(p, pTagVal->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tPutU16(p, pTagVal->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tPutU32(p, pTagVal->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tPutU64(p, pTagVal->u64); - break; - default: - ASSERT(0); - } + n += tDataTypes[pTagVal->type].bytes; + if(p) memcpy(p, &(pTagVal->i64), tDataTypes[pTagVal->type].bytes); } return n; @@ -702,42 +669,8 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - switch (pTagVal->type) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - n += tGetI8(p + n, &pTagVal->i8); - break; - case TSDB_DATA_TYPE_SMALLINT: - n += tGetI16(p, &pTagVal->i16); - break; - case TSDB_DATA_TYPE_INT: - n += tGetI32(p, &pTagVal->i32); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - n += tGetI64(p, &pTagVal->i64); - break; - case TSDB_DATA_TYPE_FLOAT: - n += tGetFloat(p, &pTagVal->f); - break; - case TSDB_DATA_TYPE_DOUBLE: - n += tGetDouble(p, &pTagVal->d); - break; - case TSDB_DATA_TYPE_UTINYINT: - n += tGetU8(p, &pTagVal->u8); - break; - case TSDB_DATA_TYPE_USMALLINT: - n += tGetU16(p, &pTagVal->u16); - break; - case TSDB_DATA_TYPE_UINT: - n += tGetU32(p, &pTagVal->u32); - break; - case TSDB_DATA_TYPE_UBIGINT: - n += tGetU64(p, &pTagVal->u64); - break; - default: - ASSERT(0); - } + n += tDataTypes[pTagVal->type].bytes; + memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); } return n; @@ -814,6 +747,26 @@ void tTagFree(STag *pTag) { if (pTag) taosMemoryFree(pTag); } +char *tTagValToData(const STagVal *value, bool isJson){ + if(!value) return NULL; + char *data = NULL; + int8_t typeBytes = 0; + if (isJson) { + typeBytes = CHAR_BYTES; + } + if(IS_VAR_DATA_TYPE(value->type)){ + data = taosMemoryCalloc(1, typeBytes + VARSTR_HEADER_SIZE + value->nData); + if(data == NULL) return NULL; + if(isJson) *data = value->type; + varDataLen(data + typeBytes) = value->nData; + memcpy(varDataVal(data + typeBytes), value->pData, value->nData); + }else{ + data = ((char*)&(value->i64)) - typeBytes; // json with type + } + + return data; +} + bool tTagGet(const STag *pTag, STagVal *pTagVal) { int16_t lidx = 0; int16_t ridx = pTag->nTag - 1; diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 2b713ff980..df5785e20a 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -78,7 +78,7 @@ void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaReadNext(SMetaReader *pReader); -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid); +const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal); #if 1 // refact APIs below (TODO) typedef SVCreateTbReq STbCfg; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index b59b6a6fd0..dead85f854 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -564,9 +564,17 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid) { +const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); - STagVal tagVal = {.cid = cid}; - tTagGet((const STag *)pEntry->ctbEntry.pTags, &tagVal); - return tagVal.pData; + STag *tag = (STag *)pEntry->ctbEntry.pTags; + tTagGet(tag, val); + + if(val->type == TSDB_DATA_TYPE_NULL){ + return NULL; + } + if (type == TSDB_DATA_TYPE_JSON){ + return tag; + }else{ + return val; + } } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index def024667b..984688fc19 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -573,15 +573,20 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA for (int32_t i = 0; i < pTagSchema->nCols; i++) { SSchema *pCol = &pTagSchema->pSchema[i]; if (iCol == i) { - tTagValPush(pTagArray, &pCol->colId, pCol->type, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal, false); + STagVal val = {0}; + val.type = pCol->type; + val.cid = pCol->colId; + if (IS_VAR_DATA_TYPE(pCol->type)) { + val.pData = pAlterTbReq->pTagVal; + val.nData = pAlterTbReq->nTagVal; + }else{ + memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal); + } + taosArrayPush(pTagArray, &val); } else { - STagVal tagVal = {.cid = pCol->colId}; - if (tTagGet(pOldTag, &tagVal) && tagVal.pData) { - if (IS_VAR_DATA_TYPE(pCol->type)) { - tTagValPush(pTagArray, &pCol->colId, pCol->type, varDataVal(tagVal.pData), varDataLen(tagVal.pData), false); - } else { - tTagValPush(pTagArray, &pCol->colId, pCol->type, tagVal.pData, pCol->bytes, false); - } + STagVal val = {0}; + if (tTagGet(pOldTag, &val)) { + taosArrayPush(pTagArray, &val); } } } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 4a275660bc..973e52d1a3 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -303,29 +303,22 @@ void addTagPseudoColumnData(STableScanInfo* pTableScanInfo, SSDataBlock* pBlock) if (fmIsScanPseudoColumnFunc(functionId)) { setTbNameColData(pTableScanInfo->readHandle.meta, pBlock, pColInfoData, functionId); } else { // these are tags - const char* p = NULL; - if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - const STag* tmp = (const STag*)mr.me.ctbEntry.pTags; + STagVal tagVal = {0}; + tagVal.cid = pExpr->base.pParam[0].pCol->colId; + const char *p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); - char* data = taosMemoryCalloc(tmp->len + 1, 1); - if (data == NULL) { - metaReaderClear(&mr); - qError("doTagScan calloc error:%d", tmp->len + 1); - return; - } - - *data = TSDB_DATA_TYPE_JSON; - memcpy(data + 1, tmp, tmp->len); - p = data; - } else { - p = metaGetTableTagVal(&mr.me, pExpr->base.pParam[0].pCol->colId); + char *data = NULL; + if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL){ + data = tTagValToData((const STagVal *)p, false); + }else { + data = (char*)p; } for (int32_t i = 0; i < pBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i, p, (p == NULL)); + colDataAppend(pColInfoData, i, data, (data == NULL)); } - if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - taosMemoryFree((void*)p); + if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ + taosMemoryFree(data); } } } @@ -1632,22 +1625,20 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { STR_TO_VARSTR(str, mr.me.name); colDataAppend(pDst, count, str, false); } else { // it is a tag value - if (pDst->info.type == TSDB_DATA_TYPE_JSON) { - const STag* tmp = (const STag*)mr.me.ctbEntry.pTags; - // TODO opt perf by realloc memory - char* data = taosMemoryCalloc(tmp->len + 1, 1); - if (data == NULL) { - qError("%s failed to malloc memory, size:%d", GET_TASKID(pTaskInfo), tmp->len + 1); - longjmp(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); - } + STagVal val = {0}; + val.cid = pExprInfo[j].base.pParam[0].pCol->colId; + const char* p = metaGetTableTagVal(&mr.me, pDst->info.type, &val); - *data = TSDB_DATA_TYPE_JSON; - memcpy(data + 1, tmp, tmp->len); - colDataAppend(pDst, count, data, false); + char *data = NULL; + if(pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL){ + data = tTagValToData((const STagVal *)p, false); + }else { + data = (char*)p; + } + colDataAppend(pDst, count, data, (data == NULL)); + + if(pDst->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ taosMemoryFree(data); - } else { - const char* p = metaGetTableTagVal(&mr.me, pExprInfo[j].base.pParam[0].pCol->colId); - colDataAppend(pDst, count, p, (p == NULL)); } } } diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 5098ea9232..35cab7db36 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -55,9 +55,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); -#ifdef JSON_TAG_REFACTOR -int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId); -#endif +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* errMsg); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 37ee6df682..d670f5ad38 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -54,7 +54,6 @@ typedef struct SInsertParseContext { SMsgBuf msg; // input STableMeta* pTableMeta; // each table SParsedDataColInfo tags; // each table - SArray* pTagVals; // each table SVCreateTbReq createTblReq; // each table SHashObj* pVgroupsHashObj; // global SHashObj* pTableBlockHashObj; // global @@ -425,7 +424,7 @@ static int parseTime(char** end, SToken* pToken, int16_t timePrec, int64_t* time return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, uint32_t type, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { +static FORCE_INLINE int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMsgBuf) { if ((pToken->type != TK_NOW && pToken->type != TK_TODAY && pToken->type != TK_NK_INTEGER && pToken->type != TK_NK_STRING && pToken->type != TK_NK_FLOAT && pToken->type != TK_NK_BOOL && pToken->type != TK_NULL && pToken->type != TK_NK_HEX && pToken->type != TK_NK_OCT && @@ -471,7 +470,7 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int uint64_t uv; char* endptr = NULL; - int32_t code = checkAndTrimValue(pToken, pSchema->type, tmpTokenBuf, pMsgBuf); + int32_t code = checkAndTrimValue(pToken, tmpTokenBuf, pMsgBuf); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -620,14 +619,12 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int case TSDB_DATA_TYPE_NCHAR: { return func(pMsgBuf, pToken->z, pToken->n, param); } -#ifdef JSON_TAG_REFACTOR case TSDB_DATA_TYPE_JSON: { if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); } return func(pMsgBuf, pToken->z, pToken->n, param); } -#endif case TSDB_DATA_TYPE_TIMESTAMP: { int64_t tmpVal; if (parseTime(end, pToken, timePrec, &tmpVal, pMsgBuf) != TSDB_CODE_SUCCESS) { @@ -752,108 +749,287 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* return TSDB_CODE_SUCCESS; } -static int32_t KvRowAppend(SMsgBuf* pMsgBuf, const void* value, int32_t len, void* param) { - SKvParam* pa = (SKvParam*)param; - - int8_t type = pa->schema->type; - int16_t colId = pa->schema->colId; - -#ifdef JSON_TAG_REFACTOR - if (TSDB_DATA_TYPE_JSON == type) { - return parseJsontoTagData(value, pa->builder, pMsgBuf, colId); - } -#endif - - if (value == NULL) { // it is a null data - // tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NULL, value, false, pa->toffset, - // pa->colIdx); - return TSDB_CODE_SUCCESS; - } - - if (TSDB_DATA_TYPE_BINARY == type) { - memcpy(pa->buf + pa->pos, value, len); - tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), len, false); - pa->pos += len; - } else if (TSDB_DATA_TYPE_NCHAR == type) { - // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long' - - ASSERT((pa->pos + pa->schema->bytes - VARSTR_HEADER_SIZE) <= TSDB_MAX_TAGS_LEN); - - int32_t output = 0; - if (!taosMbsToUcs4(value, len, (TdUcs4*)(pa->buf + pa->pos), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pa->schema->name); - } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); - return buildSyntaxErrMsg(pMsgBuf, buf, value); - } - tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), output, false); - pa->pos += output; - } else { - memcpy(pa->buf + pa->pos, value, TYPE_BYTES[type]); - tTagValPush(pa->pTagVals, &colId, type, (uint8_t*)(pa->buf + pa->pos), TYPE_BYTES[type], false); - pa->pos + TYPE_BYTES[type]; - } - ASSERT(pa->pos <= TSDB_MAX_TAGS_LEN); - - return TSDB_CODE_SUCCESS; -} - -static int32_t buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid) { +static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTag, int64_t suid) { pTbReq->type = TD_CHILD_TABLE; pTbReq->name = strdup(tname); pTbReq->ctb.suid = suid; pTbReq->ctb.pTag = (uint8_t*)pTag; - return TSDB_CODE_SUCCESS; + return; +} + +static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, + int16_t timePrec, char* tmpTokenBuf, STagVal *val, SMsgBuf* pMsgBuf) { + int64_t iv; + uint64_t uv; + char* endptr = NULL; + + if (isNullStr(pToken)) { + if (TSDB_DATA_TYPE_TIMESTAMP == pSchema->type && PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) { + return buildSyntaxErrMsg(pMsgBuf, "primary timestamp should not be null", pToken->z); + } + + return TSDB_CODE_SUCCESS; + } + + val->cid = pSchema->colId; + val->type = pSchema->bytes; + + switch (pSchema->type) { + case TSDB_DATA_TYPE_BOOL: { + if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { + if (strncmp(pToken->z, "true", pToken->n) == 0) { + *(int8_t*)(&val->i64) = TRUE_VALUE; + } else if (strncmp(pToken->z, "false", pToken->n) == 0) { + *(int8_t*)(&val->i64) = FALSE_VALUE; + } else { + return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); + } + } else if (pToken->type == TK_NK_INTEGER) { + *(int8_t*)(&val->i64) = ((taosStr2Int64(pToken->z, NULL, 10) == 0) ? FALSE_VALUE : TRUE_VALUE); + } else if (pToken->type == TK_NK_FLOAT) { + *(int8_t*)(&val->i64) = ((taosStr2Double(pToken->z, NULL) == 0) ? FALSE_VALUE : TRUE_VALUE); + } else { + return buildSyntaxErrMsg(pMsgBuf, "invalid bool data", pToken->z); + } + break; + } + + case TSDB_DATA_TYPE_TINYINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid tinyint data", pToken->z); + } else if (!IS_VALID_TINYINT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "tinyint data overflow", pToken->z); + } + + *(int8_t*)(&val->i64) = iv; + break; + } + + case TSDB_DATA_TYPE_UTINYINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned tinyint data", pToken->z); + } else if (!IS_VALID_UTINYINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned tinyint data overflow", pToken->z); + } + *(uint8_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_SMALLINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid smallint data", pToken->z); + } else if (!IS_VALID_SMALLINT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "smallint data overflow", pToken->z); + } + *(int16_t*)(&val->i64) = iv; + break; + } + + case TSDB_DATA_TYPE_USMALLINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned smallint data", pToken->z); + } else if (!IS_VALID_USMALLINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned smallint data overflow", pToken->z); + } + *(uint16_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_INT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid int data", pToken->z); + } else if (!IS_VALID_INT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "int data overflow", pToken->z); + } + *(int32_t*)(&val->i64) = iv; + break; + } + + case TSDB_DATA_TYPE_UINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned int data", pToken->z); + } else if (!IS_VALID_UINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned int data overflow", pToken->z); + } + *(uint32_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_BIGINT: { + if (TSDB_CODE_SUCCESS != toInteger(pToken->z, pToken->n, 10, &iv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid bigint data", pToken->z); + } else if (!IS_VALID_BIGINT(iv)) { + return buildSyntaxErrMsg(pMsgBuf, "bigint data overflow", pToken->z); + } + + val->i64 = iv; + break; + } + + case TSDB_DATA_TYPE_UBIGINT: { + if (TSDB_CODE_SUCCESS != toUInteger(pToken->z, pToken->n, 10, &uv)) { + return buildSyntaxErrMsg(pMsgBuf, "invalid unsigned bigint data", pToken->z); + } else if (!IS_VALID_UBIGINT(uv)) { + return buildSyntaxErrMsg(pMsgBuf, "unsigned bigint data overflow", pToken->z); + } + *(uint64_t*)(&val->i64) = uv; + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + double dv; + if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || dv > FLT_MAX || dv < -FLT_MAX || isinf(dv) || + isnan(dv)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal float data", pToken->z); + } + *(float*)(&val->i64) = dv; + break; + } + + case TSDB_DATA_TYPE_DOUBLE: { + double dv; + if (TK_NK_ILLEGAL == toDouble(pToken, &dv, &endptr)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + } + if (((dv == HUGE_VAL || dv == -HUGE_VAL) && errno == ERANGE) || isinf(dv) || isnan(dv)) { + return buildSyntaxErrMsg(pMsgBuf, "illegal double data", pToken->z); + } + + *(double*)(&val->i64) = dv; + break; + } + + case TSDB_DATA_TYPE_BINARY: { + // Too long values will raise the invalid sql error message + if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) { + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + val->pData = pToken->z; + val->nData = pToken->n; + break; + } + + case TSDB_DATA_TYPE_NCHAR: { + int32_t output = 0; + void *p = taosMemoryCalloc(1, pToken->n * TSDB_NCHAR_SIZE); + if(p == NULL){ + return TSDB_CODE_OUT_OF_MEMORY; + } + if (!taosMbsToUcs4(pToken->z, pToken->n, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + return generateSyntaxErrMsg(pMsgBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + return buildSyntaxErrMsg(pMsgBuf, buf, pToken->z); + } + val->pData = p; + val->nData = output; + break; + } + case TSDB_DATA_TYPE_JSON: { + if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); + } + //return func(pMsgBuf, pToken->z, pToken->n, param); + } + case TSDB_DATA_TYPE_TIMESTAMP: { + if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) { + return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); + } + + val->i64 = iv; + break; + } + } + + return TSDB_CODE_FAILED; } // pSql -> tag1_value, ...) static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint8_t precision, const char* tName) { - ASSERT(!pCxt->pTagVals); - if (!(pCxt->pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)))) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - - SKvParam param = {.pTagVals = pCxt->pTagVals, .pos = 0}; + int32_t code = TSDB_CODE_SUCCESS; + SArray *pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); SToken sToken; bool isParseBindParam = false; - char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // used for deleting Escape character: \\, \', \" - // TODO: JSON_TAG_REFACTOR => here would have json tag? for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); if (sToken.type == TK_NK_QUESTION) { isParseBindParam = true; if (NULL == pCxt->pStmtCb) { - return buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + code = buildSyntaxErrMsg(&pCxt->msg, "? only used in stmt", sToken.z); + goto end; } continue; } if (isParseBindParam) { - return buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); + code = buildInvalidOperationMsg(&pCxt->msg, "no mix usage for ? and tag values"); + goto end; } SSchema* pTagSchema = &pSchema[pCxt->tags.boundColumns[i]]; - param.schema = pTagSchema; - CHECK_CODE( - parseValueToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, KvRowAppend, ¶m, &pCxt->msg)); + char *tmpTokenBuf = taosMemoryCalloc(1, sToken.n); // this can be optimize with parse column + code = checkAndTrimValue(&sToken, tmpTokenBuf, &pCxt->msg); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(tmpTokenBuf); + goto end; + } + if(pTagSchema->type == TSDB_DATA_TYPE_JSON){ + if (sToken.n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msg, "json string too long than 4095", sToken.z); + taosMemoryFree(tmpTokenBuf); + goto end; + } + code = parseJsontoTagData(sToken.z, pTagVals, &pCxt->msg); + taosMemoryFree(tmpTokenBuf); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else{ + STagVal val = {0}; + code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg); + if (TSDB_CODE_SUCCESS != code) { + taosMemoryFree(tmpTokenBuf); + goto end; + } + if (pTagSchema->type != TSDB_DATA_TYPE_BINARY){ + taosMemoryFree(tmpTokenBuf); + } + taosArrayPush(pTagVals, &val); + } } if (isParseBindParam) { - return TSDB_CODE_SUCCESS; + code = TSDB_CODE_SUCCESS; + goto end; } - // TODO: JSON_TAG_REFACTOR (would be JSON tag or normal tag) STag* pTag = NULL; - if (tTagNew(param.pTagVals, 1, false, &pTag) != 0) { - return buildInvalidOperationMsg(&pCxt->msg, "out of memory"); + code = tTagNew(pTagVals, 1, false, &pTag); + if (code != TSDB_CODE_SUCCESS) { + goto end; } - return buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid); + buildCreateTbReq(&pCxt->createTblReq, tName, pTag, pCxt->pTableMeta->suid); + +end: + for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } + taosArrayDestroy(pTagVals); + return code; } static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { @@ -1072,7 +1248,6 @@ void destroyCreateSubTbReq(SVCreateTbReq* pReq) { static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); - taosArrayDestroy(pCxt->pTagVals); destroyCreateSubTbReq(&pCxt->createTblReq); } @@ -1348,41 +1523,75 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN return buildInvalidOperationMsg(&pBuf, "out of memory"); } + int32_t code = TSDB_CODE_SUCCESS; SSchema* pSchema = pDataBlock->pTableMeta->schema; - SKvParam param = {.pTagVals = pTagArray, .pos = 0}; for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { - KvRowAppend(&pBuf, NULL, 0, ¶m); continue; } SSchema* pTagSchema = &pSchema[tags->boundColumns[c]]; - param.schema = pTagSchema; int32_t colLen = pTagSchema->bytes; if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; } - CHECK_CODE(KvRowAppend(&pBuf, (char*)bind[c].buffer, colLen, ¶m)); + STagVal val = {0}; + if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ + val.pData = (uint8_t*)bind[c].buffer; + val.nData = colLen; + }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ + int32_t output = 0; + void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE); + if(p == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + goto end; + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); + goto end; + } + val.pData = p; + val.nData = output; + }else{ + memcpy(&val.i64, bind[c].buffer, colLen); + } + taosArrayPush(pTagArray, &val); } STag* pTag = NULL; - // TODO: JSON_TAG_REFACTOR (if is json or not)? + // TODO: stmt support json if (0 != tTagNew(pTagArray, 1, false, &pTag)) { - return buildInvalidOperationMsg(&pBuf, "out of memory"); + code = buildInvalidOperationMsg(&pBuf, "out of memory"); + goto end; } SVCreateTbReq tbReq = {0}; - CHECK_CODE(buildCreateTbReq(&tbReq, tName, pTag, suid)); - CHECK_CODE(buildCreateTbMsg(pDataBlock, &tbReq)); - + buildCreateTbReq(&tbReq, tName, pTag, suid); + code = buildCreateTbMsg(pDataBlock, &tbReq); destroyCreateSubTbReq(&tbReq); + +end: + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(p->type == TSDB_DATA_TYPE_NCHAR){ + taosMemoryFree(p->pData); + } + } taosArrayDestroy(pTagArray); - return TSDB_CODE_SUCCESS; + return code; } int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { @@ -1714,25 +1923,52 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p return TSDB_CODE_TSC_OUT_OF_MEMORY; } - SKvParam param = {.pTagVals = pTagArray, .pos = 0}; + int32_t code = TSDB_CODE_SUCCESS; for (int i = 0; i < tags->numOfBound; ++i) { SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; - param.schema = pTagSchema; SSmlKv* kv = taosArrayGetP(cols, i); - if (IS_VAR_DATA_TYPE(kv->type)) { - KvRowAppend(msg, kv->value, kv->length, ¶m); - } else { - KvRowAppend(msg, &(kv->value), kv->length, ¶m); + + STagVal val = {0}; + if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ + val.pData = (uint8_t *)kv->value; + val.nData = kv->length; + }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ + int32_t output = 0; + void *p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); + if(p == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + goto end; + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + code = buildSyntaxErrMsg(msg, buf, kv->value); + goto end; + } + val.pData = p; + val.nData = output; + }else{ + memcpy(&val.i64, &(kv->value), kv->length); + } + taosArrayPush(pTagArray, &val); + } + + code = tTagNew(pTagArray, 1, false, ppTag); +end: + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(p->type == TSDB_DATA_TYPE_NCHAR){ + taosMemoryFree(p->pData); } } - - if (tTagNew(pTagArray, 1, false, ppTag) != 0) { - taosArrayDestroy(pTagArray); - return TSDB_CODE_OUT_OF_MEMORY; - } - taosArrayDestroy(pTagArray); - return TSDB_CODE_SUCCESS; + return code; } int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols, bool format, STableMeta* pTableMeta, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 22357cab44..96ee694e1f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4144,26 +4144,6 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, S } } -// static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, -// SKVRowBuilder* pBuilder) { -#ifdef JSON_TAG_REFACTOR -if (pSchema->type == TSDB_DATA_TYPE_JSON) { - if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); - } - - return parseJsontoTagData(pVal->literal, pBuilder, &pCxt->msgBuf, pSchema->colId); -} -#endif - -// if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { -// tdAddColToKVRow(pBuilder, pSchema->colId, nodesGetValueFromNode(pVal), -// IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); -// } - -// return TSDB_CODE_SUCCESS; -// } - static int32_t createValueFromFunction(STranslateContext* pCxt, SFunctionNode* pFunc, SValueNode** pVal) { int32_t code = getFuncInfo(pCxt, pFunc); if (TSDB_CODE_SUCCESS == code) { @@ -4199,16 +4179,14 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); - char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); - if (!pTagArray || !pTagBuf) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); + if (!pTagArray) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); } - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; int16_t nTags = 0, nBufPos = 0; SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); - SNode * pTag, *pNode; + SNode * pTag = NULL, *pNode = NULL; + bool isJson = false; FORBOTH(pTag, pStmt->pSpecificTags, pNode, pStmt->pValsOfTags) { SColumnNode* pCol = (SColumnNode*)pTag; SSchema* pSchema = NULL; @@ -4219,57 +4197,56 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } if (NULL == pSchema) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); + code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_TAG_NAME, pCol->colName); + goto end; } SValueNode* pVal = NULL; code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pSchema, pNode, &pVal); - if (TSDB_CODE_SUCCESS == code) { - if (NULL == pVal) { - pVal = (SValueNode*)pNode; - } else { - REPLACE_LIST2_NODE(pVal); - } - } -#ifdef JSON_TAG_REFACTOR - if (TSDB_CODE_SUCCESS == code) { - code = addValToKVRow(pCxt, pVal, pSchema, pBuilder); - } -#endif - - if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - // TODO: JSON_TAG_TODO: is copy is a must? - void* nodeVal = nodesGetValueFromNode(pVal); - if (IS_VAR_DATA_TYPE(pSchema->type)) { - memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), varDataLen(nodeVal)); - tTagValPush(pTagArray, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, varDataLen(nodeVal), false); - nBufPos += varDataLen(pVal->datum.p); - } else { - memcpy(pTagBuf + nBufPos, varDataVal(nodeVal), TYPE_BYTES[pSchema->type]); - tTagValPush(pTagArray, &pSchema->colId, pSchema->type, (uint8_t*)pTagBuf + nBufPos, TYPE_BYTES[pSchema->type], - false); - nBufPos += TYPE_BYTES[pSchema->type]; - } - } - if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); - return code; + goto end; + } + + if (NULL == pVal) { + pVal = (SValueNode*)pNode; + } else { + REPLACE_LIST2_NODE(pVal); + } + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); + goto end; + } + + isJson = true; + code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + void* nodeVal = nodesGetValueFromNode(pVal); + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + val.pData = varDataVal(nodeVal); + val.nData = varDataLen(nodeVal); + } else { + memcpy(&val.i64, nodeVal, pTagSchema->bytes); + } + taosArrayPush(pTagArray, &val); } } - // TODO: JSON_TAG_TODO: version code = tTagNew(pTagArray, 1, false, ppTag); - if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); - return code; - } +end: + if(isJson){ + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } + } taosArrayDestroy(pTagArray); - taosMemoryFreeClear(pTagBuf); return TSDB_CODE_SUCCESS; } @@ -4281,71 +4258,64 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau SSchema* pTagSchemas = getTableTagSchema(pSuperTableMeta); SNode* pNode; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; int32_t index = 0; SArray* pTagArray = taosArrayInit(LIST_LENGTH(pStmt->pValsOfTags), sizeof(STagVal)); - char* pTagBuf = taosMemoryCalloc(1, TSDB_MAX_TAGS_LEN); - - const char* qTagBuf = pTagBuf; - - if (!pTagArray || !pTagBuf) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); + if (!pTagArray) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_TSC_OUT_OF_MEMORY); } + bool isJson = false; FOREACH(pNode, pStmt->pValsOfTags) { SValueNode* pVal = NULL; SSchema* pTagSchema = pTagSchemas + index; code = translateTagVal(pCxt, pSuperTableMeta->tableInfo.precision, pTagSchema, pNode, &pVal); - if (TSDB_CODE_SUCCESS == code) { - if (NULL == pVal) { - pVal = (SValueNode*)pNode; - } else { - REPLACE_NODE(pVal); - } - } -#ifdef JSON_TAG_REFACTOR - if (TSDB_CODE_SUCCESS == code) { - code = addValToKVRow(pCxt, pVal, pTagSchema + index++, pBuilder); - } -#endif - if (TSDB_CODE_SUCCESS == code) { - if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { - char* tmpVal = nodesGetValueFromNode(pVal); - if (IS_VAR_DATA_TYPE(pTagSchema->type)) { - memcpy(pTagBuf, varDataVal(tmpVal), varDataLen(tmpVal)); - tTagValPush(pTagArray, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, varDataLen(tmpVal), false); - pTagBuf += varDataLen(tmpVal); - } else { - memcpy(pTagBuf, tmpVal, TYPE_BYTES[pTagSchema->type]); - tTagValPush(pTagArray, &pTagSchema->colId, pTagSchema->type, (uint8_t*)pTagBuf, TYPE_BYTES[pTagSchema->type], - false); - pTagBuf += TYPE_BYTES[pTagSchema->type]; - } - } - ++index; - } - // TODO: buf is need to store the tags - - // TODO: JSON_TAG_TODO remove below codes if code is 0 all the time. if (TSDB_CODE_SUCCESS != code) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); - return generateSyntaxErrMsg(&pCxt->msgBuf, code); + goto end; } + if (NULL == pVal) { + pVal = (SValueNode*)pNode; + } else { + REPLACE_NODE(pVal); + } + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); + goto end; + } + + isJson = true; + code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else if (pVal->node.resType.type != TSDB_DATA_TYPE_NULL) { + char* tmpVal = nodesGetValueFromNode(pVal); + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; + if (IS_VAR_DATA_TYPE(pTagSchema->type)) { + val.pData = varDataVal(tmpVal); + val.nData = varDataLen(tmpVal); + } else { + memcpy(&val.i64, tmpVal, pTagSchema->bytes); + } + taosArrayPush(pTagArray, &val); + } + ++index; } - // TODO: JSON_TAG_TODO: version? - // TODO: JSON_TAG_REFACTOR: json or not - if (0 != (code = tTagNew(pTagArray, 1, false, ppTag))) { - taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); - return generateSyntaxErrMsg(&pCxt->msgBuf, code); + code = tTagNew(pTagArray, 1, false, ppTag); + +end: + if(isJson){ + for (int i = 0; i < taosArrayGetSize(pTagArray); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagArray, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } } taosArrayDestroy(pTagArray); - taosMemoryFreeClear(qTagBuf); - return TSDB_CODE_SUCCESS; + return code; } static int32_t checkCreateSubTable(STranslateContext* pCxt, SCreateSubTableClause* pStmt) { @@ -4602,39 +4572,42 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->isNull = (TSDB_DATA_TYPE_NULL == pStmt->pVal->node.resType.type); if (pStmt->pVal->node.resType.type == TSDB_DATA_TYPE_JSON) { -#ifdef JSON_TAG_REFACTOR - SKVRowBuilder kvRowBuilder = {0}; - int32_t code = tdInitKVRowBuilder(&kvRowBuilder); - - if (TSDB_CODE_SUCCESS != code) { - return TSDB_CODE_OUT_OF_MEMORY; - } if (pStmt->pVal->literal && strlen(pStmt->pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pStmt->pVal->literal); } - - code = parseJsontoTagData(pStmt->pVal->literal, &kvRowBuilder, &pCxt->msgBuf, pSchema->colId); - if (TSDB_CODE_SUCCESS != code) { + SArray *pTagVals = taosArrayInit(1, sizeof(STagVal)); + int32_t code = TSDB_CODE_SUCCESS; + STag* pTag = NULL; + do{ + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pCxt->msgBuf); + if (TSDB_CODE_SUCCESS != code) { + break; + } + code = tTagNew(pTagVals, 1, false, &pTag); + }while(0); + for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { + STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); + if(IS_VAR_DATA_TYPE(p->type)){ + taosMemoryFree(p->pData); + } + } + taosArrayDestroy(pTagVals); + if (code != TSDB_CODE_SUCCESS){ return code; } - - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - if (NULL == row) { - tdDestroyKVRowBuilder(&kvRowBuilder); - return TSDB_CODE_OUT_OF_MEMORY; - } - pReq->nTagVal = kvRowLen(row); - pReq->pTagVal = row; - pStmt->pVal->datum.p = row; // for free - tdDestroyKVRowBuilder(&kvRowBuilder); -#endif + pReq->nTagVal = pTag->len; + pReq->pTagVal = (uint8_t *)pTag; + pStmt->pVal->datum.p = (char*)pTag; // for free } else { pReq->nTagVal = pStmt->pVal->node.resType.bytes; - if (TSDB_DATA_TYPE_NCHAR == pStmt->pVal->node.resType.type) { - pReq->nTagVal = pReq->nTagVal * TSDB_NCHAR_SIZE; - } pReq->pTagVal = nodesGetValueFromNode(pStmt->pVal); + + // data and length are seperated for new tag format STagVal + if (IS_VAR_DATA_TYPE(pStmt->pVal->node.resType.type)) { + pReq->nTagVal = varDataLen(pReq->pTagVal); + pReq->pTagVal = varDataVal(pReq->pTagVal); + } } return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d070ffaf5c..726028355f 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -328,13 +328,9 @@ static bool isValidateTag(char* input) { return true; } -#ifdef JSON_TAG_REFACTOR -int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* pMsgBuf, int16_t startColId) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* pMsgBuf) { // set json NULL data - uint8_t jsonNULL = TSDB_DATA_TYPE_NULL; - int32_t jsonIndex = startColId + 1; if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { - tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); return TSDB_CODE_SUCCESS; } @@ -349,13 +345,12 @@ int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBu return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); } - int32_t retCode = 0; - char* tagKV = NULL; + int32_t retCode = TSDB_CODE_SUCCESS; SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); for (int32_t i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { - qError("json inner error:%d", i); + uError("json inner error:%d", i); retCode = buildSyntaxErrMsg(pMsgBuf, "json inner error", json); goto end; } @@ -367,89 +362,60 @@ int32_t parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBu } size_t keyLen = strlen(jsonKey); if (keyLen > TSDB_MAX_JSON_KEY_LEN) { - qError("json key too long error"); + uError("json key too long error"); retCode = buildSyntaxErrMsg(pMsgBuf, "json key too long, more than 256", jsonKey); goto end; } if (keyLen == 0 || taosHashGet(keyHash, jsonKey, keyLen) != NULL) { continue; } - // key: keyLen + VARSTR_HEADER_SIZE, value type: CHAR_BYTES, value reserved: DOUBLE_BYTES - tagKV = taosMemoryCalloc(keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES, 1); - if (!tagKV) { - retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; - goto end; - } - strncpy(varDataVal(tagKV), jsonKey, keyLen); - varDataSetLen(tagKV, keyLen); - if (taosHashGetSize(keyHash) == 0) { - uint8_t jsonNotNULL = TSDB_DATA_TYPE_JSON; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, &jsonNotNULL, CHAR_BYTES); // add json type - } - taosHashPut(keyHash, jsonKey, keyLen, &keyLen, - CHAR_BYTES); // add key to hash to remove dumplicate, value is useless + STagVal val = {0}; + val.pKey = jsonKey; + taosHashPut(keyHash, jsonKey, keyLen, &keyLen, CHAR_BYTES); // add key to hash to remove dumplicate, value is useless if (item->type == cJSON_String) { // add json value format: type|data char* jsonValue = item->valuestring; int32_t valLen = (int32_t)strlen(jsonValue); - int32_t totalLen = keyLen + VARSTR_HEADER_SIZE + valLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE + CHAR_BYTES; - char* tmp = taosMemoryRealloc(tagKV, totalLen); + char* tmp = taosMemoryCalloc(1, valLen * TSDB_NCHAR_SIZE); if (!tmp) { retCode = TSDB_CODE_TSC_OUT_OF_MEMORY; goto end; } - tagKV = tmp; - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); - *valueType = TSDB_DATA_TYPE_NCHAR; - if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)varDataVal(valueData), + val.type = TSDB_DATA_TYPE_NCHAR; + if (valLen > 0 && !taosMbsToUcs4(jsonValue, valLen, (TdUcs4*)tmp, (int32_t)(valLen * TSDB_NCHAR_SIZE), &valLen)) { - qError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, + uError("charset:%s to %s. val:%s, errno:%s, convert failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, jsonValue, strerror(errno)); retCode = buildSyntaxErrMsg(pMsgBuf, "charset convert json error", jsonValue); goto end; } - - varDataSetLen(valueData, valLen); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, totalLen); + val.nData = valLen; + val.pData = tmp; } else if (item->type == cJSON_Number) { if (!isfinite(item->valuedouble)) { - qError("json value is invalidate"); + uError("json value is invalidate"); retCode = buildSyntaxErrMsg(pMsgBuf, "json value number is illegal", json); goto end; } - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); - *valueType = TSDB_DATA_TYPE_DOUBLE; - *((double*)valueData) = item->valuedouble; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + DOUBLE_BYTES); + val.type = TSDB_DATA_TYPE_DOUBLE; + *((double*)&(val.i64)) = item->valuedouble; } else if (item->type == cJSON_True || item->type == cJSON_False) { - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - char* valueData = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); - *valueType = TSDB_DATA_TYPE_BOOL; - *valueData = (char)(item->valueint); - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES + CHAR_BYTES); + val.type = TSDB_DATA_TYPE_BOOL; + *((char*)&(val.i64)) = (char)(item->valueint); } else if (item->type == cJSON_NULL) { - char* valueType = POINTER_SHIFT(tagKV, keyLen + VARSTR_HEADER_SIZE); - *valueType = TSDB_DATA_TYPE_NULL; - tdAddColToKVRow(kvRowBuilder, jsonIndex++, tagKV, keyLen + VARSTR_HEADER_SIZE + CHAR_BYTES); + val.type = TSDB_DATA_TYPE_NULL; } else { retCode = buildSyntaxErrMsg(pMsgBuf, "invalidate json value", json); goto end; } - } - - if (taosHashGetSize(keyHash) == 0) { // set json NULL true - tdAddColToKVRow(kvRowBuilder, jsonIndex, &jsonNULL, CHAR_BYTES); + taosArrayPush(pTagVals, &val); } end: - taosMemoryFree(tagKV); taosHashCleanup(keyHash); cJSON_Delete(root); return retCode; } -#endif static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { if (NULL != pTablesHash) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index b68c292feb..c4b3809035 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -922,29 +922,12 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -char *getJsonValue(char *json, char *key) { // todo - json++; // jump type - - STagVal tagVal = {.pKey = key}; - tTagGet(((const STag *)json), &tagVal); - return (char *)tagVal.pData; -#if 0 - int16_t cols = kvRowNCols(json); - for (int i = 0; i < cols; ++i) { - SColIdx *pColIdx = kvRowColIdxAt(json, i); - char *data = kvRowColVal(json, pColIdx); - if(i == 0){ - if(*data == TSDB_DATA_TYPE_NULL) { - return NULL; - } - continue; - } - if(memcmp(key, data, varDataTLen(data)) == 0){ - return data + varDataTLen(data); - } +STagVal *getJsonValue(char *json, STagVal *tagVal) { + bool find = tTagGet(((const STag *)json), tagVal); // json value is null and not exist is different + if(!find){ + return NULL; } - return NULL; -#endif + return tagVal; } void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { @@ -963,13 +946,13 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO continue; } char *pLeftData = colDataGetVarData(pLeft->columnData, i); - char *value = getJsonValue(pLeftData, pRightData); - if (!value) { - colDataSetNull_var(pOutputCol, i); - pOutputCol->hasNull = true; - continue; + STagVal val = {.pKey = pRightData}; + STagVal *value = getJsonValue(pLeftData, &val); + char *data = tTagValToData(value, true); + colDataAppend(pOutputCol, i, data, data == NULL); + if(value && IS_VAR_DATA_TYPE(value->type) && data){ + taosMemoryFree(data) } - colDataAppend(pOutputCol, i, value, false); } } From f89ce48e664acf95d9caca6fdd7723a212867c90 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 20:18:02 +0800 Subject: [PATCH 13/22] feat:add new logic for new tag format --- source/libs/parser/src/parInsert.c | 72 +++++++++++-------- source/libs/parser/src/parTranslater.c | 6 +- source/libs/scalar/src/sclvector.c | 2 +- .../libs/scalar/test/scalar/scalarTests.cpp | 40 ++++------- 4 files changed, 60 insertions(+), 60 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index d670f5ad38..05e5d46c7e 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -958,6 +958,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint SArray *pTagVals = taosArrayInit(pCxt->tags.numOfBound, sizeof(STagVal)); SToken sToken; bool isParseBindParam = false; + bool isJson = false; for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); @@ -994,6 +995,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint if(code != TSDB_CODE_SUCCESS){ goto end; } + isJson = true; }else{ STagVal val = {0}; code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg); @@ -1014,7 +1016,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint } STag* pTag = NULL; - code = tTagNew(pTagVals, 1, false, &pTag); + code = tTagNew(pTagVals, 1, isJson, &pTag); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -1404,7 +1406,6 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) { .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK), .totalNum = 0, - .pTagVals = NULL, .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT), .pStmtCb = pContext->pStmtCb}; @@ -1526,6 +1527,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN int32_t code = TSDB_CODE_SUCCESS; SSchema* pSchema = pDataBlock->pTableMeta->schema; + bool isJson = false; for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { continue; @@ -1537,42 +1539,56 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN if (IS_VAR_DATA_TYPE(pTagSchema->type)) { colLen = bind[c].length[0]; } - - STagVal val = {0}; - if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ - val.pData = (uint8_t*)bind[c].buffer; - val.nData = colLen; - }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ - int32_t output = 0; - void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE); - if(p == NULL){ - code = TSDB_CODE_OUT_OF_MEMORY; + if (pTagSchema->type == TSDB_DATA_TYPE_JSON) { + if (colLen > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + code = buildSyntaxErrMsg(&pBuf, "json string too long than 4095", bind[c].buffer); goto end; } - if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { - if (errno == E2BIG) { - taosMemoryFree(p); - code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + + isJson = true; + char *tmp = taosMemoryCalloc(1, colLen + 1); + memcpy(tmp, bind[c].buffer, colLen); + code = parseJsontoTagData(tmp, pTagArray, &pBuf); + taosMemoryFree(tmp); + if(code != TSDB_CODE_SUCCESS){ + goto end; + } + }else{ + STagVal val = {0}; + if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ + val.pData = (uint8_t*)bind[c].buffer; + val.nData = colLen; + }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ + int32_t output = 0; + void *p = taosMemoryCalloc(1, colLen * TSDB_NCHAR_SIZE); + if(p == NULL){ + code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - char buf[512] = {0}; - snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); - taosMemoryFree(p); - code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); - goto end; + if (!taosMbsToUcs4(bind[c].buffer, colLen, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (errno == E2BIG) { + taosMemoryFree(p); + code = generateSyntaxErrMsg(&pBuf, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + goto end; + } + char buf[512] = {0}; + snprintf(buf, tListLen(buf), " taosMbsToUcs4 error:%s", strerror(errno)); + taosMemoryFree(p); + code = buildSyntaxErrMsg(&pBuf, buf, bind[c].buffer); + goto end; + } + val.pData = p; + val.nData = output; + }else{ + memcpy(&val.i64, bind[c].buffer, colLen); } - val.pData = p; - val.nData = output; - }else{ - memcpy(&val.i64, bind[c].buffer, colLen); + taosArrayPush(pTagArray, &val); } - taosArrayPush(pTagArray, &val); } STag* pTag = NULL; - // TODO: stmt support json - if (0 != tTagNew(pTagArray, 1, false, &pTag)) { + if (0 != tTagNew(pTagArray, 1, isJson, &pTag)) { code = buildInvalidOperationMsg(&pBuf, "out of memory"); goto end; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 96ee694e1f..bd31c16f54 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4235,7 +4235,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } - code = tTagNew(pTagArray, 1, false, ppTag); + code = tTagNew(pTagArray, 1, isJson, ppTag); end: if(isJson){ @@ -4302,7 +4302,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } ++index; } - code = tTagNew(pTagArray, 1, false, ppTag); + code = tTagNew(pTagArray, 1, isJson, ppTag); end: if(isJson){ @@ -4584,7 +4584,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS if (TSDB_CODE_SUCCESS != code) { break; } - code = tTagNew(pTagVals, 1, false, &pTag); + code = tTagNew(pTagVals, 1, true, &pTag); }while(0); for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index c4b3809035..f84f564b8d 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -951,7 +951,7 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO char *data = tTagValToData(value, true); colDataAppend(pOutputCol, i, data, data == NULL); if(value && IS_VAR_DATA_TYPE(value->type) && data){ - taosMemoryFree(data) + taosMemoryFree(data); } } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 3a16e0a969..2081c6a3f0 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1105,24 +1105,16 @@ void makeCalculate(void *json, void *key, int32_t rightType, void *rightData, do nodesDestroyNode(opNode); } -#if 0 TEST(columnTest, json_column_arith_op) { scltInitLogFile(); char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); - SKVRowBuilder kvRowBuilder; - tdInitKVRowBuilder(&kvRowBuilder); - parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1); - if(tmp == NULL){ - ASSERT_TRUE(0); - } - memmove(tmp+1, tmp, kvRowLen(tmp)); - *tmp = TSDB_DATA_TYPE_JSON; - row = tmp; + SArray *tags = taosArrayInit(1, sizeof(STagVal)); + parseJsontoTagData(rightv, tags, NULL); + STag* row = NULL; + tTagNew(tags, 1, true, &row); const int32_t len = 8; EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, @@ -1176,10 +1168,10 @@ TEST(columnTest, json_column_arith_op) { makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes5[i], op[i]); } - tdDestroyKVRowBuilder(&kvRowBuilder); + taosArrayDestroy(tags); taosMemoryFree(row); } -#endif + void *prepareNchar(char* rightData){ int32_t len = 0; int32_t inputLen = strlen(rightData); @@ -1189,24 +1181,17 @@ void *prepareNchar(char* rightData){ varDataSetLen(t, len); return t; } -#if 0 + TEST(columnTest, json_column_logic_op) { scltInitLogFile(); char *rightvTmp= "{\"k1\":4,\"k2\":\"hello\",\"k3\":null,\"k4\":true,\"k5\":5.44,\"k6\":\"6.6hello\"}"; char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); - SKVRowBuilder kvRowBuilder; - tdInitKVRowBuilder(&kvRowBuilder); - parseJsontoTagData(rightv, &kvRowBuilder, NULL, 0); - SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); - char *tmp = (char *)taosMemoryRealloc(row, kvRowLen(row)+1); - if(tmp == NULL){ - ASSERT_TRUE(0); - } - memmove(tmp+1, tmp, kvRowLen(tmp)); - *tmp = TSDB_DATA_TYPE_JSON; - row = tmp; + SArray *tags = taosArrayInit(1, sizeof(STagVal)); + parseJsontoTagData(rightv, tags, NULL); + STag* row = NULL; + tTagNew(tags, 1, true, &row); const int32_t len = 9; const int32_t len1 = 4; @@ -1306,10 +1291,9 @@ TEST(columnTest, json_column_logic_op) { taosMemoryFree(rightData); } - tdDestroyKVRowBuilder(&kvRowBuilder); + taosArrayDestroy(tags); taosMemoryFree(row); } -#endif TEST(columnTest, smallint_value_add_int_column) { scltInitLogFile(); From d95ec693c00f06215cc4625195a6f4fb72243901 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 21:05:43 +0800 Subject: [PATCH 14/22] feat:add new logic for new tag format --- include/common/tdataformat.h | 4 ++-- source/common/src/tdataformat.c | 3 +++ source/libs/parser/inc/parUtil.h | 2 +- source/libs/parser/src/parInsert.c | 16 +++++++--------- source/libs/parser/src/parTranslater.c | 11 +++++------ source/libs/parser/src/parUtil.c | 16 +++++++++++----- source/libs/scalar/test/scalar/scalarTests.cpp | 6 ++---- 7 files changed, 31 insertions(+), 27 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 38bbee1f1d..18932119ff 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -136,8 +136,8 @@ struct STagVal { }; #pragma pack(push, 1) -#define TD_TAG_JSON ((int8_t)0x80) // distinguish JSON string and JSON value with the highest bit -#define TD_TAG_LARGE ((int8_t)0x40) +#define TD_TAG_JSON ((int8_t)0x40) // distinguish JSON string and JSON value with the highest bit +#define TD_TAG_LARGE ((int8_t)0x20) struct STag { int8_t flags; int16_t len; diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 722fa57bad..71842655ec 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -567,6 +567,9 @@ static void debugPrintTagVal(int8_t type, const void *val, int32_t vlen, const c case TSDB_DATA_TYPE_UBIGINT: printf("%s:%d type:%d vlen:%d, val:%" PRIu64 "\n", tag, ln, (int32_t)type, vlen, *(uint64_t *)val); break; + case TSDB_DATA_TYPE_NULL: + printf("%s:%d type:%d vlen:%d, val:%" PRIi8 "\n", tag, ln, (int32_t)type, vlen, *(int8_t *)val); + break; default: ASSERT(0); break; diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 35cab7db36..00a8767e0c 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -55,7 +55,7 @@ int32_t getNumOfColumns(const STableMeta* pTableMeta); int32_t getNumOfTags(const STableMeta* pTableMeta); STableComInfo getTableInfo(const STableMeta* pTableMeta); STableMeta* tableMetaDup(const STableMeta* pTableMeta); -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* errMsg); +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf); int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 05e5d46c7e..df1e597b86 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -959,6 +959,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint SToken sToken; bool isParseBindParam = false; bool isJson = false; + STag* pTag = NULL; for (int i = 0; i < pCxt->tags.numOfBound; ++i) { NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); @@ -990,7 +991,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint taosMemoryFree(tmpTokenBuf); goto end; } - code = parseJsontoTagData(sToken.z, pTagVals, &pCxt->msg); + code = parseJsontoTagData(sToken.z, pTagVals, &pTag, &pCxt->msg); taosMemoryFree(tmpTokenBuf); if(code != TSDB_CODE_SUCCESS){ goto end; @@ -1015,9 +1016,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint goto end; } - STag* pTag = NULL; - code = tTagNew(pTagVals, 1, isJson, &pTag); - if (code != TSDB_CODE_SUCCESS) { + if(!isJson && (code = tTagNew(pTagVals, 1, false, &pTag)) != TSDB_CODE_SUCCESS) { goto end; } @@ -1528,6 +1527,8 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN SSchema* pSchema = pDataBlock->pTableMeta->schema; bool isJson = false; + STag* pTag = NULL; + for (int c = 0; c < tags->numOfBound; ++c) { if (bind[c].is_null && bind[c].is_null[0]) { continue; @@ -1548,7 +1549,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN isJson = true; char *tmp = taosMemoryCalloc(1, colLen + 1); memcpy(tmp, bind[c].buffer, colLen); - code = parseJsontoTagData(tmp, pTagArray, &pBuf); + code = parseJsontoTagData(tmp, pTagArray, &pTag, &pBuf); taosMemoryFree(tmp); if(code != TSDB_CODE_SUCCESS){ goto end; @@ -1586,10 +1587,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN } } - STag* pTag = NULL; - - if (0 != tTagNew(pTagArray, 1, isJson, &pTag)) { - code = buildInvalidOperationMsg(&pBuf, "out of memory"); + if (!isJson && (code = tTagNew(pTagArray, 1, false, &pTag)) != TSDB_CODE_SUCCESS) { goto end; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index bd31c16f54..16c35fce84 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4218,7 +4218,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } isJson = true; - code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); if(code != TSDB_CODE_SUCCESS){ goto end; } @@ -4235,7 +4235,7 @@ static int32_t buildKVRowForBindTags(STranslateContext* pCxt, SCreateSubTableCla } } - code = tTagNew(pTagArray, 1, isJson, ppTag); + if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: if(isJson){ @@ -4285,7 +4285,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } isJson = true; - code = parseJsontoTagData(pVal->literal, pTagArray, &pCxt->msgBuf); + code = parseJsontoTagData(pVal->literal, pTagArray, ppTag, &pCxt->msgBuf); if(code != TSDB_CODE_SUCCESS){ goto end; } @@ -4302,7 +4302,7 @@ static int32_t buildKVRowForAllTags(STranslateContext* pCxt, SCreateSubTableClau } ++index; } - code = tTagNew(pTagArray, 1, isJson, ppTag); + if(!isJson) code = tTagNew(pTagArray, 1, false, ppTag); end: if(isJson){ @@ -4580,11 +4580,10 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS int32_t code = TSDB_CODE_SUCCESS; STag* pTag = NULL; do{ - code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pCxt->msgBuf); + code = parseJsontoTagData(pStmt->pVal->literal, pTagVals, &pTag, &pCxt->msgBuf); if (TSDB_CODE_SUCCESS != code) { break; } - code = tTagNew(pTagVals, 1, true, &pTag); }while(0); for (int i = 0; i < taosArrayGetSize(pTagVals); ++i) { STagVal *p = (STagVal *)taosArrayGet(pTagVals, i); diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 726028355f..18fb5322b1 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -328,24 +328,27 @@ static bool isValidateTag(char* input) { return true; } -int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* pMsgBuf) { +int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf) { + int32_t retCode = TSDB_CODE_SUCCESS; // set json NULL data if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { - return TSDB_CODE_SUCCESS; + retCode = TSDB_CODE_SUCCESS; + goto end; } // set json real data cJSON* root = cJSON_Parse(json); if (root == NULL) { - return buildSyntaxErrMsg(pMsgBuf, "json parse error", json); + retCode = buildSyntaxErrMsg(pMsgBuf, "json parse error", json); + goto end; } int32_t size = cJSON_GetArraySize(root); if (!cJSON_IsObject(root)) { - return buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); + retCode = buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); + goto end; } - int32_t retCode = TSDB_CODE_SUCCESS; SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); for (int32_t i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); @@ -413,6 +416,9 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, SMsgBuf* pMsgBuf) end: taosHashCleanup(keyHash); + if(retCode == TSDB_CODE_SUCCESS){ + tTagNew(pTagVals, 1, true, ppTag); + } cJSON_Delete(root); return retCode; } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 2081c6a3f0..6ddf4b12b9 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -1112,9 +1112,8 @@ TEST(columnTest, json_column_arith_op) { char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); SArray *tags = taosArrayInit(1, sizeof(STagVal)); - parseJsontoTagData(rightv, tags, NULL); STag* row = NULL; - tTagNew(tags, 1, true, &row); + parseJsontoTagData(rightv, tags, &row, NULL); const int32_t len = 8; EOperatorType op[len] = {OP_TYPE_ADD, OP_TYPE_SUB, OP_TYPE_MULTI, OP_TYPE_DIV, @@ -1189,9 +1188,8 @@ TEST(columnTest, json_column_logic_op) { char rightv[256] = {0}; memcpy(rightv, rightvTmp, strlen(rightvTmp)); SArray *tags = taosArrayInit(1, sizeof(STagVal)); - parseJsontoTagData(rightv, tags, NULL); STag* row = NULL; - tTagNew(tags, 1, true, &row); + parseJsontoTagData(rightv, tags, &row, NULL); const int32_t len = 9; const int32_t len1 = 4; From f6b700ff4581c49aaf740816b8b48cf4b237dbac Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 22:30:08 +0800 Subject: [PATCH 15/22] feat:add new logic for new tag format --- source/common/src/tdataformat.c | 2 +- source/libs/scalar/src/sclvector.c | 22 +++++++++++-------- .../libs/scalar/test/scalar/scalarTests.cpp | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 71842655ec..f946aa3b01 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -672,8 +672,8 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { if (IS_VAR_DATA_TYPE(pTagVal->type)) { n += tGetBinary(p + n, &pTagVal->pData, &pTagVal->nData); } else { - n += tDataTypes[pTagVal->type].bytes; memcpy(&(pTagVal->i64), p + n, tDataTypes[pTagVal->type].bytes); + n += tDataTypes[pTagVal->type].bytes; } return n; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f84f564b8d..e844b3cdb6 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -922,12 +922,13 @@ static void doReleaseVec(SColumnInfoData* pCol, int32_t type) { } } -STagVal *getJsonValue(char *json, STagVal *tagVal) { - bool find = tTagGet(((const STag *)json), tagVal); // json value is null and not exist is different - if(!find){ - return NULL; +STagVal getJsonValue(char *json, char *key, bool *isExist) { + STagVal val = {.pKey = key}; + bool find = tTagGet(((const STag *)json), &val); // json value is null and not exist is different + if(isExist){ + *isExist = find; } - return tagVal; + return val; } void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { @@ -939,6 +940,8 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO pOut->numOfRows = TMAX(pLeft->numOfRows, pRight->numOfRows); char *pRightData = colDataGetVarData(pRight->columnData, 0); + char *jsonKey = taosMemoryCalloc(1, varDataLen(pRightData) + 1); + memcpy(jsonKey, varDataVal(pRightData), varDataLen(pRightData)); for (; i >= 0 && i < pLeft->numOfRows; i += step) { if (colDataIsNull_var(pLeft->columnData, i)) { colDataSetNull_var(pOutputCol, i); @@ -946,14 +949,15 @@ void vectorJsonArrow(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pO continue; } char *pLeftData = colDataGetVarData(pLeft->columnData, i); - STagVal val = {.pKey = pRightData}; - STagVal *value = getJsonValue(pLeftData, &val); - char *data = tTagValToData(value, true); + bool isExist = false; + STagVal value = getJsonValue(pLeftData, jsonKey, &isExist); + char *data = isExist ? tTagValToData(&value, true) : NULL; colDataAppend(pOutputCol, i, data, data == NULL); - if(value && IS_VAR_DATA_TYPE(value->type) && data){ + if(isExist && IS_VAR_DATA_TYPE(value.type) && data){ taosMemoryFree(data); } } + taosMemoryFree(jsonKey); } void vectorMathAdd(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, int32_t _ord) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 6ddf4b12b9..e971055a66 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -217,7 +217,7 @@ void scltMakeOpNode(SNode **pNode, EOperatorType opType, int32_t resType, SNode SOperatorNode *onode = (SOperatorNode *)node; onode->node.resType.type = resType; onode->node.resType.bytes = tDataTypes[resType].bytes; - + onode->opType = opType; onode->pLeft = pLeft; onode->pRight = pRight; From 51b5e359333ec62ed294b9bdcb98f9e0f4429fa2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 31 May 2022 23:38:21 +0800 Subject: [PATCH 16/22] feat:add new logic for new tag format --- include/common/tdataformat.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 16 +++++++++------- source/dnode/vnode/src/meta/metaTable.c | 14 ++++++++++---- source/libs/parser/src/parUtil.c | 9 ++++++--- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 18932119ff..53197d2d7b 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -120,6 +120,7 @@ struct SColVal { uint8_t *pData; }; +#pragma pack(push, 1) struct STagVal { union { int16_t cid; @@ -135,7 +136,6 @@ struct STagVal { }; }; -#pragma pack(push, 1) #define TD_TAG_JSON ((int8_t)0x40) // distinguish JSON string and JSON value with the highest bit #define TD_TAG_LARGE ((int8_t)0x20) struct STag { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index dead85f854..24aaeec406 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -567,14 +567,16 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); STag *tag = (STag *)pEntry->ctbEntry.pTags; - tTagGet(tag, val); + if (type == TSDB_DATA_TYPE_JSON){ + if(tag->nTag == 0){ + return NULL; + } + return tag; + } + bool find = tTagGet(tag, val); - if(val->type == TSDB_DATA_TYPE_NULL){ + if(!find){ return NULL; } - if (type == TSDB_DATA_TYPE_JSON){ - return tag; - }else{ - return val; - } + return val; } \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 984688fc19..5ac13d478d 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -787,10 +787,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - STagVal tagVal = {.cid = pTagColumn->colId}; - tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if(pTagColumn->type != TSDB_DATA_TYPE_JSON){ + STagVal tagVal = {.cid = pTagColumn->colId}; + tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + }else{ + //pTagData = pCtbEntry->ctbEntry.pTags; + //nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; + } + // update tag index #ifdef USE_INVERTED_INDEX diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 18fb5322b1..dfd89a230d 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -330,6 +330,9 @@ static bool isValidateTag(char* input) { int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMsgBuf* pMsgBuf) { int32_t retCode = TSDB_CODE_SUCCESS; + cJSON* root = NULL; + SHashObj* keyHash = NULL; + int32_t size = 0; // set json NULL data if (!json || strtrim((char*)json) == 0 || strcasecmp(json, TSDB_DATA_NULL_STR_L) == 0) { retCode = TSDB_CODE_SUCCESS; @@ -337,19 +340,19 @@ int32_t parseJsontoTagData(const char* json, SArray* pTagVals, STag **ppTag, SMs } // set json real data - cJSON* root = cJSON_Parse(json); + root = cJSON_Parse(json); if (root == NULL) { retCode = buildSyntaxErrMsg(pMsgBuf, "json parse error", json); goto end; } - int32_t size = cJSON_GetArraySize(root); + size = cJSON_GetArraySize(root); if (!cJSON_IsObject(root)) { retCode = buildSyntaxErrMsg(pMsgBuf, "json error invalide value", json); goto end; } - SHashObj* keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); + keyHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, false); for (int32_t i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { From 172834bc59c6ae1b3a4f2c3197588cbf6ef3a16e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 09:58:58 +0800 Subject: [PATCH 17/22] feat:fix conflict merge from 3.0 --- source/dnode/vnode/src/inc/meta.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 82 +++++++++++++++++++++++++ source/dnode/vnode/src/meta/metaTable.c | 2 +- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 9da603f894..52e576ee52 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -117,7 +117,7 @@ typedef struct { } SSmaIdxKey; // metaTable ================== -int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int8_t type, tb_uid_t uid, +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void* pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey** ppTagIdxKey, int32_t* nTagIdxKey); #ifndef META_REFACT diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index c4c232ff34..65a61707a3 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -589,3 +589,85 @@ const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { } return val; } + +typedef struct { + SMeta * pMeta; + TBC * pCur; + tb_uid_t suid; + int16_t cid; + int16_t type; + void * pKey; + void * pVal; + int32_t kLen; + int32_t vLen; +} SIdxCursor; + +int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { + SIdxCursor *pCursor = NULL; + + char *tagData = param->val; + + int32_t ret = 0, valid = 0; + pCursor = (SIdxCursor *)taosMemoryCalloc(1, sizeof(SIdxCursor)); + pCursor->pMeta = pMeta; + pCursor->suid = param->suid; + pCursor->cid = param->cid; + pCursor->type = param->type; + + metaRLock(pMeta); + ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL); + if (ret < 0) { + goto END; + } + STagIdxKey *pKey = NULL; + int32_t nKey = 0; + + int32_t nTagData = 0; + if(IS_VAR_DATA_TYPE(param->type)){ + nTagData = strlen(param->val); + }else{ + nTagData = tDataTypes[param->type].bytes + } + ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, nTagData, pCursor->type, + param->reverse ? INT64_MAX : INT64_MIN, &pKey, &nKey); + if (ret != 0) { + goto END; + } + int cmp = 0; + if (tdbTbcMoveTo(pCursor->pCur, pKey, nKey, &cmp) < 0) { + goto END; + } + void * entryKey = NULL, *entryVal = NULL; + int32_t nEntryKey, nEntryVal; + while (1) { + valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); + if (valid < 0) { + break; + } + STagIdxKey *p = entryKey; + if (p != NULL) { + int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); + if (cmp == 0) { + // match + tb_uid_t tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); + taosArrayPush(pUids, &tuid); + } else if (cmp == 1) { + // not match but should continue to iter + } else { + // not match and no more result + break; + } + } + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) { + break; + } + } +END: + if (pCursor->pMeta) metaULock(pCursor->pMeta); + if (pCursor->pCur) tdbTbcClose(pCursor->pCur); + + taosMemoryFree(pCursor); + + return ret; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 0962de8b00..17ef6a270c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -731,7 +731,7 @@ static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); } -static int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, +int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, STagIdxKey **ppTagIdxKey, int32_t *nTagIdxKey) { // int32_t nTagData = 0; From f99e66df3270ad18ba7ac96caa37a7ab60773b5b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 10:29:10 +0800 Subject: [PATCH 18/22] fix:fix error in new tag format --- source/dnode/vnode/src/meta/metaQuery.c | 2 +- source/dnode/vnode/src/meta/metaTable.c | 11 ++++++++--- source/libs/executor/src/scanoperator.c | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 65a61707a3..a57eb4e899 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -626,7 +626,7 @@ int32_t metaFilteTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if(IS_VAR_DATA_TYPE(param->type)){ nTagData = strlen(param->val); }else{ - nTagData = tDataTypes[param->type].bytes + nTagData = tDataTypes[param->type].bytes; } ret = metaCreateTagIdxKey(pCursor->suid, pCursor->cid, param->val, nTagData, pCursor->type, param->reverse ? INT64_MAX : INT64_MIN, &pKey, &nKey); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 17ef6a270c..9dd1543f9c 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -787,11 +787,16 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + STagVal tagVal = {.cid = pTagColumn->colId}; if(pTagColumn->type != TSDB_DATA_TYPE_JSON){ - STagVal tagVal = {.cid = pTagColumn->colId}; tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; + if(IS_VAR_DATA_TYPE(pTagColumn->type)){ + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + }else{ + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } }else{ //pTagData = pCtbEntry->ctbEntry.pTags; //nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 75f125ff11..89340f2b85 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1594,7 +1594,8 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } colDataAppend(pDst, count, data, (data == NULL)); - if(pDst->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ + if(pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL + && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data != NULL){ taosMemoryFree(data); } } From 9283c5c3e9f551f384801202e1c8854db5853229 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 11:18:10 +0800 Subject: [PATCH 19/22] fix:fix error in new tag format --- source/libs/parser/src/parInsert.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 412afee1f2..149302e125 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -781,7 +781,7 @@ static void buildCreateTbReq(SVCreateTbReq* pTbReq, const char* tname, STag* pTa } static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, - int16_t timePrec, char* tmpTokenBuf, STagVal *val, SMsgBuf* pMsgBuf) { + int16_t timePrec, STagVal *val, SMsgBuf* pMsgBuf) { int64_t iv; uint64_t uv; char* endptr = NULL; @@ -795,7 +795,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, } val->cid = pSchema->colId; - val->type = pSchema->bytes; + val->type = pSchema->type; switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { @@ -955,12 +955,6 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, val->nData = output; break; } - case TSDB_DATA_TYPE_JSON: { - if (pToken->n > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { - return buildSyntaxErrMsg(pMsgBuf, "json string too long than 4095", pToken->z); - } - //return func(pMsgBuf, pToken->z, pToken->n, param); - } case TSDB_DATA_TYPE_TIMESTAMP: { if (parseTime(end, pToken, timePrec, &iv, pMsgBuf) != TSDB_CODE_SUCCESS) { return buildSyntaxErrMsg(pMsgBuf, "invalid timestamp", pToken->z); @@ -971,7 +965,7 @@ static int32_t parseTagToken(char** end, SToken* pToken, SSchema* pSchema, } } - return TSDB_CODE_FAILED; + return TSDB_CODE_SUCCESS; } // pSql -> tag1_value, ...) @@ -1021,7 +1015,7 @@ static int32_t parseTagsClause(SInsertParseContext* pCxt, SSchema* pSchema, uint isJson = true; }else{ STagVal val = {0}; - code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, tmpTokenBuf, &val, &pCxt->msg); + code = parseTagToken(&pCxt->pSql, &sToken, pTagSchema, precision, &val, &pCxt->msg); if (TSDB_CODE_SUCCESS != code) { taosMemoryFree(tmpTokenBuf); goto end; From 094aaa207778b675145bc778f7950fa52cf15424 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 12:44:28 +0800 Subject: [PATCH 20/22] fix:fix error in new tag format --- source/libs/parser/src/parInsert.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 149302e125..529e10be65 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1733,7 +1733,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN goto end; } }else{ - STagVal val = {0}; + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ val.pData = (uint8_t*)bind[c].buffer; val.nData = colLen; @@ -2120,7 +2120,7 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p SSchema* pTagSchema = &pSchema[tags->boundColumns[i]]; SSmlKv* kv = taosArrayGetP(cols, i); - STagVal val = {0}; + STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; if(pTagSchema->type == TSDB_DATA_TYPE_BINARY){ val.pData = (uint8_t *)kv->value; val.nData = kv->length; @@ -2131,10 +2131,10 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p code = TSDB_CODE_OUT_OF_MEMORY; goto end; } - if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), pSchema->bytes - VARSTR_HEADER_SIZE, &output)) { + if (!taosMbsToUcs4(kv->value, kv->length, (TdUcs4*)(p), pTagSchema->bytes - VARSTR_HEADER_SIZE, &output)) { if (errno == E2BIG) { taosMemoryFree(p); - code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pSchema->name); + code = generateSyntaxErrMsg(msg, TSDB_CODE_PAR_VALUE_TOO_LONG, pTagSchema->name); goto end; } char buf[512] = {0}; From eb82ac04a0ca86daebc108d6b3cf0e707debad9d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 13:06:25 +0800 Subject: [PATCH 21/22] fix:fix error in new tag format --- source/libs/executor/src/scanoperator.c | 3 ++- source/libs/parser/src/parInsert.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 89340f2b85..c5d19981cf 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -327,7 +327,8 @@ void addTagPseudoColumnData(SReadHandle *pHandle, SExprInfo* pPseudoExpr, int32_ for (int32_t i = 0; i < pBlock->info.rows; ++i) { colDataAppend(pColInfoData, i, data, (data == NULL)); } - if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ + if(pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL && + IS_VAR_DATA_TYPE(((const STagVal *)p)->type) && data){ taosMemoryFree(data); } } diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 529e10be65..4cc3da5beb 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -2126,7 +2126,7 @@ static int32_t smlBuildTagRow(SArray* cols, SParsedDataColInfo* tags, SSchema* p val.nData = kv->length; }else if(pTagSchema->type == TSDB_DATA_TYPE_NCHAR){ int32_t output = 0; - void *p = taosMemoryCalloc(1, kv->length * TSDB_NCHAR_SIZE); + void *p = taosMemoryCalloc(1, pTagSchema->bytes - VARSTR_HEADER_SIZE); if(p == NULL){ code = TSDB_CODE_OUT_OF_MEMORY; goto end; From 6a15fd7e5584b20b3a095fd6f2a6046bd39a9a9d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 1 Jun 2022 13:45:29 +0800 Subject: [PATCH 22/22] fix:fix compile error in windows --- include/util/tencode.h | 187 +++++++---------------------------------- 1 file changed, 30 insertions(+), 157 deletions(-) diff --git a/include/util/tencode.h b/include/util/tencode.h index 3e1d675870..bde2185b02 100644 --- a/include/util/tencode.h +++ b/include/util/tencode.h @@ -461,151 +461,64 @@ static FORCE_INLINE void* tDecoderMalloc(SDecoder* pCoder, int32_t size) { } // =========================================== -#define tPutV(p, v) \ - do { \ - int32_t n = 0; \ - for (;;) { \ - if (v <= 0x7f) { \ - if (p) p[n] = v; \ - n++; \ - break; \ - } \ - if (p) p[n] = (v & 0x7f) | 0x80; \ - n++; \ - v >>= 7; \ - } \ - return n; \ - } while (0) +#define tPutV(p, v) \ + int32_t n = 0; \ + for (;;) { \ + if (v <= 0x7f) { \ + if (p) p[n] = v; \ + n++; \ + break; \ + } \ + if (p) p[n] = (v & 0x7f) | 0x80; \ + n++; \ + v >>= 7; \ + } \ + return n; -#define tGetV(p, v) \ - do { \ - int32_t n = 0; \ - if (v) *v = 0; \ - for (;;) { \ - if (p[n] <= 0x7f) { \ - if (v) (*v) |= (p[n] << (7 * n)); \ - n++; \ - break; \ - } \ - if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ - n++; \ - } \ - return n; \ - } while (0) +#define tGetV(p, v) \ + int32_t n = 0; \ + if (v) *v = 0; \ + for (;;) { \ + if (p[n] <= 0x7f) { \ + if (v) (*v) |= (p[n] << (7 * n)); \ + n++; \ + break; \ + } \ + if (v) (*v) |= ((p[n] & 0x7f) << (7 * n)); \ + n++; \ + } \ + return n; // PUT -static FORCE_INLINE int32_t tPutU8(uint8_t* p, uint8_t v) { - if (p) ((uint8_t*)p)[0] = v; - return sizeof(uint8_t); -} - static FORCE_INLINE int32_t tPutI8(uint8_t* p, int8_t v) { if (p) ((int8_t*)p)[0] = v; return sizeof(int8_t); } -static FORCE_INLINE int32_t tPutU16(uint8_t* p, uint16_t v) { - if (p) ((uint16_t*)p)[0] = v; - return sizeof(uint16_t); -} - -static FORCE_INLINE int32_t tPutI16(uint8_t* p, int16_t v) { - if (p) ((int16_t*)p)[0] = v; - return sizeof(int16_t); -} - -static FORCE_INLINE int32_t tPutU32(uint8_t* p, uint32_t v) { - if (p) ((uint32_t*)p)[0] = v; - return sizeof(uint32_t); -} - -static FORCE_INLINE int32_t tPutI32(uint8_t* p, int32_t v) { - if (p) ((int32_t*)p)[0] = v; - return sizeof(int32_t); -} - -static FORCE_INLINE int32_t tPutU64(uint8_t* p, uint64_t v) { - if (p) ((uint64_t*)p)[0] = v; - return sizeof(uint64_t); -} - static FORCE_INLINE int32_t tPutI64(uint8_t* p, int64_t v) { if (p) ((int64_t*)p)[0] = v; return sizeof(int64_t); } -static FORCE_INLINE int32_t tPutFloat(uint8_t* p, float f) { - union { - uint32_t ui; - float f; - } v = {.f = f}; - - return tPutU32(p, v.ui); -} - -static FORCE_INLINE int32_t tPutDouble(uint8_t* p, double d) { - union { - uint64_t ui; - double d; - } v = {.d = d}; - - return tPutU64(p, v.ui); -} - -static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v); } +static FORCE_INLINE int32_t tPutU16v(uint8_t* p, uint16_t v) { tPutV(p, v) } static FORCE_INLINE int32_t tPutI16v(uint8_t* p, int16_t v) { return tPutU16v(p, ZIGZAGE(int16_t, v)); } -static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v); } +static FORCE_INLINE int32_t tPutU32v(uint8_t* p, uint32_t v) { tPutV(p, v) } static FORCE_INLINE int32_t tPutI32v(uint8_t* p, int32_t v) { return tPutU32v(p, ZIGZAGE(int32_t, v)); } -static FORCE_INLINE int32_t tPutU64v(uint8_t* p, uint64_t v) { tPutV(p, v); } - -static FORCE_INLINE int32_t tPutI64v(uint8_t* p, int64_t v) { return tPutU64v(p, ZIGZAGE(int64_t, v)); } - -// GET -static FORCE_INLINE int32_t tGetU8(uint8_t* p, uint8_t* v) { - if (v) *v = ((uint8_t*)p)[0]; - return sizeof(uint8_t); -} - static FORCE_INLINE int32_t tGetI8(uint8_t* p, int8_t* v) { if (v) *v = ((int8_t*)p)[0]; return sizeof(int8_t); } -static FORCE_INLINE int32_t tGetU16(uint8_t* p, uint16_t* v) { - if (v) *v = ((uint16_t*)p)[0]; - return sizeof(uint16_t); -} - -static FORCE_INLINE int32_t tGetI16(uint8_t* p, int16_t* v) { - if (v) *v = ((int16_t*)p)[0]; - return sizeof(int16_t); -} - -static FORCE_INLINE int32_t tGetU32(uint8_t* p, uint32_t* v) { - if (v) *v = ((uint32_t*)p)[0]; - return sizeof(uint32_t); -} - -static FORCE_INLINE int32_t tGetI32(uint8_t* p, int32_t* v) { - if (v) *v = ((int32_t*)p)[0]; - return sizeof(int32_t); -} - -static FORCE_INLINE int32_t tGetU64(uint8_t* p, uint64_t* v) { - if (v) *v = ((uint64_t*)p)[0]; - return sizeof(uint64_t); -} - static FORCE_INLINE int32_t tGetI64(uint8_t* p, int64_t* v) { if (v) *v = ((int64_t*)p)[0]; return sizeof(int64_t); } -static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v); } +static FORCE_INLINE int32_t tGetU16v(uint8_t* p, uint16_t* v) { tGetV(p, v) } static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { int32_t n; @@ -617,7 +530,7 @@ static FORCE_INLINE int32_t tGetI16v(uint8_t* p, int16_t* v) { return n; } -static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v); } +static FORCE_INLINE int32_t tGetU32v(uint8_t* p, uint32_t* v) { tGetV(p, v) } static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { int32_t n; @@ -629,46 +542,6 @@ static FORCE_INLINE int32_t tGetI32v(uint8_t* p, int32_t* v) { return n; } -static FORCE_INLINE int32_t tGetU64v(uint8_t* p, uint64_t* v) { tGetV(p, v); } - -static FORCE_INLINE int32_t tGetI64v(uint8_t* p, int64_t* v) { - int32_t n; - uint64_t tv; - - n = tGetU64v(p, &tv); - if (v) *v = ZIGZAGD(int64_t, tv); - - return n; -} - -static FORCE_INLINE int32_t tGetFloat(uint8_t* p, float* f) { - int32_t n = 0; - - union { - uint32_t ui; - float f; - } v; - - n = tGetU32(p, &v.ui); - - *f = v.f; - return n; -} - -static FORCE_INLINE int32_t tGetDouble(uint8_t* p, double* d) { - int32_t n = 0; - - union { - uint64_t ui; - double d; - } v; - - n = tGetU64(p, &v.ui); - - *d = v.d; - return n; -} - // ===================== static FORCE_INLINE int32_t tPutBinary(uint8_t* p, uint8_t* pData, uint32_t nData) { int n = 0;