From bc2c2c07a174630a6d5a70cf3c7c740800a21e15 Mon Sep 17 00:00:00 2001 From: xywang Date: Tue, 1 Jun 2021 11:54:22 +0000 Subject: [PATCH 01/43] [TD-2574]: refactored algorithms for top and bottom functions. --- src/query/src/qAggMain.c | 201 +++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 94 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 4935633b9c..a7106a14a9 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2201,6 +2201,101 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ } while (0) +static void heapSwap(tValuePair *a, tValuePair *b, const int16_t tagLen) { + char tag[32768]; + tValuePair temp; + + memset(tag, 0, sizeof(tag)); + temp.pTags = tag; + + VALUEPAIRASSIGN(&temp, a, tagLen); + VALUEPAIRASSIGN(a, b, tagLen); + VALUEPAIRASSIGN(b, &temp, tagLen); +} + +static void heapAdjust(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t start, int32_t end, bool minRoot) { + int32_t parent = start; + int32_t child = 2 * parent + 1; + + while (child <= end) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (minRoot) { + if (child + 1 <= end && pList[child]->v.i64 < pList[child + 1]->v.i64) { + child++; + } + + if (pList[parent]->v.i64 > pList[child]->v.i64) { + break; + } + } else { + if (child + 1 <= end && pList[child]->v.i64 >= pList[child + 1]->v.i64) { + child++; + } + + if (pList[parent]->v.i64 <= pList[child]->v.i64) { + break; + } + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (minRoot) { + if (child + 1 <= end && pList[child]->v.u64 < pList[child + 1]->v.u64) { + child++; + } + + if (pList[parent]->v.u64 > pList[child]->v.u64) { + break; + } + } else { + if (child + 1 <= end && pList[child]->v.u64 >= pList[child + 1]->v.u64) { + child++; + } + + if (pList[parent]->v.u64 <= pList[child]->v.u64) { + break; + } + } + } else { + if (minRoot) { + if (child + 1 <= end && pList[child]->v.dKey < pList[child + 1]->v.dKey) { + child++; + } + + if (pList[parent]->v.dKey > pList[child]->v.dKey) { + break; + } + } else { + if (child + 1 <= end && pList[child]->v.dKey >= pList[child + 1]->v.dKey) { + child++; + } + + if (pList[parent]->v.dKey <= pList[child]->v.dKey) { + break; + } + } + } + + heapSwap(pList[parent], pList[child], tagLen); + + parent = child; + child = parent * 2 + 1; + } +} + +void heapSort(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t len, bool minRoot) { + int32_t i; + + for (i = len / 2 - 1; i >= 0; i--) { + heapAdjust(pList, type, i, tagLen, len - 1, minRoot); + } + +/* + for (i = len - 1; i > 0; i--) { + heapSwap(pList[0], pList[i], tagsLen); + heapAdjust(pList, type, tagsLen, i - 1, minRoot); + } +*/ +} + static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; @@ -2210,59 +2305,17 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, assert(pList != NULL); if (pInfo->num < maxLen) { - if (pInfo->num == 0 || - (IS_SIGNED_NUMERIC_TYPE(type) && val.i64 >= pList[pInfo->num - 1]->v.i64) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 >= pList[pInfo->num - 1]->v.u64) || - (IS_FLOAT_TYPE(type) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) { - valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); - } else { - int32_t i = pInfo->num - 1; - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.i64 > val.i64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.u64 > val.u64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else { - while (i >= 0 && pList[i]->v.dKey > val.dKey) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } - - valuePairAssign(pList[i + 1], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage); - } - + valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + + heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 0); + pInfo->num++; } else { - int32_t i = 0; - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { - // find the appropriate the slot position - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.i64 < val.i64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.u64 < val.u64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } else { - while (i + 1 < maxLen && pList[i + 1]->v.dKey < val.dKey) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } - - valuePairAssign(pList[i], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 0); } } } @@ -2276,57 +2329,17 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa assert(pList != NULL); if (pInfo->num < maxLen) { - if (pInfo->num == 0) { - valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage); - } else { - int32_t i = pInfo->num - 1; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.i64 < val.i64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.u64 < val.u64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else { - while (i >= 0 && pList[i]->v.dKey < val.dKey) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } - - valuePairAssign(pList[i + 1], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); - } - + valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + + heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 1); + pInfo->num++; } else { - int32_t i = 0; - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { - // find the appropriate the slot position - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.i64 > val.i64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.u64 > val.u64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } else { - while (i + 1 < maxLen && pList[i + 1]->v.dKey > val.dKey) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } - - valuePairAssign(pList[i], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 1); } } } From 0ef76e0c1dba1203297e1b70c11922c5ebc27f9a Mon Sep 17 00:00:00 2001 From: xywang Date: Wed, 2 Jun 2021 05:50:11 +0000 Subject: [PATCH 02/43] [TD-2574]: fixed a bug in heapSort. --- src/query/src/qAggMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index a7106a14a9..b0329c845d 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2285,13 +2285,13 @@ void heapSort(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t len, bo int32_t i; for (i = len / 2 - 1; i >= 0; i--) { - heapAdjust(pList, type, i, tagLen, len - 1, minRoot); + heapAdjust(pList, type, tagLen, i, len - 1, minRoot); } /* for (i = len - 1; i > 0; i--) { heapSwap(pList[0], pList[i], tagsLen); - heapAdjust(pList, type, tagsLen, i - 1, minRoot); + heapAdjust(pList, type, 0, tagsLen, i - 1, minRoot); } */ } From e7753404758bf8c9a3896d4936ae4b0aab54a21e Mon Sep 17 00:00:00 2001 From: xywang Date: Sat, 19 Jun 2021 16:25:04 +0800 Subject: [PATCH 03/43] [TD-2574]: added general heapsort algorithm --- src/query/src/qAggMain.c | 124 ++++++++++----------------------------- src/util/inc/talgo.h | 33 +++++++++++ src/util/src/talgo.c | 86 +++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 94 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index b0329c845d..2c7242b26a 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2201,99 +2201,35 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ } while (0) -static void heapSwap(tValuePair *a, tValuePair *b, const int16_t tagLen) { - char tag[32768]; - tValuePair temp; +static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) +{ + uint16_t type = *(uint16_t *) param; + tValuePair *val1 = *(tValuePair **) p1; + tValuePair *val2 = *(tValuePair **) p2; + + if (IS_SIGNED_NUMERIC_TYPE(type)) { + return val1->v.i64 - val2->v.i64; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + return val1->v.u64 - val2->v.u64; + } + + return val1->v.dKey - val2->v.dKey; +} + +static void topBotSwapFn(void *dst, void *src, const void *param) +{ + char tag[32768]; + tValuePair temp; + uint16_t tagLen = *(uint16_t *) param; + tValuePair *vdst = *(tValuePair **) dst; + tValuePair *vsrc = *(tValuePair **) src; memset(tag, 0, sizeof(tag)); temp.pTags = tag; - VALUEPAIRASSIGN(&temp, a, tagLen); - VALUEPAIRASSIGN(a, b, tagLen); - VALUEPAIRASSIGN(b, &temp, tagLen); -} - -static void heapAdjust(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t start, int32_t end, bool minRoot) { - int32_t parent = start; - int32_t child = 2 * parent + 1; - - while (child <= end) { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - if (minRoot) { - if (child + 1 <= end && pList[child]->v.i64 < pList[child + 1]->v.i64) { - child++; - } - - if (pList[parent]->v.i64 > pList[child]->v.i64) { - break; - } - } else { - if (child + 1 <= end && pList[child]->v.i64 >= pList[child + 1]->v.i64) { - child++; - } - - if (pList[parent]->v.i64 <= pList[child]->v.i64) { - break; - } - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - if (minRoot) { - if (child + 1 <= end && pList[child]->v.u64 < pList[child + 1]->v.u64) { - child++; - } - - if (pList[parent]->v.u64 > pList[child]->v.u64) { - break; - } - } else { - if (child + 1 <= end && pList[child]->v.u64 >= pList[child + 1]->v.u64) { - child++; - } - - if (pList[parent]->v.u64 <= pList[child]->v.u64) { - break; - } - } - } else { - if (minRoot) { - if (child + 1 <= end && pList[child]->v.dKey < pList[child + 1]->v.dKey) { - child++; - } - - if (pList[parent]->v.dKey > pList[child]->v.dKey) { - break; - } - } else { - if (child + 1 <= end && pList[child]->v.dKey >= pList[child + 1]->v.dKey) { - child++; - } - - if (pList[parent]->v.dKey <= pList[child]->v.dKey) { - break; - } - } - } - - heapSwap(pList[parent], pList[child], tagLen); - - parent = child; - child = parent * 2 + 1; - } -} - -void heapSort(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t len, bool minRoot) { - int32_t i; - - for (i = len / 2 - 1; i >= 0; i--) { - heapAdjust(pList, type, tagLen, i, len - 1, minRoot); - } - -/* - for (i = len - 1; i > 0; i--) { - heapSwap(pList[0], pList[i], tagsLen); - heapAdjust(pList, type, 0, tagsLen, i - 1, minRoot); - } -*/ + VALUEPAIRASSIGN(&temp, vdst, tagLen); + VALUEPAIRASSIGN(vdst, vsrc, tagLen); + VALUEPAIRASSIGN(vsrc, &temp, tagLen); } static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, @@ -2303,11 +2239,11 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, tValuePair **pList = pInfo->res; assert(pList != NULL); - + if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 0); + taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); pInfo->num++; } else { @@ -2315,7 +2251,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 0); + taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); } } } @@ -2331,7 +2267,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 1); + taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); pInfo->num++; } else { @@ -2339,7 +2275,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 1); + taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); } } } diff --git a/src/util/inc/talgo.h b/src/util/inc/talgo.h index 9e3692225b..4aa5430605 100644 --- a/src/util/inc/talgo.h +++ b/src/util/inc/talgo.h @@ -34,6 +34,7 @@ typedef int (*__compar_fn_t) (const void *, const void *); #define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx)) typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void *param); +typedef void (*__ext_swap_fn_t)(void *p1, void *p2, const void *param); /** * quick sort, with the compare function requiring additional parameters support @@ -59,6 +60,38 @@ void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ex */ void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t fn, int flags); +/** + * adjust heap + * + * @param base: the start address of array + * @param size: size of every item in array + * @param start: the first index + * @param end: the last index + * @param parcompar: parameters for compare function + * @param compar: user defined compare function + * @param parswap: parameters for swap function + * @param swap: user defined swap function, the default swap function doswap will be used if swap is NULL + * @param maxroot: if heap is max root heap + * @return + */ +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); + +/** + * sort heap to make sure it is a max/min root heap + * + * @param base: the start address of array + * @param size: size of every item in array + * @param len: the length of array + * @param parcompar: parameters for compare function + * @param compar: user defined compare function + * @param parswap: parameters for swap function + * @param swap: user defined swap function, the default swap function doswap will be used if swap is NULL + * @param maxroot: if heap is max root heap + * @return + */ +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); + + #ifdef __cplusplus } #endif diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 278683539e..54b7e00eb7 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -225,3 +225,89 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return NULL; } + +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) +{ + int32_t parent; + int32_t child; + char *buf; + + if (base && size > 0 && compar) { + parent = start; + child = 2 * parent + 1; + + if (swap == NULL) { + buf = calloc(1, size); + if (buf == NULL) { + return; + } + } + + if (maxroot) { + while (child <= end) { + if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) < 0) { + child++; + } + + if ((*compar)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parcompar) > 0) { + break; + } + + if (swap == NULL) { + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); + } else { + (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); + } + + parent = child; + child = 2 * parent + 1; + } + } else { + while (child <= end) { + if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) > 0) { + child++; + } + + if ((*compar)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parcompar) < 0) { + break; + } + + if (swap == NULL) { + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); + } else { + (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); + } + + parent = child; + child = 2 * parent + 1; + } + } + + if (swap == NULL) { + tfree(buf); + } + } +} + +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) +{ + int32_t i; + + if (base && size > 0) { + for (i = len / 2 - 1; i >= 0; i--) { + taosheapadjust(base, size, i, len - 1, parcompar, compar, parswap, swap, maxroot); + } + } + +/* + char *buf = calloc(1, size); + + for (i = len - 1; i > 0; i--) { + doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); + taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); + } + + tfree(buf); +*/ +} + From fe3cc4facd7306153bf41587ed77874d2fe3c46b Mon Sep 17 00:00:00 2001 From: xywang Date: Mon, 28 Jun 2021 19:32:10 +0800 Subject: [PATCH 04/43] [TD-2574]: fixed compilation errors. --- src/query/src/qAggMain.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 2c7242b26a..c8f4cf8dbd 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2208,12 +2208,24 @@ static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) tValuePair *val2 = *(tValuePair **) p2; if (IS_SIGNED_NUMERIC_TYPE(type)) { - return val1->v.i64 - val2->v.i64; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - return val1->v.u64 - val2->v.u64; + if (val1->v.i64 == val2->v.i64) { + return 0; + } + + return (val1->v.i64 > val2->v.i64) ? 1 : -1; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (val1->v.u64 == val2->v.u64) { + return 0; + } + + return (val1->v.u64 > val2->v.u64) ? 1 : -1; } - return val1->v.dKey - val2->v.dKey; + if (val1->v.dKey == val2->v.dKey) { + return 0; + } + + return (val1->v.dKey > val2->v.dKey) ? 1 : -1; } static void topBotSwapFn(void *dst, void *src, const void *param) From c4f06ca30b1a37b42b60a89285fbceba21f2cde8 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 6 Jul 2021 18:05:27 +0800 Subject: [PATCH 05/43] develop schemaless --- src/client/src/tscParseLineProtocol.c | 731 ++++++++++++++++++++++++++ 1 file changed, 731 insertions(+) create mode 100644 src/client/src/tscParseLineProtocol.c diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c new file mode 100644 index 0000000000..79e8955f9a --- /dev/null +++ b/src/client/src/tscParseLineProtocol.c @@ -0,0 +1,731 @@ +#include +#include +#include +#include +#include "os.h" +#include "osString.h" +#include "ttype.h" +#include "tmd5.h" +#include "tstrbuild.h" +#include "tname.h" +#include "taos.h" +#include "tsclient.h" +#include "tscLog.h" +#include "hash.h" +#include "tskiplist.h" +#include "tscUtil.h" + +typedef enum { + LP_ITEM_TAG, + LP_ITEM_FIELD +} LPItemKind; + +typedef struct { + SStrToken key; + SStrToken value; + + char name[TSDB_COL_NAME_LEN]; + int8_t type; + int16_t bytes; + + char* payload; +}SLPItem; + +typedef struct { + SStrToken measToken; + SStrToken tsToken; + + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; + SArray* fields; + int64_t ts; + +} SLPPoint; + +typedef enum { + LP_MEASUREMENT, + LP_TAG_KEY, + LP_TAG_VALUE, + LP_FIELD_KEY, + LP_FIELD_VALUE +} LPPart; + +int32_t scanToCommaOrSpace(SStrToken s, int32_t start, int32_t* index, LPPart part) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == ',' || s.z[i] == ' ') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t scanToEqual(SStrToken s, int32_t start, int32_t* index) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == '=') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t setPointMeasurement(SLPPoint* point, SStrToken token) { + point->measToken = token; + if (point->measToken.n < TSDB_TABLE_NAME_LEN) { + strncpy(point->sTableName, point->measToken.z, point->measToken.n); + point->sTableName[point->measToken.n] = '\0'; + } + return 0; +} + +int32_t setItemKey(SLPItem* item, SStrToken key, LPPart part) { + item->key = key; + if (item->key.n < TSDB_COL_NAME_LEN) { + strncpy(item->name, item->key.z, item->key.n); + item->name[item->key.n] = '\0'; + } + return 0; +} + +int32_t setItemValue(SLPItem* item, SStrToken value, LPPart part) { + item->value = value; + return 0; +} + +int32_t parseItemValue(SLPItem* item, LPItemKind kind) { + char* sv = item->value.z; + char* last = item->value.z + item->value.n - 1; + + if (isdigit(sv[0]) || sv[0] == '-') { + if (*last == 'i') { + item->type = TSDB_DATA_TYPE_BIGINT; + item->bytes = (int16_t)tDataTypes[item->type].bytes; + item->payload = malloc(item->bytes); + char* endptr = NULL; + *(item->payload) = strtoll(sv, &endptr, 10); + } else { + item->type = TSDB_DATA_TYPE_DOUBLE; + item->bytes = (int16_t)tDataTypes[item->type].bytes; + item->payload = malloc(item->bytes); + char* endptr = NULL; + *(item->payload) = strtold(sv, &endptr); + } + } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { + if (sv[0] == 'L') { + item->type = TSDB_DATA_TYPE_NCHAR; + uint32_t bytes = item->value.n - 3; +// uint32_t len = bytes; +// char* ucs = malloc(len); +// int32_t ncharBytes = 0; +// taosMbsToUcs4(sv+2, len, ucs, len, &ncharBytes); +// item->bytes = ncharBytes; +// item->payload = malloc(ncharBytes); +// memcpy(item->payload, ucs, ncharBytes); +// free(ucs); + item->bytes = bytes; + item->payload = malloc(bytes); + memcpy(item->payload, sv+1, bytes); + } else if (sv[0]=='"'){ + item->type = TSDB_DATA_TYPE_BINARY; + uint32_t bytes = item->value.n - 2; + item->bytes = bytes; + item->payload = malloc(bytes); + memcpy(item->payload, sv+1, bytes); + } + } else if (sv[0] == 't' || sv[0] == 'f' || sv[0]=='T' || sv[0] == 'F') { + item->type = TSDB_DATA_TYPE_BOOL; + item->bytes = tDataTypes[item->type].bytes; + item->payload = malloc(tDataTypes[item->type].bytes); + *(item->payload) = tolower(sv[0])=='t' ? true : false; + } + return 0; +} + +int32_t compareLPItemKey(const void* p1, const void* p2) { + const SLPItem* t1 = p1; + const SLPItem* t2 = p2; + uint32_t min = (t1->key.n < t2->key.n) ? t1->key.n : t2->key.n; + int res = strncmp(t1->key.z, t2->key.z, min); + if (res != 0) { + return res; + } else { + return (int)(t1->key.n) - (int)(t2->key.n); + } +} + +int32_t setPointTimeStamp(SLPPoint* point, SStrToken tsToken) { + point->tsToken = tsToken; + return 0; +} + +int32_t parsePointTime(SLPPoint* point) { + if (point->tsToken.n <= 0) { + point->ts = taosGetTimestampNs(); + } else { + char* endptr = NULL; + point->ts = strtoll(point->tsToken.z, &endptr, 10); + } + return 0; +} + +int32_t tscParseLine(SStrToken line, SLPPoint* point) { + int32_t pos = 0; + + int32_t start = 0; + int32_t err = scanToCommaOrSpace(line, start, &pos, LP_MEASUREMENT); + if (err != 0) { + tscError("a"); + return err; + } + + SStrToken measurement = {.z = line.z+start, .n = pos-start}; + setPointMeasurement(point, measurement); + point->tags = taosArrayInit(64, sizeof(SLPItem)); + start = pos + 1; + while (line.z[start] == ',') { + SLPItem item; + + err = scanToEqual(line, start, &pos); + if (err != 0) { + tscError("b"); + goto error; + } + + SStrToken tagKey = {.z = line.z + start, .n = pos-start}; + setItemKey(&item, tagKey, LP_TAG_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_TAG_VALUE); + if (err != 0) { + tscError("c"); + goto error; + } + + SStrToken tagValue = {.z = line.z + start, .n = pos-start}; + setItemValue(&item, tagValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_TAG); + taosArrayPush(point->tags, &item); + + start = pos + 1; + } + + taosArraySort(point->tags, compareLPItemKey); + + point->fields = taosArrayInit(64, sizeof(SLPItem)); + do { + SLPItem item; + err = scanToEqual(line, start, &pos); + if (err != 0) { + goto error; + } + SStrToken fieldKey = {.z = line.z + start, .n = pos- start}; + setItemKey(&item, fieldKey, LP_FIELD_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_FIELD_VALUE); + if (err != 0) { + goto error; + } + SStrToken fieldValue = {.z = line.z + start, .n = pos - start}; + setItemValue(&item, fieldValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_FIELD); + taosArrayPush(point->fields, &item); + + start = pos + 1; + } while (line.z[pos] == ','); + + taosArraySort(point->fields, compareLPItemKey); + + SStrToken tsToken = {.z = line.z+start, .n = line.n-start}; + setPointTimeStamp(point, tsToken); + parsePointTime(point); + + goto done; + +error: + // free array + return err; +done: + return 0; +} + + +int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { + for (int32_t i = 0; i < numLines; ++i) { + SStrToken tkLine = {.z = lines[i], .n = strlen(lines[i])+1}; + SLPPoint point; + tscParseLine(tkLine, &point); + taosArrayPush(points, &point); + } + return 0; +} + +TAOS_RES* taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { + SArray* points = taosArrayInit(numLines, sizeof(SLPPoint)); + tscParseLines(lines, numLines, points, NULL); + + + return NULL; +} +//================================================================================================= + +typedef struct { + char* key; + uint8_t type; + int16_t length; + char* value; +} TAOS_SML_KV; + +typedef struct { + char* stableName; + + char* childTableName; + TAOS_SML_KV* tags; + int tagNum; + + // first kv must be timestamp + TAOS_SML_KV* fields; + int fieldNum; + +} TAOS_SML_DATA_POINT; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SHashObj* tagHash; + SHashObj* fieldHash; + SArray* tags; //SArray + SArray* fields; //SArray +} SSmlSTableSchema; + + +int compareSmlColKv(const void* p1, const void* p2) { + TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; + TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; + int kvLen1 = (int)strlen(kv1->key); + int kvLen2 = (int)strlen(kv2->key); + int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} + +int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + taosStringBuilderAppendString(&sb, point->stableName); + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + taosStringBuilderAppendChar(&sb, ','); + taosStringBuilderAppendString(&sb, tagKv->key); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + return 0; +} + +int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { + int32_t code = 0; + + STscObj *pObj = (STscObj *)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + char sql[256]; + snprintf(sql, 256, "describe %s", tableName); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + taos_free_result(res); + return code; + } + taos_free_result(res); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + SStrToken tableToken = {.z=tableName, .n=strlen(tableName), .type=TK_ID}; + tGetToken(tableName, &tableToken.type); + // Check if the table name available or not + if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + return code; + } + + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + + if (code != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); + return code; + } + + tscFreeSqlObj(pSql); + + + uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* tableMeta = calloc(1, size); + taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); + + tstrncpy(schema->sTableName, tableName, strlen(tableName)); + for (int i=0; itableInfo.numOfColumns; ++i) { + SSchema field; + tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)); + field.type = tableMeta->schema[i].type; + field.bytes = tableMeta->schema[i].bytes; + SSchema* pField = taosArrayPush(schema->fields, &field); + taosHashPut(schema->fieldHash, field.name, strlen(field.name), &pField, POINTER_BYTES); + } + + for (int i=0; itableInfo.numOfTags; ++i) { + int j = i + tableMeta->tableInfo.numOfColumns; + SSchema field; + tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)); + field.type = tableMeta->schema[j].type; + field.bytes = tableMeta->schema[j].bytes; + SSchema* pField = taosArrayPush(schema->tags, &field); + taosHashPut(schema->tagHash, field.name, strlen(field.name), &pField, POINTER_BYTES); + } + + return code; + +} + +typedef enum { + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, + SCHEMA_ACTION_CREATE_CTABLE +} ESchemaAction; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; //SArray + SArray* fields; //SArray +} SCreateSTableActionInfo; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SSchema* field; +} SAlterSTableActionInfo; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + char cTableName[TSDB_TABLE_NAME_LEN]; + TAOS_SML_KV* tags; + int tagNum; +} SCreateCTableActionInfo; + +typedef struct { + ESchemaAction action; + union { + SCreateSTableActionInfo createSTable; + SAlterSTableActionInfo alterSTable; + SCreateCTableActionInfo createCTable; + }; +} SSchemaAction; + +int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { + if (!IS_VAR_DATA_TYPE(kv->type)) { + *bytes = tDataTypes[kv->type].bytes; + } else { + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + char* ucs = malloc(kv->length * TSDB_NCHAR_SIZE + 1); + int32_t bytesNeeded = 0; + //todo check conversion succeed + taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); + free(ucs); + *bytes = bytesNeeded + VARSTR_HEADER_SIZE; + + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + *bytes = kv->length + VARSTR_HEADER_SIZE; + } + } + return 0; +} + +int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array) { + SSchema* pField = NULL; + SSchema** ppField = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); + if (ppField) { + pField = *ppField; + + if (pField->type != smlKv->type) { + //TODO: + tscError("type mismatch"); + return -1; + } + + int32_t bytes = 0; + getFieldBytesFromSmlKv(smlKv, &bytes); + pField->bytes = MAX(pField->bytes, bytes); + + } else { + SSchema field; + size_t tagKeyLen = strlen(smlKv->key); + strncpy(field.name, smlKv->key, tagKeyLen); + field.name[tagKeyLen] = '\0'; + field.type = smlKv->type; + + int32_t bytes = 0; + getFieldBytesFromSmlKv(smlKv, &bytes); + field.bytes = bytes; + + pField = taosArrayPush(array, &field); + taosHashPut(hash, field.name, tagKeyLen, &pField, POINTER_BYTES); + } + return 0; +} + +int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool isTag, char sTableName[], + SSchemaAction* action, bool* actionNeeded) { + SSchema** ppDbAttr = taosHashGet(dbAttrHash, pointColField->name, strlen(pointColField->name)); + if (*ppDbAttr) { + SSchema* dbAttr = *ppDbAttr; + if (pointColField->type != dbAttr->type) { + //todo error + return -5; + } + + if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { + if (isTag) { + action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + } else { + if (isTag) { + action->action = SCHEMA_ACTION_ADD_TAG; + } else { + action->action = SCHEMA_ACTION_ADD_COLUMN; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + return 0; +} + +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray + SHashObj* sname2shema = taosHashInit(32, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int i = 0; i < numPoint; ++i) { + TAOS_SML_DATA_POINT* point = &points[i]; + SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); + SSmlSTableSchema* pStableSchema = NULL; + if (ppStableSchema) { + pStableSchema= *ppStableSchema; + } else { + SSmlSTableSchema schema; + size_t stableNameLen = strlen(point->stableName); + strncpy(schema.sTableName, point->stableName, stableNameLen); + schema.sTableName[stableNameLen] = '\0'; + schema.fields = taosArrayInit(64, sizeof(SSchema)); + schema.tags = taosArrayInit(8, sizeof(SSchema)); + schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + pStableSchema = taosArrayPush(stableArray, &schema); + taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); + } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + addTaosFieldToHashAndArray(tagKv, pStableSchema->tagHash, pStableSchema->tags); + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* fieldKv = point->fields + j; + addTaosFieldToHashAndArray(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); + } + } + + SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); + size_t numStable = taosArrayGetSize(stableArray); + for (int i = 0; i < numStable; ++i) { + SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); + SSmlSTableSchema dbSchema = {0}; + dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); + dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); + dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); + memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); + schemaAction.createSTable.tags = pointSchema->tags; + schemaAction.createSTable.fields = pointSchema->fields; + taosArrayPush(schemaActions, &schemaAction); + }else if (code == TSDB_CODE_SUCCESS) { + size_t pointTagSize = taosArrayGetSize(pointSchema->tags); + size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); + + SHashObj* dbTagHash = dbSchema.tagHash; + SHashObj* dbFieldHash = dbSchema.fieldHash; + + for (int j = 0; j < pointTagSize; ++j) { + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + + for (int j = 0; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + } else { + return code; + } + } + + return code; +} + + +int32_t buildColumnDescription(SSchema* field, + char* buf, int32_t bufSize, int32_t* outBytes) { + uint8_t type = field->type; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; + if (type == TSDB_DATA_TYPE_NCHAR) { + bytes = bytes/TSDB_NCHAR_SIZE; + } + int out = snprintf(buf, bufSize,"%s %s(%d)", + field->name,tDataTypes[field->type].name, bytes); + *outBytes = out; + } else { + int out = snprintf(buf, bufSize, "%s %s", + field->name, tDataTypes[type].name); + *outBytes = out; + } + + return 0; +} + +int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { + int32_t code = 0; + int32_t capacity = TSDB_MAX_BINARY_LEN; + int32_t outBytes = 0; + char *result = (char *)calloc(1, capacity); + + switch (action->action) { + case SCHEMA_ACTION_ADD_COLUMN: { + int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_ADD_TAG: { + int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, + result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { + int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + } + case SCHEMA_ACTION_CHANGE_TAG_SIZE: { + int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_CREATE_STABLE: { + int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); + char* pos = result + n; int freeBytes = capacity - n; + int numCols = taosArrayGetSize(action->createSTable.fields); + for (int32_t i = 0; i < numCols; ++i) { + SSchema* field = taosArrayGet(action->createSTable.fields, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + --pos; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ") tags ("); + int numTags = taosArrayGetSize(action->createSTable.tags); + pos += outBytes; freeBytes -= outBytes; + for (int32_t i = 0; i < numTags; ++i) { + SSchema* field = taosArrayGet(action->createSTable.tags, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + pos--; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ")"); + TAOS_RES* res = taos_query(taos, result); + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_CREATE_CTABLE: { + + break; + } + default: + break; + } + free(result); + return code; +} + +//todo: table/column length check +//todo: type check +//todo: taosmbs2ucs4 check From d5ab8d7ced3a76013cfc64419ec3bf970d3a822b Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 9 Jul 2021 19:37:15 +0800 Subject: [PATCH 06/43] before taos_bind/taos_multi_bind generation --- src/client/src/tscParseLineProtocol.c | 99 +++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 79e8955f9a..10d2429b5e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -277,6 +277,9 @@ typedef struct { uint8_t type; int16_t length; char* value; + + //=================================== + SSchema* fieldSchema; } TAOS_SML_KV; typedef struct { @@ -415,7 +418,6 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { } return code; - } typedef enum { @@ -465,7 +467,6 @@ int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); free(ucs); *bytes = bytesNeeded + VARSTR_HEADER_SIZE; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { *bytes = kv->length + VARSTR_HEADER_SIZE; } @@ -503,6 +504,9 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a pField = taosArrayPush(array, &field); taosHashPut(hash, field.name, tagKeyLen, &pField, POINTER_BYTES); } + + smlKv->fieldSchema = pField; + return 0; } @@ -700,9 +704,11 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { *pos = ','; ++pos; --freeBytes; } --pos; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ") tags ("); - int numTags = taosArrayGetSize(action->createSTable.tags); pos += outBytes; freeBytes -= outBytes; + + int numTags = taosArrayGetSize(action->createSTable.tags); for (int32_t i = 0; i < numTags; ++i) { SSchema* field = taosArrayGet(action->createSTable.tags, i); buildColumnDescription(field, pos, freeBytes, &outBytes); @@ -716,7 +722,43 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { break; } case SCHEMA_ACTION_CREATE_CTABLE: { - +// SCreateCTableActionInfo* pInfo = &action->createCTable; +// SArray* bindParams = taosArrayInit(2 + 2 * pInfo->tagNum, sizeof(TAOS_BIND)); +// outBytes = sprintf(result, "create table ? using ?("); +// char* pos = result + outBytes; int32_t freeBytes = capacity-outBytes; +// uintptr_t lenSTableName = strlen(pInfo->sTableName); +// uintptr_t lenCTableName = strlen(pInfo->cTableName); +// TAOS_BIND tbCTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, +// .buffer = pInfo->cTableName, .length = &lenCTableName}; +// TAOS_BIND tbSTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, +// .buffer = pInfo->sTableName, .length = &lenSTableName}; +// taosArrayPush(bindParams, &tbCTableName); +// taosArrayPush(bindParams, &tbSTableName); +// for (int32_t i = 0; i < pInfo->tagNum; ++i) { +// outBytes = snprintf(pos, freeBytes, "?,"); +// +// TAOS_SML_KV* tagKv = pInfo->tags + i; +// TAOS_BIND tbTag = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, +// .buffer = tagKv->key, .length = }; +// pos += outBytes; freeBytes -= outBytes; +// } +// --pos; ++freeBytes; +// +// outBytes = snprintf(pos, freeBytes, ") tags ("); +// pos += outBytes; freeBytes -= outBytes; +// for (int32_t i = 0; i < pInfo->tagNum; ++i) { +// TAOS_SML_KV* tagKv = pInfo->tags + i; +// outBytes = snprintf(pos, freeBytes, "?,"); +// pos += outBytes; freeBytes -= outBytes; +// } +// pos--; ++freeBytes; +// outBytes = snprintf(pos, freeBytes, ")"); +// +// TAOS_STMT* stmt = taos_stmt_init(taos); +// taos_stmt_prepare(stmt, result, strlen(result)); +// +// +// taos_stmt_bind_param(stmt, (TAOS_BIND*)bindParams); break; } default: @@ -726,6 +768,55 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } +int32_t transformIntoPreparedStatement(SArray* points) { + size_t numPoints = taosArrayGetSize(points); + +// SHashObj* tag2bind = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +// SHashObj* field2multiBind = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int32_t i = 0; i < numPoints; ++i) { + TAOS_SML_DATA_POINT * point = taosArrayGet(points, i); + char tableKey[256]; + snprintf(tableKey, 256, "%s.%s", point->stableName, point->childTableName); + + } + return 0; +} + +int32_t insertBatch(TAOS* taos, const char* sTableName, char* cTableName, SSchema* tagsSchema, int numTags, TAOS_BIND* tagBind, + SSchema* colsSchema, int numCols, TAOS_MULTI_BIND* colBind) { + TAOS_STMT* stmt = taos_stmt_init(taos); + + char result[TSDB_MAX_BINARY_LEN] = {0}; + sprintf(result, "insert into ? using %s(", sTableName); + for (int i = 0; i < numTags; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", tagsSchema[i].name); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") tags ("); + + for (int i = 0; i < numTags; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", colsSchema[i].name); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ")"); + + int32_t code = 0; + code = taos_stmt_prepare(stmt, result, strlen(result)); + + code = taos_stmt_set_tbname_tags(stmt, cTableName, tagBind); + code = taos_stmt_bind_param_batch(stmt, colBind); + code = taos_stmt_execute(stmt); + return code; +} //todo: table/column length check //todo: type check //todo: taosmbs2ucs4 check From 12e8b038d176b8c0ded55fced17185d531c61cac Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 10 Jul 2021 16:18:38 +0800 Subject: [PATCH 07/43] before debugging --- src/client/src/tscParseLineProtocol.c | 393 ++++++++++++++------------ 1 file changed, 211 insertions(+), 182 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 10d2429b5e..2738fe6f7a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -272,6 +272,14 @@ TAOS_RES* taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { } //================================================================================================= +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SHashObj* tagHash; + SHashObj* fieldHash; + SArray* tags; //SArray + SArray* fields; //SArray +} SSmlSTableSchema; + typedef struct { char* key; uint8_t type; @@ -279,7 +287,7 @@ typedef struct { char* value; //=================================== - SSchema* fieldSchema; + SSchema* schema; } TAOS_SML_KV; typedef struct { @@ -293,17 +301,10 @@ typedef struct { TAOS_SML_KV* fields; int fieldNum; + //================================ + SSmlSTableSchema* schema; } TAOS_SML_DATA_POINT; -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SHashObj* tagHash; - SHashObj* fieldHash; - SArray* tags; //SArray - SArray* fields; //SArray -} SSmlSTableSchema; - - int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -426,7 +427,6 @@ typedef enum { SCHEMA_ACTION_ADD_TAG, SCHEMA_ACTION_CHANGE_COLUMN_SIZE, SCHEMA_ACTION_CHANGE_TAG_SIZE, - SCHEMA_ACTION_CREATE_CTABLE } ESchemaAction; typedef struct { @@ -440,19 +440,11 @@ typedef struct { SSchema* field; } SAlterSTableActionInfo; -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - char cTableName[TSDB_TABLE_NAME_LEN]; - TAOS_SML_KV* tags; - int tagNum; -} SCreateCTableActionInfo; - typedef struct { ESchemaAction action; union { SCreateSTableActionInfo createSTable; SAlterSTableActionInfo alterSTable; - SCreateCTableActionInfo createCTable; }; } SSchemaAction; @@ -505,7 +497,7 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a taosHashPut(hash, field.name, tagKeyLen, &pField, POINTER_BYTES); } - smlKv->fieldSchema = pField; + smlKv->schema = pField; return 0; } @@ -545,96 +537,6 @@ int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool return 0; } -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray - SHashObj* sname2shema = taosHashInit(32, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - for (int i = 0; i < numPoint; ++i) { - TAOS_SML_DATA_POINT* point = &points[i]; - SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); - SSmlSTableSchema* pStableSchema = NULL; - if (ppStableSchema) { - pStableSchema= *ppStableSchema; - } else { - SSmlSTableSchema schema; - size_t stableNameLen = strlen(point->stableName); - strncpy(schema.sTableName, point->stableName, stableNameLen); - schema.sTableName[stableNameLen] = '\0'; - schema.fields = taosArrayInit(64, sizeof(SSchema)); - schema.tags = taosArrayInit(8, sizeof(SSchema)); - schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - pStableSchema = taosArrayPush(stableArray, &schema); - taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); - } - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - addTaosFieldToHashAndArray(tagKv, pStableSchema->tagHash, pStableSchema->tags); - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* fieldKv = point->fields + j; - addTaosFieldToHashAndArray(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); - } - } - - SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); - size_t numStable = taosArrayGetSize(stableArray); - for (int i = 0; i < numStable; ++i) { - SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); - SSmlSTableSchema dbSchema = {0}; - dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); - dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); - dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { - SSchemaAction schemaAction = {0}; - schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; - memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); - memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); - schemaAction.createSTable.tags = pointSchema->tags; - schemaAction.createSTable.fields = pointSchema->fields; - taosArrayPush(schemaActions, &schemaAction); - }else if (code == TSDB_CODE_SUCCESS) { - size_t pointTagSize = taosArrayGetSize(pointSchema->tags); - size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); - - SHashObj* dbTagHash = dbSchema.tagHash; - SHashObj* dbFieldHash = dbSchema.fieldHash; - - for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); - if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); - } - } - - for (int j = 0; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); - if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); - } - } - } else { - return code; - } - } - - return code; -} - - int32_t buildColumnDescription(SSchema* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; @@ -721,46 +623,7 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { code = taos_errno(res); break; } - case SCHEMA_ACTION_CREATE_CTABLE: { -// SCreateCTableActionInfo* pInfo = &action->createCTable; -// SArray* bindParams = taosArrayInit(2 + 2 * pInfo->tagNum, sizeof(TAOS_BIND)); -// outBytes = sprintf(result, "create table ? using ?("); -// char* pos = result + outBytes; int32_t freeBytes = capacity-outBytes; -// uintptr_t lenSTableName = strlen(pInfo->sTableName); -// uintptr_t lenCTableName = strlen(pInfo->cTableName); -// TAOS_BIND tbCTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, -// .buffer = pInfo->cTableName, .length = &lenCTableName}; -// TAOS_BIND tbSTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, -// .buffer = pInfo->sTableName, .length = &lenSTableName}; -// taosArrayPush(bindParams, &tbCTableName); -// taosArrayPush(bindParams, &tbSTableName); -// for (int32_t i = 0; i < pInfo->tagNum; ++i) { -// outBytes = snprintf(pos, freeBytes, "?,"); -// -// TAOS_SML_KV* tagKv = pInfo->tags + i; -// TAOS_BIND tbTag = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, -// .buffer = tagKv->key, .length = }; -// pos += outBytes; freeBytes -= outBytes; -// } -// --pos; ++freeBytes; -// -// outBytes = snprintf(pos, freeBytes, ") tags ("); -// pos += outBytes; freeBytes -= outBytes; -// for (int32_t i = 0; i < pInfo->tagNum; ++i) { -// TAOS_SML_KV* tagKv = pInfo->tags + i; -// outBytes = snprintf(pos, freeBytes, "?,"); -// pos += outBytes; freeBytes -= outBytes; -// } -// pos--; ++freeBytes; -// outBytes = snprintf(pos, freeBytes, ")"); -// -// TAOS_STMT* stmt = taos_stmt_init(taos); -// taos_stmt_prepare(stmt, result, strlen(result)); -// -// -// taos_stmt_bind_param(stmt, (TAOS_BIND*)bindParams); - break; - } + default: break; } @@ -768,55 +631,221 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } -int32_t transformIntoPreparedStatement(SArray* points) { - size_t numPoints = taosArrayGetSize(points); +int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsSchema, char* result, int16_t freeBytes) { + size_t numTags = taosArrayGetSize(tagsSchema); + size_t numCols = taosArrayGetSize(colsSchema); + sprintf(result, "insert into ? using %s(", sTableName); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") tags ("); -// SHashObj* tag2bind = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); -// SHashObj* field2multiBind = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + for (int i = 0; i < numTags; ++i) { + snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") ("); + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", colSchema->name); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); + return 0; +} + +int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* rowsBind) { + TAOS_STMT* stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, sql, strlen(sql)); + + taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + size_t rows = taosArrayGetSize(rowsBind); + for (int32_t i = 0; i < rows; ++i) { + TAOS_BIND* colBind = taosArrayGetP(rowsBind, i); + taos_stmt_bind_param(stmt, colBind); + taos_stmt_add_batch(stmt); + } + + taos_stmt_execute(stmt); + TAOS_RES* res = taos_stmt_use_result(stmt); + return taos_errno(res); +} + +int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) { + SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); for (int32_t i = 0; i < numPoints; ++i) { - TAOS_SML_DATA_POINT * point = taosArrayGet(points, i); - char tableKey[256]; - snprintf(tableKey, 256, "%s.%s", point->stableName, point->childTableName); + TAOS_SML_DATA_POINT * point = points + i; + if (!point->childTableName) { + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen; + getChildTableName(point, childTableName, &tableNameLen); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + } + SArray* cTablePoints = NULL; + SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); + if (pCTablePoints) { + cTablePoints = *pCTablePoints; + } else { + cTablePoints = taosArrayInit(64, sizeof(point)); + taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); + } + taosArrayPush(cTablePoints, point); + } + SArray** pCTablePoints = taosHashIterate(cname2points, NULL); + while (pCTablePoints) { + SArray* cTablePoints = *pCTablePoints; + TAOS_SML_DATA_POINT * point = taosArrayGet(cTablePoints, 0); + int32_t numTags = taosArrayGetSize(point->schema->tags); + int32_t numCols = taosArrayGetSize(point->schema->fields); + char* stableName = point->stableName; + char* ctableName = point->childTableName; + char sql[TSDB_MAX_BINARY_LEN]; + getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); + + size_t rows = taosArrayGetSize(cTablePoints); + SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); + SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); + + for (int i = 0; i < rows; ++i) { + point = taosArrayGet(cTablePoints, i); + + taosArraySetSize(tagBinds, numTags); + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); + TAOS_BIND* bind = taosArrayGet(tagBinds, idx); + bind->buffer_type = kv->type; + bind->length = (uintptr_t*)&kv->length; + bind->buffer = kv->value; + } + + SArray* colBinds = taosArrayInit(numCols, sizeof(TAOS_BIND)); + taosArraySetSize(colBinds, numCols); + for (int j = 0; jfieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); + TAOS_BIND* bind = taosArrayGet(colBinds, idx); + bind->buffer_type = kv->type; + bind->length = (uintptr_t*)&kv->length; + bind->buffer = kv->value; + } + taosArrayPush(rowsBind, &colBinds); + } + + insertBatch(taos, sql, ctableName, tagBinds, rowsBind); + + pCTablePoints = taosHashIterate(cname2points, pCTablePoints); } return 0; } -int32_t insertBatch(TAOS* taos, const char* sTableName, char* cTableName, SSchema* tagsSchema, int numTags, TAOS_BIND* tagBind, - SSchema* colsSchema, int numCols, TAOS_MULTI_BIND* colBind) { - TAOS_STMT* stmt = taos_stmt_init(taos); - char result[TSDB_MAX_BINARY_LEN] = {0}; - sprintf(result, "insert into ? using %s(", sTableName); - for (int i = 0; i < numTags; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", tagsSchema[i].name); +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray + SHashObj* sname2shema = taosHashInit(32, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int i = 0; i < numPoint; ++i) { + TAOS_SML_DATA_POINT* point = &points[i]; + SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); + SSmlSTableSchema* pStableSchema = NULL; + if (ppStableSchema) { + pStableSchema= *ppStableSchema; + } else { + SSmlSTableSchema schema; + size_t stableNameLen = strlen(point->stableName); + strncpy(schema.sTableName, point->stableName, stableNameLen); + schema.sTableName[stableNameLen] = '\0'; + schema.fields = taosArrayInit(64, sizeof(SSchema)); + schema.tags = taosArrayInit(8, sizeof(SSchema)); + schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + pStableSchema = taosArrayPush(stableArray, &schema); + taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); + } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + addTaosFieldToHashAndArray(tagKv, pStableSchema->tagHash, pStableSchema->tags); + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* fieldKv = point->fields + j; + addTaosFieldToHashAndArray(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); + } + + point->schema = pStableSchema; } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") tags ("); - for (int i = 0; i < numTags; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); + SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); + size_t numStable = taosArrayGetSize(stableArray); + for (int i = 0; i < numStable; ++i) { + SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); + SSmlSTableSchema dbSchema = {0}; + dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); + dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); + dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); + memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); + schemaAction.createSTable.tags = pointSchema->tags; + schemaAction.createSTable.fields = pointSchema->fields; + taosArrayPush(schemaActions, &schemaAction); + }else if (code == TSDB_CODE_SUCCESS) { + size_t pointTagSize = taosArrayGetSize(pointSchema->tags); + size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); + + SHashObj* dbTagHash = dbSchema.tagHash; + SHashObj* dbFieldHash = dbSchema.fieldHash; + + for (int j = 0; j < pointTagSize; ++j) { + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + + for (int j = 0; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + } else { + return code; + } } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") ("); - for (int i = 0; i < numCols; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", colsSchema[i].name); + for (int i = 0; i < taosArrayGetSize(schemaActions); ++i) { + SSchemaAction* action = taosArrayGet(schemaActions, i); + applySchemaAction(taos, action); } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") values ("); - for (int i = 0; i < numCols; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); - } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ")"); - - int32_t code = 0; - code = taos_stmt_prepare(stmt, result, strlen(result)); - - code = taos_stmt_set_tbname_tags(stmt, cTableName, tagBind); - code = taos_stmt_bind_param_batch(stmt, colBind); - code = taos_stmt_execute(stmt); + insertPoints(taos, points, numPoint); return code; } + + //todo: table/column length check //todo: type check //todo: taosmbs2ucs4 check From 2d0005d3f179b3a4f734fd75dcd104058537a3a6 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 11 Jul 2021 23:52:19 +0800 Subject: [PATCH 08/43] before getChildTableName and insertBatch --- src/client/src/tscParseLineProtocol.c | 758 +++++++++++++++----------- src/inc/taos.h | 2 + tests/examples/c/apitest.c | 15 + 3 files changed, 447 insertions(+), 328 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 2738fe6f7a..3a88d2e906 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -15,263 +15,6 @@ #include "tskiplist.h" #include "tscUtil.h" -typedef enum { - LP_ITEM_TAG, - LP_ITEM_FIELD -} LPItemKind; - -typedef struct { - SStrToken key; - SStrToken value; - - char name[TSDB_COL_NAME_LEN]; - int8_t type; - int16_t bytes; - - char* payload; -}SLPItem; - -typedef struct { - SStrToken measToken; - SStrToken tsToken; - - char sTableName[TSDB_TABLE_NAME_LEN]; - SArray* tags; - SArray* fields; - int64_t ts; - -} SLPPoint; - -typedef enum { - LP_MEASUREMENT, - LP_TAG_KEY, - LP_TAG_VALUE, - LP_FIELD_KEY, - LP_FIELD_VALUE -} LPPart; - -int32_t scanToCommaOrSpace(SStrToken s, int32_t start, int32_t* index, LPPart part) { - for (int32_t i = start; i < s.n; ++i) { - if (s.z[i] == ',' || s.z[i] == ' ') { - *index = i; - return 0; - } - } - return -1; -} - -int32_t scanToEqual(SStrToken s, int32_t start, int32_t* index) { - for (int32_t i = start; i < s.n; ++i) { - if (s.z[i] == '=') { - *index = i; - return 0; - } - } - return -1; -} - -int32_t setPointMeasurement(SLPPoint* point, SStrToken token) { - point->measToken = token; - if (point->measToken.n < TSDB_TABLE_NAME_LEN) { - strncpy(point->sTableName, point->measToken.z, point->measToken.n); - point->sTableName[point->measToken.n] = '\0'; - } - return 0; -} - -int32_t setItemKey(SLPItem* item, SStrToken key, LPPart part) { - item->key = key; - if (item->key.n < TSDB_COL_NAME_LEN) { - strncpy(item->name, item->key.z, item->key.n); - item->name[item->key.n] = '\0'; - } - return 0; -} - -int32_t setItemValue(SLPItem* item, SStrToken value, LPPart part) { - item->value = value; - return 0; -} - -int32_t parseItemValue(SLPItem* item, LPItemKind kind) { - char* sv = item->value.z; - char* last = item->value.z + item->value.n - 1; - - if (isdigit(sv[0]) || sv[0] == '-') { - if (*last == 'i') { - item->type = TSDB_DATA_TYPE_BIGINT; - item->bytes = (int16_t)tDataTypes[item->type].bytes; - item->payload = malloc(item->bytes); - char* endptr = NULL; - *(item->payload) = strtoll(sv, &endptr, 10); - } else { - item->type = TSDB_DATA_TYPE_DOUBLE; - item->bytes = (int16_t)tDataTypes[item->type].bytes; - item->payload = malloc(item->bytes); - char* endptr = NULL; - *(item->payload) = strtold(sv, &endptr); - } - } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { - if (sv[0] == 'L') { - item->type = TSDB_DATA_TYPE_NCHAR; - uint32_t bytes = item->value.n - 3; -// uint32_t len = bytes; -// char* ucs = malloc(len); -// int32_t ncharBytes = 0; -// taosMbsToUcs4(sv+2, len, ucs, len, &ncharBytes); -// item->bytes = ncharBytes; -// item->payload = malloc(ncharBytes); -// memcpy(item->payload, ucs, ncharBytes); -// free(ucs); - item->bytes = bytes; - item->payload = malloc(bytes); - memcpy(item->payload, sv+1, bytes); - } else if (sv[0]=='"'){ - item->type = TSDB_DATA_TYPE_BINARY; - uint32_t bytes = item->value.n - 2; - item->bytes = bytes; - item->payload = malloc(bytes); - memcpy(item->payload, sv+1, bytes); - } - } else if (sv[0] == 't' || sv[0] == 'f' || sv[0]=='T' || sv[0] == 'F') { - item->type = TSDB_DATA_TYPE_BOOL; - item->bytes = tDataTypes[item->type].bytes; - item->payload = malloc(tDataTypes[item->type].bytes); - *(item->payload) = tolower(sv[0])=='t' ? true : false; - } - return 0; -} - -int32_t compareLPItemKey(const void* p1, const void* p2) { - const SLPItem* t1 = p1; - const SLPItem* t2 = p2; - uint32_t min = (t1->key.n < t2->key.n) ? t1->key.n : t2->key.n; - int res = strncmp(t1->key.z, t2->key.z, min); - if (res != 0) { - return res; - } else { - return (int)(t1->key.n) - (int)(t2->key.n); - } -} - -int32_t setPointTimeStamp(SLPPoint* point, SStrToken tsToken) { - point->tsToken = tsToken; - return 0; -} - -int32_t parsePointTime(SLPPoint* point) { - if (point->tsToken.n <= 0) { - point->ts = taosGetTimestampNs(); - } else { - char* endptr = NULL; - point->ts = strtoll(point->tsToken.z, &endptr, 10); - } - return 0; -} - -int32_t tscParseLine(SStrToken line, SLPPoint* point) { - int32_t pos = 0; - - int32_t start = 0; - int32_t err = scanToCommaOrSpace(line, start, &pos, LP_MEASUREMENT); - if (err != 0) { - tscError("a"); - return err; - } - - SStrToken measurement = {.z = line.z+start, .n = pos-start}; - setPointMeasurement(point, measurement); - point->tags = taosArrayInit(64, sizeof(SLPItem)); - start = pos + 1; - while (line.z[start] == ',') { - SLPItem item; - - err = scanToEqual(line, start, &pos); - if (err != 0) { - tscError("b"); - goto error; - } - - SStrToken tagKey = {.z = line.z + start, .n = pos-start}; - setItemKey(&item, tagKey, LP_TAG_KEY); - - start = pos + 1; - err = scanToCommaOrSpace(line, start, &pos, LP_TAG_VALUE); - if (err != 0) { - tscError("c"); - goto error; - } - - SStrToken tagValue = {.z = line.z + start, .n = pos-start}; - setItemValue(&item, tagValue, LP_TAG_VALUE); - - parseItemValue(&item, LP_ITEM_TAG); - taosArrayPush(point->tags, &item); - - start = pos + 1; - } - - taosArraySort(point->tags, compareLPItemKey); - - point->fields = taosArrayInit(64, sizeof(SLPItem)); - do { - SLPItem item; - err = scanToEqual(line, start, &pos); - if (err != 0) { - goto error; - } - SStrToken fieldKey = {.z = line.z + start, .n = pos- start}; - setItemKey(&item, fieldKey, LP_FIELD_KEY); - - start = pos + 1; - err = scanToCommaOrSpace(line, start, &pos, LP_FIELD_VALUE); - if (err != 0) { - goto error; - } - SStrToken fieldValue = {.z = line.z + start, .n = pos - start}; - setItemValue(&item, fieldValue, LP_TAG_VALUE); - - parseItemValue(&item, LP_ITEM_FIELD); - taosArrayPush(point->fields, &item); - - start = pos + 1; - } while (line.z[pos] == ','); - - taosArraySort(point->fields, compareLPItemKey); - - SStrToken tsToken = {.z = line.z+start, .n = line.n-start}; - setPointTimeStamp(point, tsToken); - parsePointTime(point); - - goto done; - -error: - // free array - return err; -done: - return 0; -} - - -int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { - for (int32_t i = 0; i < numLines; ++i) { - SStrToken tkLine = {.z = lines[i], .n = strlen(lines[i])+1}; - SLPPoint point; - tscParseLine(tkLine, &point); - taosArrayPush(points, &point); - } - return 0; -} - -TAOS_RES* taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { - SArray* points = taosArrayInit(numLines, sizeof(SLPPoint)); - tscParseLines(lines, numLines, points, NULL); - - - return NULL; -} -//================================================================================================= - typedef struct { char sTableName[TSDB_TABLE_NAME_LEN]; SHashObj* tagHash; @@ -305,6 +48,8 @@ typedef struct { SSmlSTableSchema* schema; } TAOS_SML_DATA_POINT; +//================================================================================================= + int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -318,32 +63,6 @@ int compareSmlColKv(const void* p1, const void* p2) { } } -int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - taosStringBuilderAppendString(&sb, point->stableName); - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - taosStringBuilderAppendChar(&sb, ','); - taosStringBuilderAppendString(&sb, tagKv->key); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - return 0; -} - int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { int32_t code = 0; @@ -393,15 +112,14 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { tscFreeSqlObj(pSql); - uint32_t size = tscGetTableMetaMaxSize(); STableMeta* tableMeta = calloc(1, size); taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); - tstrncpy(schema->sTableName, tableName, strlen(tableName)); + tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); for (int i=0; itableInfo.numOfColumns; ++i) { SSchema field; - tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)); + tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); field.type = tableMeta->schema[i].type; field.bytes = tableMeta->schema[i].bytes; SSchema* pField = taosArrayPush(schema->fields, &field); @@ -411,13 +129,13 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { for (int i=0; itableInfo.numOfTags; ++i) { int j = i + tableMeta->tableInfo.numOfColumns; SSchema field; - tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)); + tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); field.type = tableMeta->schema[j].type; field.bytes = tableMeta->schema[j].bytes; SSchema* pField = taosArrayPush(schema->tags, &field); taosHashPut(schema->tagHash, field.name, strlen(field.name), &pField, POINTER_BYTES); } - + free(tableMeta); tableMeta = NULL; return code; } @@ -586,6 +304,7 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); @@ -631,15 +350,46 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } +int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + taosStringBuilderAppendString(&sb, point->stableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + taosStringBuilderAppendString(&sb, tagKv->key); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "tbl%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + taosStringBuilderDestroy(&sb); + return 0; +} + int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsSchema, char* result, int16_t freeBytes) { size_t numTags = taosArrayGetSize(tagsSchema); size_t numCols = taosArrayGetSize(colsSchema); - sprintf(result, "insert into ? using %s(", sTableName); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); - } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") tags ("); + sprintf(result, "insert into ? using %s", sTableName); + +// snprintf(result+strlen(result), freeBytes-strlen(result), "("); +// for (int i = 0; i < numTags; ++i) { +// SSchema* tagSchema = taosArrayGet(tagsSchema, i); +// snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); +// } +// snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); + + snprintf(result + strlen(result), freeBytes-strlen(result), " tags ("); for (int i = 0; i < numTags; ++i) { snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); @@ -661,17 +411,38 @@ int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsS int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* rowsBind) { TAOS_STMT* stmt = taos_stmt_init(taos); - taos_stmt_prepare(stmt, sql, strlen(sql)); - - taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); - size_t rows = taosArrayGetSize(rowsBind); - for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colBind = taosArrayGetP(rowsBind, i); - taos_stmt_bind_param(stmt, colBind); - taos_stmt_add_batch(stmt); + int32_t code; + code = taos_stmt_prepare(stmt, sql, strlen(sql)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; } - taos_stmt_execute(stmt); + code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + size_t rows = taosArrayGetSize(rowsBind); + for (int32_t i = 0; i < rows; ++i) { + SArray* colBind = taosArrayGetP(rowsBind, i); + code = taos_stmt_bind_param(stmt, TARRAY_GET_START(colBind)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } TAOS_RES* res = taos_stmt_use_result(stmt); return taos_errno(res); } @@ -682,7 +453,7 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) TAOS_SML_DATA_POINT * point = points + i; if (!point->childTableName) { char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; getChildTableName(point, childTableName, &tableNameLen); point->childTableName = calloc(1, tableNameLen+1); strncpy(point->childTableName, childTableName, tableNameLen); @@ -696,50 +467,62 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) cTablePoints = taosArrayInit(64, sizeof(point)); taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); } - taosArrayPush(cTablePoints, point); + taosArrayPush(cTablePoints, &point); } + int isNullColBind = TSDB_TRUE; SArray** pCTablePoints = taosHashIterate(cname2points, NULL); while (pCTablePoints) { SArray* cTablePoints = *pCTablePoints; - TAOS_SML_DATA_POINT * point = taosArrayGet(cTablePoints, 0); + + TAOS_SML_DATA_POINT * point = taosArrayGetP(cTablePoints, 0); int32_t numTags = taosArrayGetSize(point->schema->tags); int32_t numCols = taosArrayGetSize(point->schema->fields); char* stableName = point->stableName; char* ctableName = point->childTableName; - char sql[TSDB_MAX_BINARY_LEN]; - getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); + + SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); + taosArraySetSize(tagBinds, numTags); + for (int j = 0; j < numTags; ++j) { + TAOS_BIND* bind = taosArrayGet(tagBinds, j); + bind->is_null = &isNullColBind; + } + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); + TAOS_BIND* bind = taosArrayGet(tagBinds, idx); + bind->buffer_type = kv->type; + bind->length = (uintptr_t*)&kv->length; + bind->buffer = kv->value; + bind->is_null = NULL; + } size_t rows = taosArrayGetSize(cTablePoints); SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); - SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); for (int i = 0; i < rows; ++i) { - point = taosArrayGet(cTablePoints, i); - - taosArraySetSize(tagBinds, numTags); - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* kv = point->tags + j; - int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); - TAOS_BIND* bind = taosArrayGet(tagBinds, idx); - bind->buffer_type = kv->type; - bind->length = (uintptr_t*)&kv->length; - bind->buffer = kv->value; - } + point = taosArrayGetP(cTablePoints, i); SArray* colBinds = taosArrayInit(numCols, sizeof(TAOS_BIND)); taosArraySetSize(colBinds, numCols); - for (int j = 0; jfieldNum; ++j) { + for (int j = 0; j < numCols; ++j) { + TAOS_BIND* bind = taosArrayGet(colBinds, j); + bind->is_null = &isNullColBind; + } + for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); TAOS_BIND* bind = taosArrayGet(colBinds, idx); bind->buffer_type = kv->type; bind->length = (uintptr_t*)&kv->length; bind->buffer = kv->value; + bind->is_null = NULL; } taosArrayPush(rowsBind, &colBinds); } + char sql[TSDB_MAX_BINARY_LEN]; + getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); insertBatch(taos, sql, ctableName, tagBinds, rowsBind); pCTablePoints = taosHashIterate(cname2points, pCTablePoints); @@ -747,7 +530,6 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) return 0; } - int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { int32_t code = TSDB_CODE_SUCCESS; SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray @@ -792,11 +574,13 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { for (int i = 0; i < numStable; ++i) { SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); SSmlSTableSchema dbSchema = {0}; - dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); + dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; @@ -813,19 +597,23 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { SHashObj* dbFieldHash = dbSchema.fieldHash; for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); SSchemaAction schemaAction = {0}; - bool actionNeeded = false; + bool actionNeeded = false; generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { taosArrayPush(schemaActions, &schemaAction); } } - for (int j = 0; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->tags, j); + SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); + SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); + memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); + + for (int j = 1; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->fields, j); SSchemaAction schemaAction = {0}; - bool actionNeeded = false; + bool actionNeeded = false; generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { taosArrayPush(schemaActions, &schemaAction); @@ -849,3 +637,317 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { //todo: table/column length check //todo: type check //todo: taosmbs2ucs4 check + +//========================================================================= + +typedef enum { + LP_ITEM_TAG, + LP_ITEM_FIELD +} LPItemKind; + +typedef struct { + SStrToken keyToken; + SStrToken valueToken; + + char key[TSDB_COL_NAME_LEN]; + int8_t type; + int16_t length; + + char* value; +}SLPItem; + +typedef struct { + SStrToken measToken; + SStrToken tsToken; + + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; + SArray* fields; + int64_t ts; + +} SLPPoint; + +typedef enum { + LP_MEASUREMENT, + LP_TAG_KEY, + LP_TAG_VALUE, + LP_FIELD_KEY, + LP_FIELD_VALUE +} LPPart; + +int32_t scanToCommaOrSpace(SStrToken s, int32_t start, int32_t* index, LPPart part) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == ',' || s.z[i] == ' ') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t scanToEqual(SStrToken s, int32_t start, int32_t* index) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == '=') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t setPointMeasurement(SLPPoint* point, SStrToken token) { + point->measToken = token; + if (point->measToken.n < TSDB_TABLE_NAME_LEN) { + strncpy(point->sTableName, point->measToken.z, point->measToken.n); + point->sTableName[point->measToken.n] = '\0'; + } + return 0; +} + +int32_t setItemKey(SLPItem* item, SStrToken key, LPPart part) { + item->keyToken = key; + if (item->keyToken.n < TSDB_COL_NAME_LEN) { + strncpy(item->key, item->keyToken.z, item->keyToken.n); + item->key[item->keyToken.n] = '\0'; + } + return 0; +} + +int32_t setItemValue(SLPItem* item, SStrToken value, LPPart part) { + item->valueToken = value; + return 0; +} + +int32_t parseItemValue(SLPItem* item, LPItemKind kind) { + char* sv = item->valueToken.z; + char* last = item->valueToken.z + item->valueToken.n - 1; + + if (isdigit(sv[0]) || sv[0] == '-') { + if (*last == 'i') { + item->type = TSDB_DATA_TYPE_BIGINT; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(item->value) = strtoll(sv, &endptr, 10); + } else { + item->type = TSDB_DATA_TYPE_DOUBLE; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(item->value) = strtold(sv, &endptr); + } + } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { + if (sv[0] == 'L') { + item->type = TSDB_DATA_TYPE_NCHAR; + uint32_t bytes = item->valueToken.n - 3; + item->length = bytes; + item->value = malloc(bytes); + memcpy(item->value, sv+1, bytes); + } else if (sv[0]=='"'){ + item->type = TSDB_DATA_TYPE_BINARY; + uint32_t bytes = item->valueToken.n - 2; + item->length = bytes; + item->value = malloc(bytes); + memcpy(item->value, sv+1, bytes); + } + } else if (sv[0] == 't' || sv[0] == 'f' || sv[0]=='T' || sv[0] == 'F') { + item->type = TSDB_DATA_TYPE_BOOL; + item->length = tDataTypes[item->type].bytes; + item->value = malloc(tDataTypes[item->type].bytes); + *(item->value) = tolower(sv[0])=='t' ? TSDB_TRUE : TSDB_FALSE; + } + return 0; +} + +int32_t compareLPItemKey(const void* p1, const void* p2) { + const SLPItem* t1 = p1; + const SLPItem* t2 = p2; + uint32_t min = (t1->keyToken.n < t2->keyToken.n) ? t1->keyToken.n : t2->keyToken.n; + int res = strncmp(t1->keyToken.z, t2->keyToken.z, min); + if (res != 0) { + return res; + } else { + return (int)(t1->keyToken.n) - (int)(t2->keyToken.n); + } +} + +int32_t setPointTimeStamp(SLPPoint* point, SStrToken tsToken) { + point->tsToken = tsToken; + return 0; +} + +int32_t parsePointTime(SLPPoint* point) { + if (point->tsToken.n <= 0) { + point->ts = taosGetTimestampNs(); + } else { + char* endptr = NULL; + point->ts = strtoll(point->tsToken.z, &endptr, 10); + } + return 0; +} + +int32_t tscParseLine(SStrToken line, SLPPoint* point) { + int32_t pos = 0; + + int32_t start = 0; + int32_t err = scanToCommaOrSpace(line, start, &pos, LP_MEASUREMENT); + if (err != 0) { + tscError("a"); + return err; + } + + SStrToken measurement = {.z = line.z+start, .n = pos-start}; + setPointMeasurement(point, measurement); + point->tags = taosArrayInit(64, sizeof(SLPItem)); + start = pos; + while (line.z[start] == ',') { + SLPItem item; + + start++; + err = scanToEqual(line, start, &pos); + if (err != 0) { + tscError("b"); + goto error; + } + + SStrToken tagKey = {.z = line.z + start, .n = pos-start}; + setItemKey(&item, tagKey, LP_TAG_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_TAG_VALUE); + if (err != 0) { + tscError("c"); + goto error; + } + + SStrToken tagValue = {.z = line.z + start, .n = pos-start}; + setItemValue(&item, tagValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_TAG); + taosArrayPush(point->tags, &item); + + start = pos; + } + + taosArraySort(point->tags, compareLPItemKey); + + point->fields = taosArrayInit(64, sizeof(SLPItem)); + + start++; + do { + SLPItem item; + + err = scanToEqual(line, start, &pos); + if (err != 0) { + goto error; + } + SStrToken fieldKey = {.z = line.z + start, .n = pos- start}; + setItemKey(&item, fieldKey, LP_FIELD_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_FIELD_VALUE); + if (err != 0) { + goto error; + } + SStrToken fieldValue = {.z = line.z + start, .n = pos - start}; + setItemValue(&item, fieldValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_FIELD); + taosArrayPush(point->fields, &item); + + start = pos + 1; + } while (line.z[pos] == ','); + + taosArraySort(point->fields, compareLPItemKey); + + SStrToken tsToken = {.z = line.z+start, .n = line.n-start}; + setPointTimeStamp(point, tsToken); + parsePointTime(point); + + goto done; + + error: + // free array + return err; + done: + return 0; +} + + +int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { + for (int32_t i = 0; i < numLines; ++i) { + SStrToken tkLine = {.z = lines[i], .n = strlen(lines[i])+1}; + SLPPoint point; + tscParseLine(tkLine, &point); + taosArrayPush(points, &point); + } + return 0; +} + +int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { + SArray* lpPoints = taosArrayInit(numLines, sizeof(SLPPoint)); + tscParseLines(lines, numLines, lpPoints, NULL); + + size_t numPoints = taosArrayGetSize(lpPoints); + TAOS_SML_DATA_POINT* points = calloc(numPoints, sizeof(TAOS_SML_DATA_POINT)); + for (int i = 0; i < numPoints; ++i) { + SLPPoint* lpPoint = taosArrayGet(lpPoints, i); + TAOS_SML_DATA_POINT* point = points+i; + point->stableName = calloc(1, strlen(lpPoint->sTableName)+1); + strncpy(point->stableName, lpPoint->sTableName, strlen(lpPoint->sTableName)); + point->stableName[strlen(lpPoint->sTableName)] = '\0'; + + size_t lpTagSize = taosArrayGetSize(lpPoint->tags); + point->tags = calloc(lpTagSize, sizeof(TAOS_SML_KV)); + point->tagNum = lpTagSize; + for (int j=0; jtags, j); + TAOS_SML_KV* tagKv = point->tags + j; + + size_t kenLen = strlen(lpTag->key); + tagKv->key = calloc(1, kenLen+1); + strncpy(tagKv->key, lpTag->key, kenLen); + tagKv->key[kenLen] = '\0'; + + tagKv->type = lpTag->type; + tagKv->length = lpTag->length; + tagKv->value = malloc(tagKv->length); + memcpy(tagKv->value, lpTag->value, tagKv->length); + } + + size_t lpFieldsSize = taosArrayGetSize(lpPoint->fields); + point->fields = calloc(lpFieldsSize + 1, sizeof(TAOS_SML_KV)); + point->fieldNum = lpFieldsSize + 1; + + TAOS_SML_KV* tsField = point->fields + 0; + char tsKey[256]; + snprintf(tsKey, 256, "_%s_ts", point->stableName); + size_t tsKeyLen = strlen(tsKey); + tsField->key = calloc(1, tsKeyLen+1); + strncpy(tsField->key, tsKey, tsKeyLen); + tsField->key[tsKeyLen] = '\0'; + tsField->type = TSDB_DATA_TYPE_TIMESTAMP; + tsField->length = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + tsField->value = malloc(tsField->length); + memcpy(tsField->value, &(lpPoint->ts), tsField->length); + + for (int j=0; jfields, j); + TAOS_SML_KV* fieldKv = point->fields + j + 1; + + size_t kenLen = strlen(lpField->key); + fieldKv->key = calloc(1, kenLen+1); + strncpy(fieldKv->key, lpField->key, kenLen); + fieldKv->key[kenLen] = '\0'; + + fieldKv->type = lpField->type; + fieldKv->length = lpField->length; + fieldKv->value = malloc(fieldKv->length); + memcpy(fieldKv->value, lpField->value, fieldKv->length); + } + } + + taos_sml_insert(taos, points, numPoints); + return 0; +} + diff --git a/src/inc/taos.h b/src/inc/taos.h index 9f72945ef0..ca18c4fb93 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -169,6 +169,8 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); +DLL_EXPORT int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines); + #ifdef __cplusplus } #endif diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 0f24df0f47..dc77677774 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -949,6 +949,15 @@ void verify_stream(TAOS* taos) { taos_close_stream(strm); } +int32_t verify_schema_less(TAOS* taos) { + prepare_data(taos); + char* lines[] = { + "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639162922" + }; + int code = taos_insert_by_lines(taos, lines , 1); + return code; +} + int main(int argc, char *argv[]) { const char* host = "127.0.0.1"; const char* user = "root"; @@ -967,6 +976,12 @@ int main(int argc, char *argv[]) { info = taos_get_client_info(taos); printf("client info: %s\n", info); + printf("************ verify query *************\n"); + int code = verify_schema_less(taos); + if (code == 0) { + return code; + } + printf("************ verify query *************\n"); verify_query(taos); From cb5a0012f411675b7d5a40d0ec6cd88361c157ec Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 00:39:16 +0800 Subject: [PATCH 09/43] before time precision and create table with tags instead of autocreate --- src/client/src/tscParseLineProtocol.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 3a88d2e906..4b9191a91f 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -492,7 +492,8 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); TAOS_BIND* bind = taosArrayGet(tagBinds, idx); bind->buffer_type = kv->type; - bind->length = (uintptr_t*)&kv->length; + bind->length = malloc(sizeof(uintptr_t*)); + *bind->length = kv->length; bind->buffer = kv->value; bind->is_null = NULL; } @@ -514,7 +515,8 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); TAOS_BIND* bind = taosArrayGet(colBinds, idx); bind->buffer_type = kv->type; - bind->length = (uintptr_t*)&kv->length; + bind->length = malloc(sizeof(uintptr_t*)); + *bind->length = kv->length; bind->buffer = kv->value; bind->is_null = NULL; } From 3d25ecdf29c3d912bb91e81f47c72b97fb737cea Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 08:38:51 +0800 Subject: [PATCH 10/43] can add two lines of one supertable/diffrent subtable --- src/client/src/tscParseLineProtocol.c | 12 ++++++------ tests/examples/c/apitest.c | 8 +++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 4b9191a91f..1fde741c2b 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -540,13 +540,13 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { for (int i = 0; i < numPoint; ++i) { TAOS_SML_DATA_POINT* point = &points[i]; - SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); + size_t stableNameLen = strlen(point->stableName); + SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, stableNameLen); SSmlSTableSchema* pStableSchema = NULL; if (ppStableSchema) { pStableSchema= *ppStableSchema; } else { SSmlSTableSchema schema; - size_t stableNameLen = strlen(point->stableName); strncpy(schema.sTableName, point->stableName, stableNameLen); schema.sTableName[stableNameLen] = '\0'; schema.fields = taosArrayInit(64, sizeof(SSchema)); @@ -730,13 +730,13 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(item->value) = strtoll(sv, &endptr, 10); + *(int64_t*)(item->value) = strtoll(sv, &endptr, 10); } else { item->type = TSDB_DATA_TYPE_DOUBLE; item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(item->value) = strtold(sv, &endptr); + *(double*)(item->value) = strtold(sv, &endptr); } } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { if (sv[0] == 'L') { @@ -744,7 +744,7 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { uint32_t bytes = item->valueToken.n - 3; item->length = bytes; item->value = malloc(bytes); - memcpy(item->value, sv+1, bytes); + memcpy(item->value, sv+2, bytes); } else if (sv[0]=='"'){ item->type = TSDB_DATA_TYPE_BINARY; uint32_t bytes = item->valueToken.n - 2; @@ -756,7 +756,7 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { item->type = TSDB_DATA_TYPE_BOOL; item->length = tDataTypes[item->type].bytes; item->value = malloc(tDataTypes[item->type].bytes); - *(item->value) = tolower(sv[0])=='t' ? TSDB_TRUE : TSDB_FALSE; + *(uint8_t*)(item->value) = tolower(sv[0])=='t' ? TSDB_TRUE : TSDB_FALSE; } return 0; } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index dc77677774..04a03df6fa 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -952,9 +952,11 @@ void verify_stream(TAOS* taos) { int32_t verify_schema_less(TAOS* taos) { prepare_data(taos); char* lines[] = { - "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639162922" + "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642" }; - int code = taos_insert_by_lines(taos, lines , 1); + int code = taos_insert_by_lines(taos, lines , 2); return code; } @@ -976,7 +978,7 @@ int main(int argc, char *argv[]) { info = taos_get_client_info(taos); printf("client info: %s\n", info); - printf("************ verify query *************\n"); + printf("************ verify shemaless *************\n"); int code = verify_schema_less(taos); if (code == 0) { return code; From 4a75ebe4afacbcce488ec35fbdefc5e900ca781f Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 09:17:15 +0800 Subject: [PATCH 11/43] create child table with tags --- src/client/src/tscParseLineProtocol.c | 87 +++++++++++++++++---------- tests/examples/c/apitest.c | 2 +- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 1fde741c2b..dd0e64ba84 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -377,39 +377,26 @@ int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tabl return 0; } -int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsSchema, char* result, int16_t freeBytes) { +int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind) { size_t numTags = taosArrayGetSize(tagsSchema); - size_t numCols = taosArrayGetSize(colsSchema); - sprintf(result, "insert into ? using %s", sTableName); + char sql[TSDB_MAX_BINARY_LEN] = {0}; + int freeBytes = TSDB_MAX_BINARY_LEN; + sprintf(sql, "create table if not exists %s using %s", cTableName, sTableName); -// snprintf(result+strlen(result), freeBytes-strlen(result), "("); -// for (int i = 0; i < numTags; ++i) { -// SSchema* tagSchema = taosArrayGet(tagsSchema, i); -// snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); -// } -// snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "("); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); + } + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - snprintf(result + strlen(result), freeBytes-strlen(result), " tags ("); + snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); for (int i = 0; i < numTags; ++i) { - snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") ("); + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", colSchema->name); - } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") values ("); - - for (int i = 0; i < numCols; ++i) { - snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); - } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); - return 0; -} - -int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* rowsBind) { TAOS_STMT* stmt = taos_stmt_init(taos); int32_t code; code = taos_stmt_prepare(stmt, sql, strlen(sql)); @@ -418,7 +405,47 @@ int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, S return code; } - code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + TAOS_RES* res = taos_stmt_use_result(stmt); + return taos_errno(res); +} + +int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind) { + size_t numCols = taosArrayGetSize(colsSchema); + char sql[TSDB_MAX_BINARY_LEN]; + int32_t freeBytes = TSDB_MAX_BINARY_LEN; + sprintf(sql, "insert into ? ("); + + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); + + TAOS_STMT* stmt = taos_stmt_init(taos); + int32_t code; + code = taos_stmt_prepare(stmt, sql, strlen(sql)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + + code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { printf("%s", taos_stmt_errstr(stmt)); return code; @@ -478,7 +505,6 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) TAOS_SML_DATA_POINT * point = taosArrayGetP(cTablePoints, 0); int32_t numTags = taosArrayGetSize(point->schema->tags); int32_t numCols = taosArrayGetSize(point->schema->fields); - char* stableName = point->stableName; char* ctableName = point->childTableName; SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); @@ -523,9 +549,8 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) taosArrayPush(rowsBind, &colBinds); } - char sql[TSDB_MAX_BINARY_LEN]; - getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); - insertBatch(taos, sql, ctableName, tagBinds, rowsBind); + creatChildTableIfNotExists(taos, point->childTableName, point->stableName, point->schema->tags, tagBinds); + insertBatch(taos, ctableName, point->schema->fields, rowsBind); pCTablePoints = taosHashIterate(cname2points, pCTablePoints); } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 04a03df6fa..5272dafaf1 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -956,7 +956,7 @@ int32_t verify_schema_less(TAOS* taos) { "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642" }; - int code = taos_insert_by_lines(taos, lines , 2); + int code = taos_insert_by_lines(taos, lines , 3); return code; } From 62e4b0b599ba7a066cd6a0be435c60bac3a3cabd Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 10:16:30 +0800 Subject: [PATCH 12/43] add timestamp precision support --- src/client/src/tscParseLineProtocol.c | 39 ++++++++++++++++++++------- tests/examples/c/apitest.c | 18 +++++++++---- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index dd0e64ba84..c0cc2ea3ae 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -21,6 +21,7 @@ typedef struct { SHashObj* fieldHash; SArray* tags; //SArray SArray* fields; //SArray + uint8_t precision; } SSmlSTableSchema; typedef struct { @@ -117,6 +118,7 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); + schema->precision = tableMeta->tableInfo.precision; for (int i=0; itableInfo.numOfColumns; ++i) { SSchema field; tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); @@ -486,6 +488,25 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) strncpy(point->childTableName, childTableName, tableNameLen); point->childTableName[tableNameLen] = '\0'; } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, point->schema->precision); + *(int64_t*)(kv->value) = ts; + } + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, point->schema->precision); + *(int64_t*)(kv->value) = ts; + } + } + SArray* cTablePoints = NULL; SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); if (pCTablePoints) { @@ -596,7 +617,6 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { point->schema = pStableSchema; } - SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); size_t numStable = taosArrayGetSize(stableArray); for (int i = 0; i < numStable; ++i) { SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); @@ -615,8 +635,10 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); schemaAction.createSTable.tags = pointSchema->tags; schemaAction.createSTable.fields = pointSchema->fields; - taosArrayPush(schemaActions, &schemaAction); - }else if (code == TSDB_CODE_SUCCESS) { + applySchemaAction(taos, &schemaAction); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + pointSchema->precision = dbSchema.precision; + } else if (code == TSDB_CODE_SUCCESS) { size_t pointTagSize = taosArrayGetSize(pointSchema->tags); size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); @@ -629,7 +651,7 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { bool actionNeeded = false; generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); + applySchemaAction(taos, &schemaAction); } } @@ -643,19 +665,16 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { bool actionNeeded = false; generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); + applySchemaAction(taos, &schemaAction); } } + + pointSchema->precision = dbSchema.precision; } else { return code; } } - for (int i = 0; i < taosArrayGetSize(schemaActions); ++i) { - SSchemaAction* action = taosArrayGet(schemaActions, i); - applySchemaAction(taos, action); - } - insertPoints(taos, points, numPoint); return code; } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 5272dafaf1..656d5fd217 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -12,7 +12,7 @@ static void prepare_data(TAOS* taos) { result = taos_query(taos, "drop database if exists test;"); taos_free_result(result); usleep(100000); - result = taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test precision 'us';"); taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); @@ -950,11 +950,19 @@ void verify_stream(TAOS* taos) { } int32_t verify_schema_less(TAOS* taos) { - prepare_data(taos); + TAOS_RES *result; + result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database test precision 'us';"); + taos_free_result(result); + usleep(100000); + taos_select_db(taos, "test"); + char* lines[] = { - "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642" + "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000" }; int code = taos_insert_by_lines(taos, lines , 3); return code; From d8c57f0be0bb51f2a81157f12d8885f5afa70f23 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 10:30:54 +0800 Subject: [PATCH 13/43] fixing exsiting stable --- src/client/src/tscParseLineProtocol.c | 2 +- tests/examples/c/apitest.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index c0cc2ea3ae..61f9cd9e76 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -225,7 +225,7 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool isTag, char sTableName[], SSchemaAction* action, bool* actionNeeded) { SSchema** ppDbAttr = taosHashGet(dbAttrHash, pointColField->name, strlen(pointColField->name)); - if (*ppDbAttr) { + if (ppDbAttr) { SSchema* dbAttr = *ppDbAttr; if (pointColField->type != dbAttr->type) { //todo error diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 656d5fd217..5cb3f5762a 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -958,13 +958,15 @@ int32_t verify_schema_less(TAOS* taos) { taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); + result = taos_query(taos, "create stable ste(ts timestamp, f int) tags(t1 bigint)"); char* lines[] = { "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000" + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", + "ste,t2=5,t3=L\"ste\" c1=true,c2=4 1626056811823316532" }; - int code = taos_insert_by_lines(taos, lines , 3); + int code = taos_insert_by_lines(taos, lines , 4); return code; } From 958b89b4630c11c3c2c360d7d757c82213862461 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 11:27:34 +0800 Subject: [PATCH 14/43] before tsim/log/memory leak handling --- tests/examples/c/apitest.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 5cb3f5762a..409140e0f2 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -957,16 +957,20 @@ int32_t verify_schema_less(TAOS* taos) { result = taos_query(taos, "create database test precision 'us';"); taos_free_result(result); usleep(100000); + taos_select_db(taos, "test"); result = taos_query(taos, "create stable ste(ts timestamp, f int) tags(t1 bigint)"); + taos_free_result(result); + usleep(100000); char* lines[] = { "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", - "ste,t2=5,t3=L\"ste\" c1=true,c2=4 1626056811823316532" + "ste,t2=5,t3=L\"ste\" c1=true,c2=4,c3=\"iam\" 1626056811823316532", + "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532" }; - int code = taos_insert_by_lines(taos, lines , 4); + int code = taos_insert_by_lines(taos, lines , 5); return code; } From 64fb524796c4746a1a75f5a5fac29021305686d1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 6 Jul 2021 18:05:27 +0800 Subject: [PATCH 15/43] develop schemaless --- src/client/src/tscParseLineProtocol.c | 731 ++++++++++++++++++++++++++ 1 file changed, 731 insertions(+) create mode 100644 src/client/src/tscParseLineProtocol.c diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c new file mode 100644 index 0000000000..79e8955f9a --- /dev/null +++ b/src/client/src/tscParseLineProtocol.c @@ -0,0 +1,731 @@ +#include +#include +#include +#include +#include "os.h" +#include "osString.h" +#include "ttype.h" +#include "tmd5.h" +#include "tstrbuild.h" +#include "tname.h" +#include "taos.h" +#include "tsclient.h" +#include "tscLog.h" +#include "hash.h" +#include "tskiplist.h" +#include "tscUtil.h" + +typedef enum { + LP_ITEM_TAG, + LP_ITEM_FIELD +} LPItemKind; + +typedef struct { + SStrToken key; + SStrToken value; + + char name[TSDB_COL_NAME_LEN]; + int8_t type; + int16_t bytes; + + char* payload; +}SLPItem; + +typedef struct { + SStrToken measToken; + SStrToken tsToken; + + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; + SArray* fields; + int64_t ts; + +} SLPPoint; + +typedef enum { + LP_MEASUREMENT, + LP_TAG_KEY, + LP_TAG_VALUE, + LP_FIELD_KEY, + LP_FIELD_VALUE +} LPPart; + +int32_t scanToCommaOrSpace(SStrToken s, int32_t start, int32_t* index, LPPart part) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == ',' || s.z[i] == ' ') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t scanToEqual(SStrToken s, int32_t start, int32_t* index) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == '=') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t setPointMeasurement(SLPPoint* point, SStrToken token) { + point->measToken = token; + if (point->measToken.n < TSDB_TABLE_NAME_LEN) { + strncpy(point->sTableName, point->measToken.z, point->measToken.n); + point->sTableName[point->measToken.n] = '\0'; + } + return 0; +} + +int32_t setItemKey(SLPItem* item, SStrToken key, LPPart part) { + item->key = key; + if (item->key.n < TSDB_COL_NAME_LEN) { + strncpy(item->name, item->key.z, item->key.n); + item->name[item->key.n] = '\0'; + } + return 0; +} + +int32_t setItemValue(SLPItem* item, SStrToken value, LPPart part) { + item->value = value; + return 0; +} + +int32_t parseItemValue(SLPItem* item, LPItemKind kind) { + char* sv = item->value.z; + char* last = item->value.z + item->value.n - 1; + + if (isdigit(sv[0]) || sv[0] == '-') { + if (*last == 'i') { + item->type = TSDB_DATA_TYPE_BIGINT; + item->bytes = (int16_t)tDataTypes[item->type].bytes; + item->payload = malloc(item->bytes); + char* endptr = NULL; + *(item->payload) = strtoll(sv, &endptr, 10); + } else { + item->type = TSDB_DATA_TYPE_DOUBLE; + item->bytes = (int16_t)tDataTypes[item->type].bytes; + item->payload = malloc(item->bytes); + char* endptr = NULL; + *(item->payload) = strtold(sv, &endptr); + } + } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { + if (sv[0] == 'L') { + item->type = TSDB_DATA_TYPE_NCHAR; + uint32_t bytes = item->value.n - 3; +// uint32_t len = bytes; +// char* ucs = malloc(len); +// int32_t ncharBytes = 0; +// taosMbsToUcs4(sv+2, len, ucs, len, &ncharBytes); +// item->bytes = ncharBytes; +// item->payload = malloc(ncharBytes); +// memcpy(item->payload, ucs, ncharBytes); +// free(ucs); + item->bytes = bytes; + item->payload = malloc(bytes); + memcpy(item->payload, sv+1, bytes); + } else if (sv[0]=='"'){ + item->type = TSDB_DATA_TYPE_BINARY; + uint32_t bytes = item->value.n - 2; + item->bytes = bytes; + item->payload = malloc(bytes); + memcpy(item->payload, sv+1, bytes); + } + } else if (sv[0] == 't' || sv[0] == 'f' || sv[0]=='T' || sv[0] == 'F') { + item->type = TSDB_DATA_TYPE_BOOL; + item->bytes = tDataTypes[item->type].bytes; + item->payload = malloc(tDataTypes[item->type].bytes); + *(item->payload) = tolower(sv[0])=='t' ? true : false; + } + return 0; +} + +int32_t compareLPItemKey(const void* p1, const void* p2) { + const SLPItem* t1 = p1; + const SLPItem* t2 = p2; + uint32_t min = (t1->key.n < t2->key.n) ? t1->key.n : t2->key.n; + int res = strncmp(t1->key.z, t2->key.z, min); + if (res != 0) { + return res; + } else { + return (int)(t1->key.n) - (int)(t2->key.n); + } +} + +int32_t setPointTimeStamp(SLPPoint* point, SStrToken tsToken) { + point->tsToken = tsToken; + return 0; +} + +int32_t parsePointTime(SLPPoint* point) { + if (point->tsToken.n <= 0) { + point->ts = taosGetTimestampNs(); + } else { + char* endptr = NULL; + point->ts = strtoll(point->tsToken.z, &endptr, 10); + } + return 0; +} + +int32_t tscParseLine(SStrToken line, SLPPoint* point) { + int32_t pos = 0; + + int32_t start = 0; + int32_t err = scanToCommaOrSpace(line, start, &pos, LP_MEASUREMENT); + if (err != 0) { + tscError("a"); + return err; + } + + SStrToken measurement = {.z = line.z+start, .n = pos-start}; + setPointMeasurement(point, measurement); + point->tags = taosArrayInit(64, sizeof(SLPItem)); + start = pos + 1; + while (line.z[start] == ',') { + SLPItem item; + + err = scanToEqual(line, start, &pos); + if (err != 0) { + tscError("b"); + goto error; + } + + SStrToken tagKey = {.z = line.z + start, .n = pos-start}; + setItemKey(&item, tagKey, LP_TAG_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_TAG_VALUE); + if (err != 0) { + tscError("c"); + goto error; + } + + SStrToken tagValue = {.z = line.z + start, .n = pos-start}; + setItemValue(&item, tagValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_TAG); + taosArrayPush(point->tags, &item); + + start = pos + 1; + } + + taosArraySort(point->tags, compareLPItemKey); + + point->fields = taosArrayInit(64, sizeof(SLPItem)); + do { + SLPItem item; + err = scanToEqual(line, start, &pos); + if (err != 0) { + goto error; + } + SStrToken fieldKey = {.z = line.z + start, .n = pos- start}; + setItemKey(&item, fieldKey, LP_FIELD_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_FIELD_VALUE); + if (err != 0) { + goto error; + } + SStrToken fieldValue = {.z = line.z + start, .n = pos - start}; + setItemValue(&item, fieldValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_FIELD); + taosArrayPush(point->fields, &item); + + start = pos + 1; + } while (line.z[pos] == ','); + + taosArraySort(point->fields, compareLPItemKey); + + SStrToken tsToken = {.z = line.z+start, .n = line.n-start}; + setPointTimeStamp(point, tsToken); + parsePointTime(point); + + goto done; + +error: + // free array + return err; +done: + return 0; +} + + +int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { + for (int32_t i = 0; i < numLines; ++i) { + SStrToken tkLine = {.z = lines[i], .n = strlen(lines[i])+1}; + SLPPoint point; + tscParseLine(tkLine, &point); + taosArrayPush(points, &point); + } + return 0; +} + +TAOS_RES* taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { + SArray* points = taosArrayInit(numLines, sizeof(SLPPoint)); + tscParseLines(lines, numLines, points, NULL); + + + return NULL; +} +//================================================================================================= + +typedef struct { + char* key; + uint8_t type; + int16_t length; + char* value; +} TAOS_SML_KV; + +typedef struct { + char* stableName; + + char* childTableName; + TAOS_SML_KV* tags; + int tagNum; + + // first kv must be timestamp + TAOS_SML_KV* fields; + int fieldNum; + +} TAOS_SML_DATA_POINT; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SHashObj* tagHash; + SHashObj* fieldHash; + SArray* tags; //SArray + SArray* fields; //SArray +} SSmlSTableSchema; + + +int compareSmlColKv(const void* p1, const void* p2) { + TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; + TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; + int kvLen1 = (int)strlen(kv1->key); + int kvLen2 = (int)strlen(kv2->key); + int res = strncasecmp(kv1->key, kv2->key, MIN(kvLen1, kvLen2)); + if (res != 0) { + return res; + } else { + return kvLen1-kvLen2; + } +} + +int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + taosStringBuilderAppendString(&sb, point->stableName); + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + taosStringBuilderAppendChar(&sb, ','); + taosStringBuilderAppendString(&sb, tagKv->key); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + return 0; +} + +int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { + int32_t code = 0; + + STscObj *pObj = (STscObj *)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + char sql[256]; + snprintf(sql, 256, "describe %s", tableName); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + taos_free_result(res); + return code; + } + taos_free_result(res); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + SStrToken tableToken = {.z=tableName, .n=strlen(tableName), .type=TK_ID}; + tGetToken(tableName, &tableToken.type); + // Check if the table name available or not + if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + return code; + } + + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + + if (code != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); + return code; + } + + tscFreeSqlObj(pSql); + + + uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* tableMeta = calloc(1, size); + taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); + + tstrncpy(schema->sTableName, tableName, strlen(tableName)); + for (int i=0; itableInfo.numOfColumns; ++i) { + SSchema field; + tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)); + field.type = tableMeta->schema[i].type; + field.bytes = tableMeta->schema[i].bytes; + SSchema* pField = taosArrayPush(schema->fields, &field); + taosHashPut(schema->fieldHash, field.name, strlen(field.name), &pField, POINTER_BYTES); + } + + for (int i=0; itableInfo.numOfTags; ++i) { + int j = i + tableMeta->tableInfo.numOfColumns; + SSchema field; + tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)); + field.type = tableMeta->schema[j].type; + field.bytes = tableMeta->schema[j].bytes; + SSchema* pField = taosArrayPush(schema->tags, &field); + taosHashPut(schema->tagHash, field.name, strlen(field.name), &pField, POINTER_BYTES); + } + + return code; + +} + +typedef enum { + SCHEMA_ACTION_CREATE_STABLE, + SCHEMA_ACTION_ADD_COLUMN, + SCHEMA_ACTION_ADD_TAG, + SCHEMA_ACTION_CHANGE_COLUMN_SIZE, + SCHEMA_ACTION_CHANGE_TAG_SIZE, + SCHEMA_ACTION_CREATE_CTABLE +} ESchemaAction; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; //SArray + SArray* fields; //SArray +} SCreateSTableActionInfo; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SSchema* field; +} SAlterSTableActionInfo; + +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + char cTableName[TSDB_TABLE_NAME_LEN]; + TAOS_SML_KV* tags; + int tagNum; +} SCreateCTableActionInfo; + +typedef struct { + ESchemaAction action; + union { + SCreateSTableActionInfo createSTable; + SAlterSTableActionInfo alterSTable; + SCreateCTableActionInfo createCTable; + }; +} SSchemaAction; + +int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { + if (!IS_VAR_DATA_TYPE(kv->type)) { + *bytes = tDataTypes[kv->type].bytes; + } else { + if (kv->type == TSDB_DATA_TYPE_NCHAR) { + char* ucs = malloc(kv->length * TSDB_NCHAR_SIZE + 1); + int32_t bytesNeeded = 0; + //todo check conversion succeed + taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); + free(ucs); + *bytes = bytesNeeded + VARSTR_HEADER_SIZE; + + } else if (kv->type == TSDB_DATA_TYPE_BINARY) { + *bytes = kv->length + VARSTR_HEADER_SIZE; + } + } + return 0; +} + +int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array) { + SSchema* pField = NULL; + SSchema** ppField = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); + if (ppField) { + pField = *ppField; + + if (pField->type != smlKv->type) { + //TODO: + tscError("type mismatch"); + return -1; + } + + int32_t bytes = 0; + getFieldBytesFromSmlKv(smlKv, &bytes); + pField->bytes = MAX(pField->bytes, bytes); + + } else { + SSchema field; + size_t tagKeyLen = strlen(smlKv->key); + strncpy(field.name, smlKv->key, tagKeyLen); + field.name[tagKeyLen] = '\0'; + field.type = smlKv->type; + + int32_t bytes = 0; + getFieldBytesFromSmlKv(smlKv, &bytes); + field.bytes = bytes; + + pField = taosArrayPush(array, &field); + taosHashPut(hash, field.name, tagKeyLen, &pField, POINTER_BYTES); + } + return 0; +} + +int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool isTag, char sTableName[], + SSchemaAction* action, bool* actionNeeded) { + SSchema** ppDbAttr = taosHashGet(dbAttrHash, pointColField->name, strlen(pointColField->name)); + if (*ppDbAttr) { + SSchema* dbAttr = *ppDbAttr; + if (pointColField->type != dbAttr->type) { + //todo error + return -5; + } + + if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { + if (isTag) { + action->action = SCHEMA_ACTION_CHANGE_TAG_SIZE; + } else { + action->action = SCHEMA_ACTION_CHANGE_COLUMN_SIZE; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + } else { + if (isTag) { + action->action = SCHEMA_ACTION_ADD_TAG; + } else { + action->action = SCHEMA_ACTION_ADD_COLUMN; + } + memset(&action->alterSTable, 0, sizeof(SAlterSTableActionInfo)); + memcpy(action->alterSTable.sTableName, sTableName, TSDB_TABLE_NAME_LEN); + action->alterSTable.field = pointColField; + *actionNeeded = true; + } + return 0; +} + +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray + SHashObj* sname2shema = taosHashInit(32, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int i = 0; i < numPoint; ++i) { + TAOS_SML_DATA_POINT* point = &points[i]; + SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); + SSmlSTableSchema* pStableSchema = NULL; + if (ppStableSchema) { + pStableSchema= *ppStableSchema; + } else { + SSmlSTableSchema schema; + size_t stableNameLen = strlen(point->stableName); + strncpy(schema.sTableName, point->stableName, stableNameLen); + schema.sTableName[stableNameLen] = '\0'; + schema.fields = taosArrayInit(64, sizeof(SSchema)); + schema.tags = taosArrayInit(8, sizeof(SSchema)); + schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + pStableSchema = taosArrayPush(stableArray, &schema); + taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); + } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + addTaosFieldToHashAndArray(tagKv, pStableSchema->tagHash, pStableSchema->tags); + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* fieldKv = point->fields + j; + addTaosFieldToHashAndArray(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); + } + } + + SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); + size_t numStable = taosArrayGetSize(stableArray); + for (int i = 0; i < numStable; ++i) { + SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); + SSmlSTableSchema dbSchema = {0}; + dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); + dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); + dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); + memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); + schemaAction.createSTable.tags = pointSchema->tags; + schemaAction.createSTable.fields = pointSchema->fields; + taosArrayPush(schemaActions, &schemaAction); + }else if (code == TSDB_CODE_SUCCESS) { + size_t pointTagSize = taosArrayGetSize(pointSchema->tags); + size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); + + SHashObj* dbTagHash = dbSchema.tagHash; + SHashObj* dbFieldHash = dbSchema.fieldHash; + + for (int j = 0; j < pointTagSize; ++j) { + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + + for (int j = 0; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + } else { + return code; + } + } + + return code; +} + + +int32_t buildColumnDescription(SSchema* field, + char* buf, int32_t bufSize, int32_t* outBytes) { + uint8_t type = field->type; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + int32_t bytes = field->bytes - VARSTR_HEADER_SIZE; + if (type == TSDB_DATA_TYPE_NCHAR) { + bytes = bytes/TSDB_NCHAR_SIZE; + } + int out = snprintf(buf, bufSize,"%s %s(%d)", + field->name,tDataTypes[field->type].name, bytes); + *outBytes = out; + } else { + int out = snprintf(buf, bufSize, "%s %s", + field->name, tDataTypes[type].name); + *outBytes = out; + } + + return 0; +} + +int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { + int32_t code = 0; + int32_t capacity = TSDB_MAX_BINARY_LEN; + int32_t outBytes = 0; + char *result = (char *)calloc(1, capacity); + + switch (action->action) { + case SCHEMA_ACTION_ADD_COLUMN: { + int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_ADD_TAG: { + int n = sprintf(result, "alter stable %s add tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, + result+n, capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { + int n = sprintf(result, "alter stable %s modify column ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + } + case SCHEMA_ACTION_CHANGE_TAG_SIZE: { + int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); + buildColumnDescription(action->alterSTable.field, result+n, + capacity-n, &outBytes); + TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_CREATE_STABLE: { + int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); + char* pos = result + n; int freeBytes = capacity - n; + int numCols = taosArrayGetSize(action->createSTable.fields); + for (int32_t i = 0; i < numCols; ++i) { + SSchema* field = taosArrayGet(action->createSTable.fields, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + --pos; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ") tags ("); + int numTags = taosArrayGetSize(action->createSTable.tags); + pos += outBytes; freeBytes -= outBytes; + for (int32_t i = 0; i < numTags; ++i) { + SSchema* field = taosArrayGet(action->createSTable.tags, i); + buildColumnDescription(field, pos, freeBytes, &outBytes); + pos += outBytes; freeBytes -= outBytes; + *pos = ','; ++pos; --freeBytes; + } + pos--; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ")"); + TAOS_RES* res = taos_query(taos, result); + code = taos_errno(res); + break; + } + case SCHEMA_ACTION_CREATE_CTABLE: { + + break; + } + default: + break; + } + free(result); + return code; +} + +//todo: table/column length check +//todo: type check +//todo: taosmbs2ucs4 check From ca52c73bdeb9064a573857501035c648787a3521 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Fri, 9 Jul 2021 19:37:15 +0800 Subject: [PATCH 16/43] before taos_bind/taos_multi_bind generation --- src/client/src/tscParseLineProtocol.c | 99 +++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 79e8955f9a..10d2429b5e 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -277,6 +277,9 @@ typedef struct { uint8_t type; int16_t length; char* value; + + //=================================== + SSchema* fieldSchema; } TAOS_SML_KV; typedef struct { @@ -415,7 +418,6 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { } return code; - } typedef enum { @@ -465,7 +467,6 @@ int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); free(ucs); *bytes = bytesNeeded + VARSTR_HEADER_SIZE; - } else if (kv->type == TSDB_DATA_TYPE_BINARY) { *bytes = kv->length + VARSTR_HEADER_SIZE; } @@ -503,6 +504,9 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a pField = taosArrayPush(array, &field); taosHashPut(hash, field.name, tagKeyLen, &pField, POINTER_BYTES); } + + smlKv->fieldSchema = pField; + return 0; } @@ -700,9 +704,11 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { *pos = ','; ++pos; --freeBytes; } --pos; ++freeBytes; + outBytes = snprintf(pos, freeBytes, ") tags ("); - int numTags = taosArrayGetSize(action->createSTable.tags); pos += outBytes; freeBytes -= outBytes; + + int numTags = taosArrayGetSize(action->createSTable.tags); for (int32_t i = 0; i < numTags; ++i) { SSchema* field = taosArrayGet(action->createSTable.tags, i); buildColumnDescription(field, pos, freeBytes, &outBytes); @@ -716,7 +722,43 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { break; } case SCHEMA_ACTION_CREATE_CTABLE: { - +// SCreateCTableActionInfo* pInfo = &action->createCTable; +// SArray* bindParams = taosArrayInit(2 + 2 * pInfo->tagNum, sizeof(TAOS_BIND)); +// outBytes = sprintf(result, "create table ? using ?("); +// char* pos = result + outBytes; int32_t freeBytes = capacity-outBytes; +// uintptr_t lenSTableName = strlen(pInfo->sTableName); +// uintptr_t lenCTableName = strlen(pInfo->cTableName); +// TAOS_BIND tbCTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, +// .buffer = pInfo->cTableName, .length = &lenCTableName}; +// TAOS_BIND tbSTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, +// .buffer = pInfo->sTableName, .length = &lenSTableName}; +// taosArrayPush(bindParams, &tbCTableName); +// taosArrayPush(bindParams, &tbSTableName); +// for (int32_t i = 0; i < pInfo->tagNum; ++i) { +// outBytes = snprintf(pos, freeBytes, "?,"); +// +// TAOS_SML_KV* tagKv = pInfo->tags + i; +// TAOS_BIND tbTag = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, +// .buffer = tagKv->key, .length = }; +// pos += outBytes; freeBytes -= outBytes; +// } +// --pos; ++freeBytes; +// +// outBytes = snprintf(pos, freeBytes, ") tags ("); +// pos += outBytes; freeBytes -= outBytes; +// for (int32_t i = 0; i < pInfo->tagNum; ++i) { +// TAOS_SML_KV* tagKv = pInfo->tags + i; +// outBytes = snprintf(pos, freeBytes, "?,"); +// pos += outBytes; freeBytes -= outBytes; +// } +// pos--; ++freeBytes; +// outBytes = snprintf(pos, freeBytes, ")"); +// +// TAOS_STMT* stmt = taos_stmt_init(taos); +// taos_stmt_prepare(stmt, result, strlen(result)); +// +// +// taos_stmt_bind_param(stmt, (TAOS_BIND*)bindParams); break; } default: @@ -726,6 +768,55 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } +int32_t transformIntoPreparedStatement(SArray* points) { + size_t numPoints = taosArrayGetSize(points); + +// SHashObj* tag2bind = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +// SHashObj* field2multiBind = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int32_t i = 0; i < numPoints; ++i) { + TAOS_SML_DATA_POINT * point = taosArrayGet(points, i); + char tableKey[256]; + snprintf(tableKey, 256, "%s.%s", point->stableName, point->childTableName); + + } + return 0; +} + +int32_t insertBatch(TAOS* taos, const char* sTableName, char* cTableName, SSchema* tagsSchema, int numTags, TAOS_BIND* tagBind, + SSchema* colsSchema, int numCols, TAOS_MULTI_BIND* colBind) { + TAOS_STMT* stmt = taos_stmt_init(taos); + + char result[TSDB_MAX_BINARY_LEN] = {0}; + sprintf(result, "insert into ? using %s(", sTableName); + for (int i = 0; i < numTags; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", tagsSchema[i].name); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") tags ("); + + for (int i = 0; i < numTags; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", colsSchema[i].name); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ")"); + + int32_t code = 0; + code = taos_stmt_prepare(stmt, result, strlen(result)); + + code = taos_stmt_set_tbname_tags(stmt, cTableName, tagBind); + code = taos_stmt_bind_param_batch(stmt, colBind); + code = taos_stmt_execute(stmt); + return code; +} //todo: table/column length check //todo: type check //todo: taosmbs2ucs4 check From 35cba8c21f858a22b045cf7cf2ad5cedb2cd1631 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sat, 10 Jul 2021 16:18:38 +0800 Subject: [PATCH 17/43] before debugging --- src/client/src/tscParseLineProtocol.c | 393 ++++++++++++++------------ 1 file changed, 211 insertions(+), 182 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 10d2429b5e..2738fe6f7a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -272,6 +272,14 @@ TAOS_RES* taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { } //================================================================================================= +typedef struct { + char sTableName[TSDB_TABLE_NAME_LEN]; + SHashObj* tagHash; + SHashObj* fieldHash; + SArray* tags; //SArray + SArray* fields; //SArray +} SSmlSTableSchema; + typedef struct { char* key; uint8_t type; @@ -279,7 +287,7 @@ typedef struct { char* value; //=================================== - SSchema* fieldSchema; + SSchema* schema; } TAOS_SML_KV; typedef struct { @@ -293,17 +301,10 @@ typedef struct { TAOS_SML_KV* fields; int fieldNum; + //================================ + SSmlSTableSchema* schema; } TAOS_SML_DATA_POINT; -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - SHashObj* tagHash; - SHashObj* fieldHash; - SArray* tags; //SArray - SArray* fields; //SArray -} SSmlSTableSchema; - - int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -426,7 +427,6 @@ typedef enum { SCHEMA_ACTION_ADD_TAG, SCHEMA_ACTION_CHANGE_COLUMN_SIZE, SCHEMA_ACTION_CHANGE_TAG_SIZE, - SCHEMA_ACTION_CREATE_CTABLE } ESchemaAction; typedef struct { @@ -440,19 +440,11 @@ typedef struct { SSchema* field; } SAlterSTableActionInfo; -typedef struct { - char sTableName[TSDB_TABLE_NAME_LEN]; - char cTableName[TSDB_TABLE_NAME_LEN]; - TAOS_SML_KV* tags; - int tagNum; -} SCreateCTableActionInfo; - typedef struct { ESchemaAction action; union { SCreateSTableActionInfo createSTable; SAlterSTableActionInfo alterSTable; - SCreateCTableActionInfo createCTable; }; } SSchemaAction; @@ -505,7 +497,7 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a taosHashPut(hash, field.name, tagKeyLen, &pField, POINTER_BYTES); } - smlKv->fieldSchema = pField; + smlKv->schema = pField; return 0; } @@ -545,96 +537,6 @@ int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool return 0; } -int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { - int32_t code = TSDB_CODE_SUCCESS; - SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray - SHashObj* sname2shema = taosHashInit(32, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - for (int i = 0; i < numPoint; ++i) { - TAOS_SML_DATA_POINT* point = &points[i]; - SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); - SSmlSTableSchema* pStableSchema = NULL; - if (ppStableSchema) { - pStableSchema= *ppStableSchema; - } else { - SSmlSTableSchema schema; - size_t stableNameLen = strlen(point->stableName); - strncpy(schema.sTableName, point->stableName, stableNameLen); - schema.sTableName[stableNameLen] = '\0'; - schema.fields = taosArrayInit(64, sizeof(SSchema)); - schema.tags = taosArrayInit(8, sizeof(SSchema)); - schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - pStableSchema = taosArrayPush(stableArray, &schema); - taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); - } - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - addTaosFieldToHashAndArray(tagKv, pStableSchema->tagHash, pStableSchema->tags); - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* fieldKv = point->fields + j; - addTaosFieldToHashAndArray(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); - } - } - - SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); - size_t numStable = taosArrayGetSize(stableArray); - for (int i = 0; i < numStable; ++i) { - SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); - SSmlSTableSchema dbSchema = {0}; - dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); - dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); - dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { - SSchemaAction schemaAction = {0}; - schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; - memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); - memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); - schemaAction.createSTable.tags = pointSchema->tags; - schemaAction.createSTable.fields = pointSchema->fields; - taosArrayPush(schemaActions, &schemaAction); - }else if (code == TSDB_CODE_SUCCESS) { - size_t pointTagSize = taosArrayGetSize(pointSchema->tags); - size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); - - SHashObj* dbTagHash = dbSchema.tagHash; - SHashObj* dbFieldHash = dbSchema.fieldHash; - - for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); - if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); - } - } - - for (int j = 0; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); - if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); - } - } - } else { - return code; - } - } - - return code; -} - - int32_t buildColumnDescription(SSchema* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; @@ -721,46 +623,7 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { code = taos_errno(res); break; } - case SCHEMA_ACTION_CREATE_CTABLE: { -// SCreateCTableActionInfo* pInfo = &action->createCTable; -// SArray* bindParams = taosArrayInit(2 + 2 * pInfo->tagNum, sizeof(TAOS_BIND)); -// outBytes = sprintf(result, "create table ? using ?("); -// char* pos = result + outBytes; int32_t freeBytes = capacity-outBytes; -// uintptr_t lenSTableName = strlen(pInfo->sTableName); -// uintptr_t lenCTableName = strlen(pInfo->cTableName); -// TAOS_BIND tbCTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, -// .buffer = pInfo->cTableName, .length = &lenCTableName}; -// TAOS_BIND tbSTableName = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, -// .buffer = pInfo->sTableName, .length = &lenSTableName}; -// taosArrayPush(bindParams, &tbCTableName); -// taosArrayPush(bindParams, &tbSTableName); -// for (int32_t i = 0; i < pInfo->tagNum; ++i) { -// outBytes = snprintf(pos, freeBytes, "?,"); -// -// TAOS_SML_KV* tagKv = pInfo->tags + i; -// TAOS_BIND tbTag = {.is_null = NULL, .buffer_type = TSDB_DATA_TYPE_BINARY, -// .buffer = tagKv->key, .length = }; -// pos += outBytes; freeBytes -= outBytes; -// } -// --pos; ++freeBytes; -// -// outBytes = snprintf(pos, freeBytes, ") tags ("); -// pos += outBytes; freeBytes -= outBytes; -// for (int32_t i = 0; i < pInfo->tagNum; ++i) { -// TAOS_SML_KV* tagKv = pInfo->tags + i; -// outBytes = snprintf(pos, freeBytes, "?,"); -// pos += outBytes; freeBytes -= outBytes; -// } -// pos--; ++freeBytes; -// outBytes = snprintf(pos, freeBytes, ")"); -// -// TAOS_STMT* stmt = taos_stmt_init(taos); -// taos_stmt_prepare(stmt, result, strlen(result)); -// -// -// taos_stmt_bind_param(stmt, (TAOS_BIND*)bindParams); - break; - } + default: break; } @@ -768,55 +631,221 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } -int32_t transformIntoPreparedStatement(SArray* points) { - size_t numPoints = taosArrayGetSize(points); +int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsSchema, char* result, int16_t freeBytes) { + size_t numTags = taosArrayGetSize(tagsSchema); + size_t numCols = taosArrayGetSize(colsSchema); + sprintf(result, "insert into ? using %s(", sTableName); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") tags ("); -// SHashObj* tag2bind = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); -// SHashObj* field2multiBind = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + for (int i = 0; i < numTags; ++i) { + snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") ("); + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", colSchema->name); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); + } + snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); + return 0; +} + +int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* rowsBind) { + TAOS_STMT* stmt = taos_stmt_init(taos); + taos_stmt_prepare(stmt, sql, strlen(sql)); + + taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + size_t rows = taosArrayGetSize(rowsBind); + for (int32_t i = 0; i < rows; ++i) { + TAOS_BIND* colBind = taosArrayGetP(rowsBind, i); + taos_stmt_bind_param(stmt, colBind); + taos_stmt_add_batch(stmt); + } + + taos_stmt_execute(stmt); + TAOS_RES* res = taos_stmt_use_result(stmt); + return taos_errno(res); +} + +int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) { + SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); for (int32_t i = 0; i < numPoints; ++i) { - TAOS_SML_DATA_POINT * point = taosArrayGet(points, i); - char tableKey[256]; - snprintf(tableKey, 256, "%s.%s", point->stableName, point->childTableName); + TAOS_SML_DATA_POINT * point = points + i; + if (!point->childTableName) { + char childTableName[TSDB_TABLE_NAME_LEN]; + int32_t tableNameLen; + getChildTableName(point, childTableName, &tableNameLen); + point->childTableName = calloc(1, tableNameLen+1); + strncpy(point->childTableName, childTableName, tableNameLen); + point->childTableName[tableNameLen] = '\0'; + } + SArray* cTablePoints = NULL; + SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); + if (pCTablePoints) { + cTablePoints = *pCTablePoints; + } else { + cTablePoints = taosArrayInit(64, sizeof(point)); + taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); + } + taosArrayPush(cTablePoints, point); + } + SArray** pCTablePoints = taosHashIterate(cname2points, NULL); + while (pCTablePoints) { + SArray* cTablePoints = *pCTablePoints; + TAOS_SML_DATA_POINT * point = taosArrayGet(cTablePoints, 0); + int32_t numTags = taosArrayGetSize(point->schema->tags); + int32_t numCols = taosArrayGetSize(point->schema->fields); + char* stableName = point->stableName; + char* ctableName = point->childTableName; + char sql[TSDB_MAX_BINARY_LEN]; + getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); + + size_t rows = taosArrayGetSize(cTablePoints); + SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); + SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); + + for (int i = 0; i < rows; ++i) { + point = taosArrayGet(cTablePoints, i); + + taosArraySetSize(tagBinds, numTags); + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); + TAOS_BIND* bind = taosArrayGet(tagBinds, idx); + bind->buffer_type = kv->type; + bind->length = (uintptr_t*)&kv->length; + bind->buffer = kv->value; + } + + SArray* colBinds = taosArrayInit(numCols, sizeof(TAOS_BIND)); + taosArraySetSize(colBinds, numCols); + for (int j = 0; jfieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); + TAOS_BIND* bind = taosArrayGet(colBinds, idx); + bind->buffer_type = kv->type; + bind->length = (uintptr_t*)&kv->length; + bind->buffer = kv->value; + } + taosArrayPush(rowsBind, &colBinds); + } + + insertBatch(taos, sql, ctableName, tagBinds, rowsBind); + + pCTablePoints = taosHashIterate(cname2points, pCTablePoints); } return 0; } -int32_t insertBatch(TAOS* taos, const char* sTableName, char* cTableName, SSchema* tagsSchema, int numTags, TAOS_BIND* tagBind, - SSchema* colsSchema, int numCols, TAOS_MULTI_BIND* colBind) { - TAOS_STMT* stmt = taos_stmt_init(taos); - char result[TSDB_MAX_BINARY_LEN] = {0}; - sprintf(result, "insert into ? using %s(", sTableName); - for (int i = 0; i < numTags; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", tagsSchema[i].name); +int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + int32_t code = TSDB_CODE_SUCCESS; + SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray + SHashObj* sname2shema = taosHashInit(32, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int i = 0; i < numPoint; ++i) { + TAOS_SML_DATA_POINT* point = &points[i]; + SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); + SSmlSTableSchema* pStableSchema = NULL; + if (ppStableSchema) { + pStableSchema= *ppStableSchema; + } else { + SSmlSTableSchema schema; + size_t stableNameLen = strlen(point->stableName); + strncpy(schema.sTableName, point->stableName, stableNameLen); + schema.sTableName[stableNameLen] = '\0'; + schema.fields = taosArrayInit(64, sizeof(SSchema)); + schema.tags = taosArrayInit(8, sizeof(SSchema)); + schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + pStableSchema = taosArrayPush(stableArray, &schema); + taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); + } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + addTaosFieldToHashAndArray(tagKv, pStableSchema->tagHash, pStableSchema->tags); + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* fieldKv = point->fields + j; + addTaosFieldToHashAndArray(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); + } + + point->schema = pStableSchema; } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") tags ("); - for (int i = 0; i < numTags; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); + SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); + size_t numStable = taosArrayGetSize(stableArray); + for (int i = 0; i < numStable; ++i) { + SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); + SSmlSTableSchema dbSchema = {0}; + dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); + dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); + dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); + memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); + schemaAction.createSTable.tags = pointSchema->tags; + schemaAction.createSTable.fields = pointSchema->fields; + taosArrayPush(schemaActions, &schemaAction); + }else if (code == TSDB_CODE_SUCCESS) { + size_t pointTagSize = taosArrayGetSize(pointSchema->tags); + size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); + + SHashObj* dbTagHash = dbSchema.tagHash; + SHashObj* dbFieldHash = dbSchema.fieldHash; + + for (int j = 0; j < pointTagSize; ++j) { + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + + for (int j = 0; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + taosArrayPush(schemaActions, &schemaAction); + } + } + } else { + return code; + } } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") ("); - for (int i = 0; i < numCols; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "%s,", colsSchema[i].name); + for (int i = 0; i < taosArrayGetSize(schemaActions); ++i) { + SSchemaAction* action = taosArrayGet(schemaActions, i); + applySchemaAction(taos, action); } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ") values ("); - for (int i = 0; i < numCols; ++i) { - snprintf(result+strlen(result), TSDB_MAX_BINARY_LEN-strlen(result), "?,"); - } - snprintf(result + strlen(result)-1, TSDB_MAX_BINARY_LEN-strlen(result)+1, ")"); - - int32_t code = 0; - code = taos_stmt_prepare(stmt, result, strlen(result)); - - code = taos_stmt_set_tbname_tags(stmt, cTableName, tagBind); - code = taos_stmt_bind_param_batch(stmt, colBind); - code = taos_stmt_execute(stmt); + insertPoints(taos, points, numPoint); return code; } + + //todo: table/column length check //todo: type check //todo: taosmbs2ucs4 check From dbac6f2081636d78628191b4bf615c9289682b68 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 11 Jul 2021 23:52:19 +0800 Subject: [PATCH 18/43] before getChildTableName and insertBatch --- src/client/src/tscParseLineProtocol.c | 758 +++++++++++++++----------- src/inc/taos.h | 2 + tests/examples/c/apitest.c | 15 + 3 files changed, 447 insertions(+), 328 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 2738fe6f7a..3a88d2e906 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -15,263 +15,6 @@ #include "tskiplist.h" #include "tscUtil.h" -typedef enum { - LP_ITEM_TAG, - LP_ITEM_FIELD -} LPItemKind; - -typedef struct { - SStrToken key; - SStrToken value; - - char name[TSDB_COL_NAME_LEN]; - int8_t type; - int16_t bytes; - - char* payload; -}SLPItem; - -typedef struct { - SStrToken measToken; - SStrToken tsToken; - - char sTableName[TSDB_TABLE_NAME_LEN]; - SArray* tags; - SArray* fields; - int64_t ts; - -} SLPPoint; - -typedef enum { - LP_MEASUREMENT, - LP_TAG_KEY, - LP_TAG_VALUE, - LP_FIELD_KEY, - LP_FIELD_VALUE -} LPPart; - -int32_t scanToCommaOrSpace(SStrToken s, int32_t start, int32_t* index, LPPart part) { - for (int32_t i = start; i < s.n; ++i) { - if (s.z[i] == ',' || s.z[i] == ' ') { - *index = i; - return 0; - } - } - return -1; -} - -int32_t scanToEqual(SStrToken s, int32_t start, int32_t* index) { - for (int32_t i = start; i < s.n; ++i) { - if (s.z[i] == '=') { - *index = i; - return 0; - } - } - return -1; -} - -int32_t setPointMeasurement(SLPPoint* point, SStrToken token) { - point->measToken = token; - if (point->measToken.n < TSDB_TABLE_NAME_LEN) { - strncpy(point->sTableName, point->measToken.z, point->measToken.n); - point->sTableName[point->measToken.n] = '\0'; - } - return 0; -} - -int32_t setItemKey(SLPItem* item, SStrToken key, LPPart part) { - item->key = key; - if (item->key.n < TSDB_COL_NAME_LEN) { - strncpy(item->name, item->key.z, item->key.n); - item->name[item->key.n] = '\0'; - } - return 0; -} - -int32_t setItemValue(SLPItem* item, SStrToken value, LPPart part) { - item->value = value; - return 0; -} - -int32_t parseItemValue(SLPItem* item, LPItemKind kind) { - char* sv = item->value.z; - char* last = item->value.z + item->value.n - 1; - - if (isdigit(sv[0]) || sv[0] == '-') { - if (*last == 'i') { - item->type = TSDB_DATA_TYPE_BIGINT; - item->bytes = (int16_t)tDataTypes[item->type].bytes; - item->payload = malloc(item->bytes); - char* endptr = NULL; - *(item->payload) = strtoll(sv, &endptr, 10); - } else { - item->type = TSDB_DATA_TYPE_DOUBLE; - item->bytes = (int16_t)tDataTypes[item->type].bytes; - item->payload = malloc(item->bytes); - char* endptr = NULL; - *(item->payload) = strtold(sv, &endptr); - } - } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { - if (sv[0] == 'L') { - item->type = TSDB_DATA_TYPE_NCHAR; - uint32_t bytes = item->value.n - 3; -// uint32_t len = bytes; -// char* ucs = malloc(len); -// int32_t ncharBytes = 0; -// taosMbsToUcs4(sv+2, len, ucs, len, &ncharBytes); -// item->bytes = ncharBytes; -// item->payload = malloc(ncharBytes); -// memcpy(item->payload, ucs, ncharBytes); -// free(ucs); - item->bytes = bytes; - item->payload = malloc(bytes); - memcpy(item->payload, sv+1, bytes); - } else if (sv[0]=='"'){ - item->type = TSDB_DATA_TYPE_BINARY; - uint32_t bytes = item->value.n - 2; - item->bytes = bytes; - item->payload = malloc(bytes); - memcpy(item->payload, sv+1, bytes); - } - } else if (sv[0] == 't' || sv[0] == 'f' || sv[0]=='T' || sv[0] == 'F') { - item->type = TSDB_DATA_TYPE_BOOL; - item->bytes = tDataTypes[item->type].bytes; - item->payload = malloc(tDataTypes[item->type].bytes); - *(item->payload) = tolower(sv[0])=='t' ? true : false; - } - return 0; -} - -int32_t compareLPItemKey(const void* p1, const void* p2) { - const SLPItem* t1 = p1; - const SLPItem* t2 = p2; - uint32_t min = (t1->key.n < t2->key.n) ? t1->key.n : t2->key.n; - int res = strncmp(t1->key.z, t2->key.z, min); - if (res != 0) { - return res; - } else { - return (int)(t1->key.n) - (int)(t2->key.n); - } -} - -int32_t setPointTimeStamp(SLPPoint* point, SStrToken tsToken) { - point->tsToken = tsToken; - return 0; -} - -int32_t parsePointTime(SLPPoint* point) { - if (point->tsToken.n <= 0) { - point->ts = taosGetTimestampNs(); - } else { - char* endptr = NULL; - point->ts = strtoll(point->tsToken.z, &endptr, 10); - } - return 0; -} - -int32_t tscParseLine(SStrToken line, SLPPoint* point) { - int32_t pos = 0; - - int32_t start = 0; - int32_t err = scanToCommaOrSpace(line, start, &pos, LP_MEASUREMENT); - if (err != 0) { - tscError("a"); - return err; - } - - SStrToken measurement = {.z = line.z+start, .n = pos-start}; - setPointMeasurement(point, measurement); - point->tags = taosArrayInit(64, sizeof(SLPItem)); - start = pos + 1; - while (line.z[start] == ',') { - SLPItem item; - - err = scanToEqual(line, start, &pos); - if (err != 0) { - tscError("b"); - goto error; - } - - SStrToken tagKey = {.z = line.z + start, .n = pos-start}; - setItemKey(&item, tagKey, LP_TAG_KEY); - - start = pos + 1; - err = scanToCommaOrSpace(line, start, &pos, LP_TAG_VALUE); - if (err != 0) { - tscError("c"); - goto error; - } - - SStrToken tagValue = {.z = line.z + start, .n = pos-start}; - setItemValue(&item, tagValue, LP_TAG_VALUE); - - parseItemValue(&item, LP_ITEM_TAG); - taosArrayPush(point->tags, &item); - - start = pos + 1; - } - - taosArraySort(point->tags, compareLPItemKey); - - point->fields = taosArrayInit(64, sizeof(SLPItem)); - do { - SLPItem item; - err = scanToEqual(line, start, &pos); - if (err != 0) { - goto error; - } - SStrToken fieldKey = {.z = line.z + start, .n = pos- start}; - setItemKey(&item, fieldKey, LP_FIELD_KEY); - - start = pos + 1; - err = scanToCommaOrSpace(line, start, &pos, LP_FIELD_VALUE); - if (err != 0) { - goto error; - } - SStrToken fieldValue = {.z = line.z + start, .n = pos - start}; - setItemValue(&item, fieldValue, LP_TAG_VALUE); - - parseItemValue(&item, LP_ITEM_FIELD); - taosArrayPush(point->fields, &item); - - start = pos + 1; - } while (line.z[pos] == ','); - - taosArraySort(point->fields, compareLPItemKey); - - SStrToken tsToken = {.z = line.z+start, .n = line.n-start}; - setPointTimeStamp(point, tsToken); - parsePointTime(point); - - goto done; - -error: - // free array - return err; -done: - return 0; -} - - -int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { - for (int32_t i = 0; i < numLines; ++i) { - SStrToken tkLine = {.z = lines[i], .n = strlen(lines[i])+1}; - SLPPoint point; - tscParseLine(tkLine, &point); - taosArrayPush(points, &point); - } - return 0; -} - -TAOS_RES* taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { - SArray* points = taosArrayInit(numLines, sizeof(SLPPoint)); - tscParseLines(lines, numLines, points, NULL); - - - return NULL; -} -//================================================================================================= - typedef struct { char sTableName[TSDB_TABLE_NAME_LEN]; SHashObj* tagHash; @@ -305,6 +48,8 @@ typedef struct { SSmlSTableSchema* schema; } TAOS_SML_DATA_POINT; +//================================================================================================= + int compareSmlColKv(const void* p1, const void* p2) { TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1; TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2; @@ -318,32 +63,6 @@ int compareSmlColKv(const void* p1, const void* p2) { } } -int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { - qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); - - SStringBuilder sb; memset(&sb, 0, sizeof(sb)); - taosStringBuilderAppendString(&sb, point->stableName); - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - taosStringBuilderAppendChar(&sb, ','); - taosStringBuilderAppendString(&sb, tagKv->key); - taosStringBuilderAppendChar(&sb, '='); - taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); - } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); - MD5_CTX context; - MD5Init(&context); - MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); - MD5Final(&context); - *tableNameLen = snprintf(tableName, *tableNameLen, - "%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], - context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], - context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], - context.digest[12], context.digest[13], context.digest[14], context.digest[15]); - return 0; -} - int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { int32_t code = 0; @@ -393,15 +112,14 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { tscFreeSqlObj(pSql); - uint32_t size = tscGetTableMetaMaxSize(); STableMeta* tableMeta = calloc(1, size); taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); - tstrncpy(schema->sTableName, tableName, strlen(tableName)); + tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); for (int i=0; itableInfo.numOfColumns; ++i) { SSchema field; - tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)); + tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); field.type = tableMeta->schema[i].type; field.bytes = tableMeta->schema[i].bytes; SSchema* pField = taosArrayPush(schema->fields, &field); @@ -411,13 +129,13 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { for (int i=0; itableInfo.numOfTags; ++i) { int j = i + tableMeta->tableInfo.numOfColumns; SSchema field; - tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)); + tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); field.type = tableMeta->schema[j].type; field.bytes = tableMeta->schema[j].bytes; SSchema* pField = taosArrayPush(schema->tags, &field); taosHashPut(schema->tagHash, field.name, strlen(field.name), &pField, POINTER_BYTES); } - + free(tableMeta); tableMeta = NULL; return code; } @@ -586,6 +304,7 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { int n = sprintf(result, "alter stable %s modify tag ", action->alterSTable.sTableName); @@ -631,15 +350,46 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } +int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { + qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); + + SStringBuilder sb; memset(&sb, 0, sizeof(sb)); + taosStringBuilderAppendString(&sb, point->stableName); + for (int j = 0; j < point->tagNum; ++j) { + taosStringBuilderAppendChar(&sb, ','); + TAOS_SML_KV* tagKv = point->tags + j; + taosStringBuilderAppendString(&sb, tagKv->key); + taosStringBuilderAppendChar(&sb, '='); + taosStringBuilderAppend(&sb, tagKv->value, tagKv->length); + } + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); + MD5_CTX context; + MD5Init(&context); + MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); + MD5Final(&context); + *tableNameLen = snprintf(tableName, *tableNameLen, + "tbl%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], + context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], + context.digest[12], context.digest[13], context.digest[14], context.digest[15]); + taosStringBuilderDestroy(&sb); + return 0; +} + int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsSchema, char* result, int16_t freeBytes) { size_t numTags = taosArrayGetSize(tagsSchema); size_t numCols = taosArrayGetSize(colsSchema); - sprintf(result, "insert into ? using %s(", sTableName); - for (int i = 0; i < numTags; ++i) { - SSchema* tagSchema = taosArrayGet(tagsSchema, i); - snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); - } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") tags ("); + sprintf(result, "insert into ? using %s", sTableName); + +// snprintf(result+strlen(result), freeBytes-strlen(result), "("); +// for (int i = 0; i < numTags; ++i) { +// SSchema* tagSchema = taosArrayGet(tagsSchema, i); +// snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); +// } +// snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); + + snprintf(result + strlen(result), freeBytes-strlen(result), " tags ("); for (int i = 0; i < numTags; ++i) { snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); @@ -661,17 +411,38 @@ int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsS int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* rowsBind) { TAOS_STMT* stmt = taos_stmt_init(taos); - taos_stmt_prepare(stmt, sql, strlen(sql)); - - taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); - size_t rows = taosArrayGetSize(rowsBind); - for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colBind = taosArrayGetP(rowsBind, i); - taos_stmt_bind_param(stmt, colBind); - taos_stmt_add_batch(stmt); + int32_t code; + code = taos_stmt_prepare(stmt, sql, strlen(sql)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; } - taos_stmt_execute(stmt); + code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + size_t rows = taosArrayGetSize(rowsBind); + for (int32_t i = 0; i < rows; ++i) { + SArray* colBind = taosArrayGetP(rowsBind, i); + code = taos_stmt_bind_param(stmt, TARRAY_GET_START(colBind)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } TAOS_RES* res = taos_stmt_use_result(stmt); return taos_errno(res); } @@ -682,7 +453,7 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) TAOS_SML_DATA_POINT * point = points + i; if (!point->childTableName) { char childTableName[TSDB_TABLE_NAME_LEN]; - int32_t tableNameLen; + int32_t tableNameLen = TSDB_TABLE_NAME_LEN; getChildTableName(point, childTableName, &tableNameLen); point->childTableName = calloc(1, tableNameLen+1); strncpy(point->childTableName, childTableName, tableNameLen); @@ -696,50 +467,62 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) cTablePoints = taosArrayInit(64, sizeof(point)); taosHashPut(cname2points, point->childTableName, strlen(point->childTableName), &cTablePoints, POINTER_BYTES); } - taosArrayPush(cTablePoints, point); + taosArrayPush(cTablePoints, &point); } + int isNullColBind = TSDB_TRUE; SArray** pCTablePoints = taosHashIterate(cname2points, NULL); while (pCTablePoints) { SArray* cTablePoints = *pCTablePoints; - TAOS_SML_DATA_POINT * point = taosArrayGet(cTablePoints, 0); + + TAOS_SML_DATA_POINT * point = taosArrayGetP(cTablePoints, 0); int32_t numTags = taosArrayGetSize(point->schema->tags); int32_t numCols = taosArrayGetSize(point->schema->fields); char* stableName = point->stableName; char* ctableName = point->childTableName; - char sql[TSDB_MAX_BINARY_LEN]; - getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); + + SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); + taosArraySetSize(tagBinds, numTags); + for (int j = 0; j < numTags; ++j) { + TAOS_BIND* bind = taosArrayGet(tagBinds, j); + bind->is_null = &isNullColBind; + } + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); + TAOS_BIND* bind = taosArrayGet(tagBinds, idx); + bind->buffer_type = kv->type; + bind->length = (uintptr_t*)&kv->length; + bind->buffer = kv->value; + bind->is_null = NULL; + } size_t rows = taosArrayGetSize(cTablePoints); SArray* rowsBind = taosArrayInit(rows, POINTER_BYTES); - SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); for (int i = 0; i < rows; ++i) { - point = taosArrayGet(cTablePoints, i); - - taosArraySetSize(tagBinds, numTags); - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* kv = point->tags + j; - int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); - TAOS_BIND* bind = taosArrayGet(tagBinds, idx); - bind->buffer_type = kv->type; - bind->length = (uintptr_t*)&kv->length; - bind->buffer = kv->value; - } + point = taosArrayGetP(cTablePoints, i); SArray* colBinds = taosArrayInit(numCols, sizeof(TAOS_BIND)); taosArraySetSize(colBinds, numCols); - for (int j = 0; jfieldNum; ++j) { + for (int j = 0; j < numCols; ++j) { + TAOS_BIND* bind = taosArrayGet(colBinds, j); + bind->is_null = &isNullColBind; + } + for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); TAOS_BIND* bind = taosArrayGet(colBinds, idx); bind->buffer_type = kv->type; bind->length = (uintptr_t*)&kv->length; bind->buffer = kv->value; + bind->is_null = NULL; } taosArrayPush(rowsBind, &colBinds); } + char sql[TSDB_MAX_BINARY_LEN]; + getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); insertBatch(taos, sql, ctableName, tagBinds, rowsBind); pCTablePoints = taosHashIterate(cname2points, pCTablePoints); @@ -747,7 +530,6 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) return 0; } - int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { int32_t code = TSDB_CODE_SUCCESS; SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray @@ -792,11 +574,13 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { for (int i = 0; i < numStable; ++i) { SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); SSmlSTableSchema dbSchema = {0}; - dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); + dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; @@ -813,19 +597,23 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { SHashObj* dbFieldHash = dbSchema.fieldHash; for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); SSchemaAction schemaAction = {0}; - bool actionNeeded = false; + bool actionNeeded = false; generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { taosArrayPush(schemaActions, &schemaAction); } } - for (int j = 0; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->tags, j); + SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); + SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); + memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); + + for (int j = 1; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->fields, j); SSchemaAction schemaAction = {0}; - bool actionNeeded = false; + bool actionNeeded = false; generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { taosArrayPush(schemaActions, &schemaAction); @@ -849,3 +637,317 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { //todo: table/column length check //todo: type check //todo: taosmbs2ucs4 check + +//========================================================================= + +typedef enum { + LP_ITEM_TAG, + LP_ITEM_FIELD +} LPItemKind; + +typedef struct { + SStrToken keyToken; + SStrToken valueToken; + + char key[TSDB_COL_NAME_LEN]; + int8_t type; + int16_t length; + + char* value; +}SLPItem; + +typedef struct { + SStrToken measToken; + SStrToken tsToken; + + char sTableName[TSDB_TABLE_NAME_LEN]; + SArray* tags; + SArray* fields; + int64_t ts; + +} SLPPoint; + +typedef enum { + LP_MEASUREMENT, + LP_TAG_KEY, + LP_TAG_VALUE, + LP_FIELD_KEY, + LP_FIELD_VALUE +} LPPart; + +int32_t scanToCommaOrSpace(SStrToken s, int32_t start, int32_t* index, LPPart part) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == ',' || s.z[i] == ' ') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t scanToEqual(SStrToken s, int32_t start, int32_t* index) { + for (int32_t i = start; i < s.n; ++i) { + if (s.z[i] == '=') { + *index = i; + return 0; + } + } + return -1; +} + +int32_t setPointMeasurement(SLPPoint* point, SStrToken token) { + point->measToken = token; + if (point->measToken.n < TSDB_TABLE_NAME_LEN) { + strncpy(point->sTableName, point->measToken.z, point->measToken.n); + point->sTableName[point->measToken.n] = '\0'; + } + return 0; +} + +int32_t setItemKey(SLPItem* item, SStrToken key, LPPart part) { + item->keyToken = key; + if (item->keyToken.n < TSDB_COL_NAME_LEN) { + strncpy(item->key, item->keyToken.z, item->keyToken.n); + item->key[item->keyToken.n] = '\0'; + } + return 0; +} + +int32_t setItemValue(SLPItem* item, SStrToken value, LPPart part) { + item->valueToken = value; + return 0; +} + +int32_t parseItemValue(SLPItem* item, LPItemKind kind) { + char* sv = item->valueToken.z; + char* last = item->valueToken.z + item->valueToken.n - 1; + + if (isdigit(sv[0]) || sv[0] == '-') { + if (*last == 'i') { + item->type = TSDB_DATA_TYPE_BIGINT; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(item->value) = strtoll(sv, &endptr, 10); + } else { + item->type = TSDB_DATA_TYPE_DOUBLE; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(item->value) = strtold(sv, &endptr); + } + } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { + if (sv[0] == 'L') { + item->type = TSDB_DATA_TYPE_NCHAR; + uint32_t bytes = item->valueToken.n - 3; + item->length = bytes; + item->value = malloc(bytes); + memcpy(item->value, sv+1, bytes); + } else if (sv[0]=='"'){ + item->type = TSDB_DATA_TYPE_BINARY; + uint32_t bytes = item->valueToken.n - 2; + item->length = bytes; + item->value = malloc(bytes); + memcpy(item->value, sv+1, bytes); + } + } else if (sv[0] == 't' || sv[0] == 'f' || sv[0]=='T' || sv[0] == 'F') { + item->type = TSDB_DATA_TYPE_BOOL; + item->length = tDataTypes[item->type].bytes; + item->value = malloc(tDataTypes[item->type].bytes); + *(item->value) = tolower(sv[0])=='t' ? TSDB_TRUE : TSDB_FALSE; + } + return 0; +} + +int32_t compareLPItemKey(const void* p1, const void* p2) { + const SLPItem* t1 = p1; + const SLPItem* t2 = p2; + uint32_t min = (t1->keyToken.n < t2->keyToken.n) ? t1->keyToken.n : t2->keyToken.n; + int res = strncmp(t1->keyToken.z, t2->keyToken.z, min); + if (res != 0) { + return res; + } else { + return (int)(t1->keyToken.n) - (int)(t2->keyToken.n); + } +} + +int32_t setPointTimeStamp(SLPPoint* point, SStrToken tsToken) { + point->tsToken = tsToken; + return 0; +} + +int32_t parsePointTime(SLPPoint* point) { + if (point->tsToken.n <= 0) { + point->ts = taosGetTimestampNs(); + } else { + char* endptr = NULL; + point->ts = strtoll(point->tsToken.z, &endptr, 10); + } + return 0; +} + +int32_t tscParseLine(SStrToken line, SLPPoint* point) { + int32_t pos = 0; + + int32_t start = 0; + int32_t err = scanToCommaOrSpace(line, start, &pos, LP_MEASUREMENT); + if (err != 0) { + tscError("a"); + return err; + } + + SStrToken measurement = {.z = line.z+start, .n = pos-start}; + setPointMeasurement(point, measurement); + point->tags = taosArrayInit(64, sizeof(SLPItem)); + start = pos; + while (line.z[start] == ',') { + SLPItem item; + + start++; + err = scanToEqual(line, start, &pos); + if (err != 0) { + tscError("b"); + goto error; + } + + SStrToken tagKey = {.z = line.z + start, .n = pos-start}; + setItemKey(&item, tagKey, LP_TAG_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_TAG_VALUE); + if (err != 0) { + tscError("c"); + goto error; + } + + SStrToken tagValue = {.z = line.z + start, .n = pos-start}; + setItemValue(&item, tagValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_TAG); + taosArrayPush(point->tags, &item); + + start = pos; + } + + taosArraySort(point->tags, compareLPItemKey); + + point->fields = taosArrayInit(64, sizeof(SLPItem)); + + start++; + do { + SLPItem item; + + err = scanToEqual(line, start, &pos); + if (err != 0) { + goto error; + } + SStrToken fieldKey = {.z = line.z + start, .n = pos- start}; + setItemKey(&item, fieldKey, LP_FIELD_KEY); + + start = pos + 1; + err = scanToCommaOrSpace(line, start, &pos, LP_FIELD_VALUE); + if (err != 0) { + goto error; + } + SStrToken fieldValue = {.z = line.z + start, .n = pos - start}; + setItemValue(&item, fieldValue, LP_TAG_VALUE); + + parseItemValue(&item, LP_ITEM_FIELD); + taosArrayPush(point->fields, &item); + + start = pos + 1; + } while (line.z[pos] == ','); + + taosArraySort(point->fields, compareLPItemKey); + + SStrToken tsToken = {.z = line.z+start, .n = line.n-start}; + setPointTimeStamp(point, tsToken); + parsePointTime(point); + + goto done; + + error: + // free array + return err; + done: + return 0; +} + + +int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { + for (int32_t i = 0; i < numLines; ++i) { + SStrToken tkLine = {.z = lines[i], .n = strlen(lines[i])+1}; + SLPPoint point; + tscParseLine(tkLine, &point); + taosArrayPush(points, &point); + } + return 0; +} + +int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { + SArray* lpPoints = taosArrayInit(numLines, sizeof(SLPPoint)); + tscParseLines(lines, numLines, lpPoints, NULL); + + size_t numPoints = taosArrayGetSize(lpPoints); + TAOS_SML_DATA_POINT* points = calloc(numPoints, sizeof(TAOS_SML_DATA_POINT)); + for (int i = 0; i < numPoints; ++i) { + SLPPoint* lpPoint = taosArrayGet(lpPoints, i); + TAOS_SML_DATA_POINT* point = points+i; + point->stableName = calloc(1, strlen(lpPoint->sTableName)+1); + strncpy(point->stableName, lpPoint->sTableName, strlen(lpPoint->sTableName)); + point->stableName[strlen(lpPoint->sTableName)] = '\0'; + + size_t lpTagSize = taosArrayGetSize(lpPoint->tags); + point->tags = calloc(lpTagSize, sizeof(TAOS_SML_KV)); + point->tagNum = lpTagSize; + for (int j=0; jtags, j); + TAOS_SML_KV* tagKv = point->tags + j; + + size_t kenLen = strlen(lpTag->key); + tagKv->key = calloc(1, kenLen+1); + strncpy(tagKv->key, lpTag->key, kenLen); + tagKv->key[kenLen] = '\0'; + + tagKv->type = lpTag->type; + tagKv->length = lpTag->length; + tagKv->value = malloc(tagKv->length); + memcpy(tagKv->value, lpTag->value, tagKv->length); + } + + size_t lpFieldsSize = taosArrayGetSize(lpPoint->fields); + point->fields = calloc(lpFieldsSize + 1, sizeof(TAOS_SML_KV)); + point->fieldNum = lpFieldsSize + 1; + + TAOS_SML_KV* tsField = point->fields + 0; + char tsKey[256]; + snprintf(tsKey, 256, "_%s_ts", point->stableName); + size_t tsKeyLen = strlen(tsKey); + tsField->key = calloc(1, tsKeyLen+1); + strncpy(tsField->key, tsKey, tsKeyLen); + tsField->key[tsKeyLen] = '\0'; + tsField->type = TSDB_DATA_TYPE_TIMESTAMP; + tsField->length = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes; + tsField->value = malloc(tsField->length); + memcpy(tsField->value, &(lpPoint->ts), tsField->length); + + for (int j=0; jfields, j); + TAOS_SML_KV* fieldKv = point->fields + j + 1; + + size_t kenLen = strlen(lpField->key); + fieldKv->key = calloc(1, kenLen+1); + strncpy(fieldKv->key, lpField->key, kenLen); + fieldKv->key[kenLen] = '\0'; + + fieldKv->type = lpField->type; + fieldKv->length = lpField->length; + fieldKv->value = malloc(fieldKv->length); + memcpy(fieldKv->value, lpField->value, fieldKv->length); + } + } + + taos_sml_insert(taos, points, numPoints); + return 0; +} + diff --git a/src/inc/taos.h b/src/inc/taos.h index 9f72945ef0..ca18c4fb93 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -169,6 +169,8 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); +DLL_EXPORT int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines); + #ifdef __cplusplus } #endif diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 0f24df0f47..dc77677774 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -949,6 +949,15 @@ void verify_stream(TAOS* taos) { taos_close_stream(strm); } +int32_t verify_schema_less(TAOS* taos) { + prepare_data(taos); + char* lines[] = { + "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639162922" + }; + int code = taos_insert_by_lines(taos, lines , 1); + return code; +} + int main(int argc, char *argv[]) { const char* host = "127.0.0.1"; const char* user = "root"; @@ -967,6 +976,12 @@ int main(int argc, char *argv[]) { info = taos_get_client_info(taos); printf("client info: %s\n", info); + printf("************ verify query *************\n"); + int code = verify_schema_less(taos); + if (code == 0) { + return code; + } + printf("************ verify query *************\n"); verify_query(taos); From ee10305ddc472b5480587bd2c225f4645fdc9fb1 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 00:39:16 +0800 Subject: [PATCH 19/43] before time precision and create table with tags instead of autocreate --- src/client/src/tscParseLineProtocol.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 3a88d2e906..4b9191a91f 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -492,7 +492,8 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); TAOS_BIND* bind = taosArrayGet(tagBinds, idx); bind->buffer_type = kv->type; - bind->length = (uintptr_t*)&kv->length; + bind->length = malloc(sizeof(uintptr_t*)); + *bind->length = kv->length; bind->buffer = kv->value; bind->is_null = NULL; } @@ -514,7 +515,8 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); TAOS_BIND* bind = taosArrayGet(colBinds, idx); bind->buffer_type = kv->type; - bind->length = (uintptr_t*)&kv->length; + bind->length = malloc(sizeof(uintptr_t*)); + *bind->length = kv->length; bind->buffer = kv->value; bind->is_null = NULL; } From 93c6fee9c5295cf3246fff619f2e1fbc378645ac Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 08:38:51 +0800 Subject: [PATCH 20/43] can add two lines of one supertable/diffrent subtable --- src/client/src/tscParseLineProtocol.c | 12 ++++++------ tests/examples/c/apitest.c | 8 +++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 4b9191a91f..1fde741c2b 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -540,13 +540,13 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { for (int i = 0; i < numPoint; ++i) { TAOS_SML_DATA_POINT* point = &points[i]; - SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, TSDB_TABLE_NAME_LEN); + size_t stableNameLen = strlen(point->stableName); + SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, stableNameLen); SSmlSTableSchema* pStableSchema = NULL; if (ppStableSchema) { pStableSchema= *ppStableSchema; } else { SSmlSTableSchema schema; - size_t stableNameLen = strlen(point->stableName); strncpy(schema.sTableName, point->stableName, stableNameLen); schema.sTableName[stableNameLen] = '\0'; schema.fields = taosArrayInit(64, sizeof(SSchema)); @@ -730,13 +730,13 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(item->value) = strtoll(sv, &endptr, 10); + *(int64_t*)(item->value) = strtoll(sv, &endptr, 10); } else { item->type = TSDB_DATA_TYPE_DOUBLE; item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(item->value) = strtold(sv, &endptr); + *(double*)(item->value) = strtold(sv, &endptr); } } else if ((sv[0] == 'L' && sv[1] =='"') || sv[0] == '"' ) { if (sv[0] == 'L') { @@ -744,7 +744,7 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { uint32_t bytes = item->valueToken.n - 3; item->length = bytes; item->value = malloc(bytes); - memcpy(item->value, sv+1, bytes); + memcpy(item->value, sv+2, bytes); } else if (sv[0]=='"'){ item->type = TSDB_DATA_TYPE_BINARY; uint32_t bytes = item->valueToken.n - 2; @@ -756,7 +756,7 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { item->type = TSDB_DATA_TYPE_BOOL; item->length = tDataTypes[item->type].bytes; item->value = malloc(tDataTypes[item->type].bytes); - *(item->value) = tolower(sv[0])=='t' ? TSDB_TRUE : TSDB_FALSE; + *(uint8_t*)(item->value) = tolower(sv[0])=='t' ? TSDB_TRUE : TSDB_FALSE; } return 0; } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index dc77677774..04a03df6fa 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -952,9 +952,11 @@ void verify_stream(TAOS* taos) { int32_t verify_schema_less(TAOS* taos) { prepare_data(taos); char* lines[] = { - "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639162922" + "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642" }; - int code = taos_insert_by_lines(taos, lines , 1); + int code = taos_insert_by_lines(taos, lines , 2); return code; } @@ -976,7 +978,7 @@ int main(int argc, char *argv[]) { info = taos_get_client_info(taos); printf("client info: %s\n", info); - printf("************ verify query *************\n"); + printf("************ verify shemaless *************\n"); int code = verify_schema_less(taos); if (code == 0) { return code; From 3d70319913feb1a486f47542e4b9e3412e045bcc Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 09:17:15 +0800 Subject: [PATCH 21/43] create child table with tags --- src/client/src/tscParseLineProtocol.c | 87 +++++++++++++++++---------- tests/examples/c/apitest.c | 2 +- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 1fde741c2b..dd0e64ba84 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -377,39 +377,26 @@ int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tabl return 0; } -int32_t getPreparedSQL(const char* sTableName, SArray* tagsSchema, SArray* colsSchema, char* result, int16_t freeBytes) { +int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind) { size_t numTags = taosArrayGetSize(tagsSchema); - size_t numCols = taosArrayGetSize(colsSchema); - sprintf(result, "insert into ? using %s", sTableName); + char sql[TSDB_MAX_BINARY_LEN] = {0}; + int freeBytes = TSDB_MAX_BINARY_LEN; + sprintf(sql, "create table if not exists %s using %s", cTableName, sTableName); -// snprintf(result+strlen(result), freeBytes-strlen(result), "("); -// for (int i = 0; i < numTags; ++i) { -// SSchema* tagSchema = taosArrayGet(tagsSchema, i); -// snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", tagSchema->name); -// } -// snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "("); + for (int i = 0; i < numTags; ++i) { + SSchema* tagSchema = taosArrayGet(tagsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", tagSchema->name); + } + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - snprintf(result + strlen(result), freeBytes-strlen(result), " tags ("); + snprintf(sql + strlen(sql), freeBytes-strlen(sql), " tags ("); for (int i = 0; i < numTags; ++i) { - snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") ("); + snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); - for (int i = 0; i < numCols; ++i) { - SSchema* colSchema = taosArrayGet(colsSchema, i); - snprintf(result+strlen(result), freeBytes-strlen(result), "%s,", colSchema->name); - } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ") values ("); - - for (int i = 0; i < numCols; ++i) { - snprintf(result+strlen(result), freeBytes-strlen(result), "?,"); - } - snprintf(result + strlen(result)-1, freeBytes-strlen(result)+1, ")"); - return 0; -} - -int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, SArray* rowsBind) { TAOS_STMT* stmt = taos_stmt_init(taos); int32_t code; code = taos_stmt_prepare(stmt, sql, strlen(sql)); @@ -418,7 +405,47 @@ int32_t insertBatch(TAOS* taos, char* sql, char* cTableName, SArray* tagsBind, S return code; } - code = taos_stmt_set_tbname_tags(stmt, cTableName, TARRAY_GET_START(tagsBind)); + code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + TAOS_RES* res = taos_stmt_use_result(stmt); + return taos_errno(res); +} + +int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind) { + size_t numCols = taosArrayGetSize(colsSchema); + char sql[TSDB_MAX_BINARY_LEN]; + int32_t freeBytes = TSDB_MAX_BINARY_LEN; + sprintf(sql, "insert into ? ("); + + for (int i = 0; i < numCols; ++i) { + SSchema* colSchema = taosArrayGet(colsSchema, i); + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "%s,", colSchema->name); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ") values ("); + + for (int i = 0; i < numCols; ++i) { + snprintf(sql+strlen(sql), freeBytes-strlen(sql), "?,"); + } + snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); + + TAOS_STMT* stmt = taos_stmt_init(taos); + int32_t code; + code = taos_stmt_prepare(stmt, sql, strlen(sql)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + + code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { printf("%s", taos_stmt_errstr(stmt)); return code; @@ -478,7 +505,6 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) TAOS_SML_DATA_POINT * point = taosArrayGetP(cTablePoints, 0); int32_t numTags = taosArrayGetSize(point->schema->tags); int32_t numCols = taosArrayGetSize(point->schema->fields); - char* stableName = point->stableName; char* ctableName = point->childTableName; SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); @@ -523,9 +549,8 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) taosArrayPush(rowsBind, &colBinds); } - char sql[TSDB_MAX_BINARY_LEN]; - getPreparedSQL(stableName, point->schema->tags, point->schema->fields, sql, TSDB_MAX_BINARY_LEN); - insertBatch(taos, sql, ctableName, tagBinds, rowsBind); + creatChildTableIfNotExists(taos, point->childTableName, point->stableName, point->schema->tags, tagBinds); + insertBatch(taos, ctableName, point->schema->fields, rowsBind); pCTablePoints = taosHashIterate(cname2points, pCTablePoints); } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 04a03df6fa..5272dafaf1 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -956,7 +956,7 @@ int32_t verify_schema_less(TAOS* taos) { "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642" }; - int code = taos_insert_by_lines(taos, lines , 2); + int code = taos_insert_by_lines(taos, lines , 3); return code; } From dec30e4473346b7d4529c6633549ffcb0867501a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 10:16:30 +0800 Subject: [PATCH 22/43] add timestamp precision support --- src/client/src/tscParseLineProtocol.c | 39 ++++++++++++++++++++------- tests/examples/c/apitest.c | 18 +++++++++---- 2 files changed, 42 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index dd0e64ba84..c0cc2ea3ae 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -21,6 +21,7 @@ typedef struct { SHashObj* fieldHash; SArray* tags; //SArray SArray* fields; //SArray + uint8_t precision; } SSmlSTableSchema; typedef struct { @@ -117,6 +118,7 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); + schema->precision = tableMeta->tableInfo.precision; for (int i=0; itableInfo.numOfColumns; ++i) { SSchema field; tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); @@ -486,6 +488,25 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) strncpy(point->childTableName, childTableName, tableNameLen); point->childTableName[tableNameLen] = '\0'; } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* kv = point->tags + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, point->schema->precision); + *(int64_t*)(kv->value) = ts; + } + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* kv = point->fields + j; + if (kv->type == TSDB_DATA_TYPE_TIMESTAMP) { + int64_t ts = *(int64_t*)(kv->value); + ts = convertTimePrecision(ts, TSDB_TIME_PRECISION_NANO, point->schema->precision); + *(int64_t*)(kv->value) = ts; + } + } + SArray* cTablePoints = NULL; SArray** pCTablePoints = taosHashGet(cname2points, point->childTableName, strlen(point->childTableName)); if (pCTablePoints) { @@ -596,7 +617,6 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { point->schema = pStableSchema; } - SArray* schemaActions = taosArrayInit(32, sizeof(SSchemaAction)); size_t numStable = taosArrayGetSize(stableArray); for (int i = 0; i < numStable; ++i) { SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); @@ -615,8 +635,10 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); schemaAction.createSTable.tags = pointSchema->tags; schemaAction.createSTable.fields = pointSchema->fields; - taosArrayPush(schemaActions, &schemaAction); - }else if (code == TSDB_CODE_SUCCESS) { + applySchemaAction(taos, &schemaAction); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + pointSchema->precision = dbSchema.precision; + } else if (code == TSDB_CODE_SUCCESS) { size_t pointTagSize = taosArrayGetSize(pointSchema->tags); size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); @@ -629,7 +651,7 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { bool actionNeeded = false; generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); + applySchemaAction(taos, &schemaAction); } } @@ -643,19 +665,16 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { bool actionNeeded = false; generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { - taosArrayPush(schemaActions, &schemaAction); + applySchemaAction(taos, &schemaAction); } } + + pointSchema->precision = dbSchema.precision; } else { return code; } } - for (int i = 0; i < taosArrayGetSize(schemaActions); ++i) { - SSchemaAction* action = taosArrayGet(schemaActions, i); - applySchemaAction(taos, action); - } - insertPoints(taos, points, numPoint); return code; } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 5272dafaf1..656d5fd217 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -12,7 +12,7 @@ static void prepare_data(TAOS* taos) { result = taos_query(taos, "drop database if exists test;"); taos_free_result(result); usleep(100000); - result = taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test precision 'us';"); taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); @@ -950,11 +950,19 @@ void verify_stream(TAOS* taos) { } int32_t verify_schema_less(TAOS* taos) { - prepare_data(taos); + TAOS_RES *result; + result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); + usleep(100000); + result = taos_query(taos, "create database test precision 'us';"); + taos_free_result(result); + usleep(100000); + taos_select_db(taos, "test"); + char* lines[] = { - "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642" + "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000" }; int code = taos_insert_by_lines(taos, lines , 3); return code; From ab67d612f0ee3e365b1357ffa5ec3aef4936b6b2 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 10:30:54 +0800 Subject: [PATCH 23/43] fixing exsiting stable --- src/client/src/tscParseLineProtocol.c | 2 +- tests/examples/c/apitest.c | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index c0cc2ea3ae..61f9cd9e76 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -225,7 +225,7 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool isTag, char sTableName[], SSchemaAction* action, bool* actionNeeded) { SSchema** ppDbAttr = taosHashGet(dbAttrHash, pointColField->name, strlen(pointColField->name)); - if (*ppDbAttr) { + if (ppDbAttr) { SSchema* dbAttr = *ppDbAttr; if (pointColField->type != dbAttr->type) { //todo error diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 656d5fd217..5cb3f5762a 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -958,13 +958,15 @@ int32_t verify_schema_less(TAOS* taos) { taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); + result = taos_query(taos, "create stable ste(ts timestamp, f int) tags(t1 bigint)"); char* lines[] = { "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000" + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", + "ste,t2=5,t3=L\"ste\" c1=true,c2=4 1626056811823316532" }; - int code = taos_insert_by_lines(taos, lines , 3); + int code = taos_insert_by_lines(taos, lines , 4); return code; } From 14cab541f7cb6235a8b5ab0d16fc1a1aacb124ed Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 11:27:34 +0800 Subject: [PATCH 24/43] before tsim/log/memory leak handling --- tests/examples/c/apitest.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 5cb3f5762a..409140e0f2 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -957,16 +957,20 @@ int32_t verify_schema_less(TAOS* taos) { result = taos_query(taos, "create database test precision 'us';"); taos_free_result(result); usleep(100000); + taos_select_db(taos, "test"); result = taos_query(taos, "create stable ste(ts timestamp, f int) tags(t1 bigint)"); + taos_free_result(result); + usleep(100000); char* lines[] = { "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", - "ste,t2=5,t3=L\"ste\" c1=true,c2=4 1626056811823316532" + "ste,t2=5,t3=L\"ste\" c1=true,c2=4,c3=\"iam\" 1626056811823316532", + "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532" }; - int code = taos_insert_by_lines(taos, lines , 4); + int code = taos_insert_by_lines(taos, lines , 5); return code; } From aee50c892962fbd750591e81ecc9280e9fdc9c4d Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 12 Jul 2021 18:09:50 +0800 Subject: [PATCH 25/43] add unit test --- src/client/src/tscParseLineProtocol.c | 46 ++++++++++++++----------- tests/examples/c/apitest.c | 18 ++++++++-- tests/tsim/inc/sim.h | 4 +++ tests/tsim/src/simExe.c | 46 +++++++++++++++++++++++++ tests/tsim/src/simParse.c | 48 +++++++++++++++++++++++++++ 5 files changed, 139 insertions(+), 23 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 61f9cd9e76..77d65f3d15 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -65,6 +65,11 @@ int compareSmlColKv(const void* p1, const void* p2) { } int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { + schema->tags = taosArrayInit(8, sizeof(SSchema)); + schema->fields = taosArrayInit(64, sizeof(SSchema)); + schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + int32_t code = 0; STscObj *pObj = (STscObj *)taos; @@ -203,7 +208,7 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a pField->bytes = MAX(pField->bytes, bytes); } else { - SSchema field; + SSchema field = {0}; size_t tagKeyLen = strlen(smlKv->key); strncpy(field.name, smlKv->key, tagKeyLen); field.name[tagKeyLen] = '\0'; @@ -371,7 +376,7 @@ int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tabl MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); MD5Final(&context); *tableNameLen = snprintf(tableName, *tableNameLen, - "tbl%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + "tbl_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); @@ -418,14 +423,15 @@ int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const cha printf("%s", taos_stmt_errstr(stmt)); return code; } - TAOS_RES* res = taos_stmt_use_result(stmt); - return taos_errno(res); + + taos_stmt_close(stmt); + return 0; } int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind) { size_t numCols = taosArrayGetSize(colsSchema); - char sql[TSDB_MAX_BINARY_LEN]; - int32_t freeBytes = TSDB_MAX_BINARY_LEN; + char sql[4096]; + int32_t freeBytes = 4096; sprintf(sql, "insert into ? ("); for (int i = 0; i < numCols; ++i) { @@ -454,8 +460,8 @@ int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* r } size_t rows = taosArrayGetSize(rowsBind); for (int32_t i = 0; i < rows; ++i) { - SArray* colBind = taosArrayGetP(rowsBind, i); - code = taos_stmt_bind_param(stmt, TARRAY_GET_START(colBind)); + TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); + code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { printf("%s", taos_stmt_errstr(stmt)); return code; @@ -472,8 +478,10 @@ int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* r printf("%s", taos_stmt_errstr(stmt)); return code; } - TAOS_RES* res = taos_stmt_use_result(stmt); - return taos_errno(res); + + + taos_stmt_close(stmt); + return code; } int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) { @@ -526,7 +534,6 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) TAOS_SML_DATA_POINT * point = taosArrayGetP(cTablePoints, 0); int32_t numTags = taosArrayGetSize(point->schema->tags); int32_t numCols = taosArrayGetSize(point->schema->fields); - char* ctableName = point->childTableName; SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); taosArraySetSize(tagBinds, numTags); @@ -551,16 +558,15 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) for (int i = 0; i < rows; ++i) { point = taosArrayGetP(cTablePoints, i); - SArray* colBinds = taosArrayInit(numCols, sizeof(TAOS_BIND)); - taosArraySetSize(colBinds, numCols); + TAOS_BIND* colBinds = calloc(numCols, sizeof(TAOS_BIND)); for (int j = 0; j < numCols; ++j) { - TAOS_BIND* bind = taosArrayGet(colBinds, j); + TAOS_BIND* bind = colBinds + j; bind->is_null = &isNullColBind; } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); - TAOS_BIND* bind = taosArrayGet(colBinds, idx); + TAOS_BIND* bind = colBinds + idx; bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); *bind->length = kv->length; @@ -571,7 +577,7 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) } creatChildTableIfNotExists(taos, point->childTableName, point->stableName, point->schema->tags, tagBinds); - insertBatch(taos, ctableName, point->schema->fields, rowsBind); + insertBatch(taos, point->childTableName, point->schema->fields, rowsBind); pCTablePoints = taosHashIterate(cname2points, pCTablePoints); } @@ -621,10 +627,6 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { for (int i = 0; i < numStable; ++i) { SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); SSmlSTableSchema dbSchema = {0}; - dbSchema.tags = taosArrayInit(8, sizeof(SSchema)); - dbSchema.fields = taosArrayInit(64, sizeof(SSchema)); - dbSchema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - dbSchema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); @@ -652,6 +654,8 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { applySchemaAction(taos, &schemaAction); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + pointSchema->precision = dbSchema.precision; } } @@ -666,6 +670,8 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); if (actionNeeded) { applySchemaAction(taos, &schemaAction); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + pointSchema->precision = dbSchema.precision; } } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 409140e0f2..75262d74d8 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -965,12 +965,24 @@ int32_t verify_schema_less(TAOS* taos) { char* lines[] = { "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", + "st,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", "ste,t2=5,t3=L\"ste\" c1=true,c2=4,c3=\"iam\" 1626056811823316532", "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532" }; - int code = taos_insert_by_lines(taos, lines , 5); + +// int code = taos_insert_by_lines(taos, lines , 5); + int code = taos_insert_by_lines(taos, &(lines[0]), 1); + + code = taos_insert_by_lines(taos, &(lines[1]), 1); + +// code = taos_insert_by_lines(taos, &(lines[2]), 1); +// +// code = taos_insert_by_lines(taos, &(lines[3]), 1); +// +// code = taos_insert_by_lines(taos, &(lines[4]), 1); + + return code; } @@ -980,7 +992,7 @@ int main(int argc, char *argv[]) { const char* passwd = "taosdata"; taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - + taos_options(TSDB_OPTION_CONFIGDIR, "/etc/taos"); TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 58314d2e50..2e19dde3d9 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -87,6 +87,8 @@ enum { SIM_CMD_RESTFUL, SIM_CMD_TEST, SIM_CMD_RETURN, + SIM_CMD_LINE_INSERT, + SIM_CMD_LINE_INSERT_ERROR, SIM_CMD_END }; @@ -172,6 +174,8 @@ bool simExecuteSqlCmd(SScript *script, char *option); bool simExecuteSqlErrorCmd(SScript *script, char *rest); bool simExecuteSqlSlowCmd(SScript *script, char *option); bool simExecuteRestfulCmd(SScript *script, char *rest); +bool simExecuteLineInsertCmd(SScript *script, char *option); +bool simExecuteLineInsertErrorCmd(SScript *script, char *option); void simVisuallizeOption(SScript *script, char *src, char *dst); #endif \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 7d74946e93..2615448e4b 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -1067,3 +1067,49 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return false; } + +bool simExecuteLineInsertCmd(SScript *script, char *rest) { + char buf[TSDB_MAX_BINARY_LEN]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + SCmdLine *line = &script->lines[script->linePos]; + + simInfo("script:%s, %s", script->fileName, rest); + simLogSql(buf, true); + char * lines[] = {rest}; + int32_t ret = taos_insert_by_lines(script->taos, lines, 1); + if (ret == TSDB_CODE_SUCCESS) { + simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); + script->linePos++; + return true; + } else { + sprintf(script->error, "lineNum: %d. line: %s failed, ret:%d:%s", line->lineNum, rest, + ret & 0XFFFF, tstrerror(ret)); + return false; + } +} + +bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { + char buf[TSDB_MAX_BINARY_LEN]; + + simVisuallizeOption(script, rest, buf); + rest = buf; + + SCmdLine *line = &script->lines[script->linePos]; + + simInfo("script:%s, %s", script->fileName, rest); + simLogSql(buf, true); + char * lines[] = {rest}; + int32_t ret = taos_insert_by_lines(script->taos, lines, 1); + if (ret == TSDB_CODE_SUCCESS) { + sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest); + script->linePos++; + return false; + } else { + simDebug("lineNum: %d. line: %s failed, ret:%d:%s. Expect failed, so success", line->lineNum, rest, + ret & 0XFFFF, tstrerror(ret)); + return true; + } +} diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index b909f5bd8f..1acdcd2ac6 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -838,6 +838,38 @@ bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { return true; } +bool simParseLineInsertCmd(char* rest, SCommand* pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + +bool simParseLineInsertErrorCmd(char* rest, SCommand* pCmd, int32_t lineNum) { + int32_t expLen; + + rest++; + cmdLine[numOfLines].cmdno = SIM_CMD_LINE_INSERT; + cmdLine[numOfLines].lineNum = lineNum; + cmdLine[numOfLines].optionOffset = optionOffset; + expLen = (int32_t)strlen(rest); + memcpy(optionBuffer + optionOffset, rest, expLen); + optionOffset += expLen + 1; + *(optionBuffer + optionOffset - 1) = 0; + + numOfLines++; + return true; +} + void simInitsimCmdList() { int32_t cmdno; memset(simCmdList, 0, SIM_CMD_END * sizeof(SCommand)); @@ -1049,4 +1081,20 @@ void simInitsimCmdList() { simCmdList[cmdno].parseCmd = simParseReturnCmd; simCmdList[cmdno].executeCmd = simExecuteReturnCmd; simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_LINE_INSERT; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "line_insert"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseLineInsertCmd; + simCmdList[cmdno].executeCmd = simExecuteLineInsertCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); + + cmdno = SIM_CMD_LINE_INSERT_ERROR; + simCmdList[cmdno].cmdno = cmdno; + strcpy(simCmdList[cmdno].name, "line_insert_error"); + simCmdList[cmdno].nlen = (int16_t)strlen(simCmdList[cmdno].name); + simCmdList[cmdno].parseCmd = simParseLineInsertErrorCmd; + simCmdList[cmdno].executeCmd = simExecuteLineInsertErrorCmd; + simAddCmdIntoHash(&(simCmdList[cmdno])); } From 4404919d3b7efd943efffdc8b9e640e3c8d6ed01 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 13 Jul 2021 08:22:13 +0800 Subject: [PATCH 26/43] [TD-5201]:record SSqlObjectId id in log --- src/client/src/tscParseInsert.c | 1 + src/client/src/tscPrepare.c | 1 + src/client/src/tscUtil.c | 1 + 3 files changed, 3 insertions(+) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cd56153bc4..26d9cf0e49 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1129,6 +1129,7 @@ int tsParseInsertSql(SSqlObj *pSql) { SSqlCmd *pCmd = &pSql->cmd; SInsertStatementParam* pInsertParam = &pCmd->insertParam; + pInsertParam->objectId = pSql->self; char* str = pInsertParam->sql; int32_t totalNum = 0; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 08d3cc599e..798e79dbcb 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1470,6 +1470,7 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) { pSql->fetchFp = waitForQueryRsp; pCmd->insertParam.insertType = TSDB_QUERY_TYPE_STMT_INSERT; + pCmd->insertParam.objectId = pSql->self; pSql->sqlstr = realloc(pSql->sqlstr, sqlLen + 1); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 173db05e10..460fe739e6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3342,6 +3342,7 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, __async_cb_func_t pnCmd->insertParam.numOfTables = 0; pnCmd->insertParam.pTableNameList = NULL; pnCmd->insertParam.pTableBlockHashList = NULL; + pnCmd->insertParam.objectId = pNew->self; memset(&pnCmd->insertParam.tagData, 0, sizeof(STagData)); From 136189e3c892075a628d753395ea9b5724739764 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 13 Jul 2021 10:52:58 +0800 Subject: [PATCH 27/43] memory handling --- src/client/src/tscParseLineProtocol.c | 440 +++++++++++--------- tests/examples/c/apitest.c | 17 +- tests/script/general/parser/line_insert.sim | 32 ++ 3 files changed, 286 insertions(+), 203 deletions(-) create mode 100644 tests/script/general/parser/line_insert.sim diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 77d65f3d15..85a6c41ccf 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -64,88 +64,6 @@ int compareSmlColKv(const void* p1, const void* p2) { } } -int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { - schema->tags = taosArrayInit(8, sizeof(SSchema)); - schema->fields = taosArrayInit(64, sizeof(SSchema)); - schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - int32_t code = 0; - - STscObj *pObj = (STscObj *)taos; - if (pObj == NULL || pObj->signature != pObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; - return TSDB_CODE_TSC_DISCONNECTED; - } - - char sql[256]; - snprintf(sql, 256, "describe %s", tableName); - TAOS_RES* res = taos_query(taos, sql); - code = taos_errno(res); - if (code != 0) { - taos_free_result(res); - return code; - } - taos_free_result(res); - - SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); - pSql->pTscObj = taos; - pSql->signature = pSql; - pSql->fp = NULL; - - SStrToken tableToken = {.z=tableName, .n=strlen(tableName), .type=TK_ID}; - tGetToken(tableName, &tableToken.type); - // Check if the table name available or not - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; - sprintf(pSql->cmd.payload, "table name is invalid"); - return code; - } - - SName sname = {0}; - if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { - return code; - } - - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; - memset(fullTableName, 0, tListLen(fullTableName)); - tNameExtractFullName(&sname, fullTableName); - - if (code != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pSql); - return code; - } - - tscFreeSqlObj(pSql); - - uint32_t size = tscGetTableMetaMaxSize(); - STableMeta* tableMeta = calloc(1, size); - taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); - - tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); - schema->precision = tableMeta->tableInfo.precision; - for (int i=0; itableInfo.numOfColumns; ++i) { - SSchema field; - tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); - field.type = tableMeta->schema[i].type; - field.bytes = tableMeta->schema[i].bytes; - SSchema* pField = taosArrayPush(schema->fields, &field); - taosHashPut(schema->fieldHash, field.name, strlen(field.name), &pField, POINTER_BYTES); - } - - for (int i=0; itableInfo.numOfTags; ++i) { - int j = i + tableMeta->tableInfo.numOfColumns; - SSchema field; - tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); - field.type = tableMeta->schema[j].type; - field.bytes = tableMeta->schema[j].bytes; - SSchema* pField = taosArrayPush(schema->tags, &field); - taosHashPut(schema->tagHash, field.name, strlen(field.name), &pField, POINTER_BYTES); - } - free(tableMeta); tableMeta = NULL; - return code; -} - typedef enum { SCHEMA_ACTION_CREATE_STABLE, SCHEMA_ACTION_ADD_COLUMN, @@ -173,7 +91,7 @@ typedef struct { }; } SSchemaAction; -int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { +static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { if (!IS_VAR_DATA_TYPE(kv->type)) { *bytes = tDataTypes[kv->type].bytes; } else { @@ -191,7 +109,7 @@ int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { return 0; } -int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array) { +static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array) { SSchema* pField = NULL; SSchema** ppField = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); if (ppField) { @@ -227,7 +145,55 @@ int32_t addTaosFieldToHashAndArray(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* a return 0; } -int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool isTag, char sTableName[], +static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas) { + SHashObj* sname2shema = taosHashInit(32, + taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + for (int i = 0; i < numPoint; ++i) { + TAOS_SML_DATA_POINT* point = &points[i]; + size_t stableNameLen = strlen(point->stableName); + SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, stableNameLen); + SSmlSTableSchema* pStableSchema = NULL; + if (ppStableSchema) { + pStableSchema= *ppStableSchema; + } else { + SSmlSTableSchema schema; + strncpy(schema.sTableName, point->stableName, stableNameLen); + schema.sTableName[stableNameLen] = '\0'; + schema.fields = taosArrayInit(64, sizeof(SSchema)); + schema.tags = taosArrayInit(8, sizeof(SSchema)); + schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + pStableSchema = taosArrayPush(stableSchemas, &schema); + taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); + } + + for (int j = 0; j < point->tagNum; ++j) { + TAOS_SML_KV* tagKv = point->tags + j; + buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags); + } + + for (int j = 0; j < point->fieldNum; ++j) { + TAOS_SML_KV* fieldKv = point->fields + j; + buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); + } + + point->schema = pStableSchema; + } + + size_t numStables = taosArrayGetSize(stableSchemas); + for (int32_t i = 0; i < numStables; ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + taosHashCleanup(schema->tagHash); + taosHashCleanup(schema->fieldHash); + } + taosHashCleanup(sname2shema); + + return 0; +} + +static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool isTag, char sTableName[], SSchemaAction* action, bool* actionNeeded) { SSchema** ppDbAttr = taosHashGet(dbAttrHash, pointColField->name, strlen(pointColField->name)); if (ppDbAttr) { @@ -262,7 +228,7 @@ int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash, bool return 0; } -int32_t buildColumnDescription(SSchema* field, +static int32_t buildColumnDescription(SSchema* field, char* buf, int32_t bufSize, int32_t* outBytes) { uint8_t type = field->type; @@ -283,7 +249,8 @@ int32_t buildColumnDescription(SSchema* field, return 0; } -int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { + +static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { int32_t code = 0; int32_t capacity = TSDB_MAX_BINARY_LEN; int32_t outBytes = 0; @@ -357,7 +324,160 @@ int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } -int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { +static int32_t destorySmlSTableSchema(SSmlSTableSchema* schema) { + taosHashCleanup(schema->tagHash); + taosHashCleanup(schema->fieldHash); + taosArrayDestroy(schema->tags); + taosArrayDestroy(schema->fields); + return 0; +} + +int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { + int32_t code = 0; + + STscObj *pObj = (STscObj *)taos; + if (pObj == NULL || pObj->signature != pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return TSDB_CODE_TSC_DISCONNECTED; + } + + char sql[256]; + snprintf(sql, 256, "describe %s", tableName); + TAOS_RES* res = taos_query(taos, sql); + code = taos_errno(res); + if (code != 0) { + taos_free_result(res); + return code; + } + taos_free_result(res); + + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + pSql->pTscObj = taos; + pSql->signature = pSql; + pSql->fp = NULL; + + SStrToken tableToken = {.z=tableName, .n=strlen(tableName), .type=TK_ID}; + tGetToken(tableName, &tableToken.type); + // Check if the table name available or not + if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; + sprintf(pSql->cmd.payload, "table name is invalid"); + return code; + } + + SName sname = {0}; + if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { + return code; + } + + char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; + memset(fullTableName, 0, tListLen(fullTableName)); + tNameExtractFullName(&sname, fullTableName); + + if (code != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); + return code; + } + + tscFreeSqlObj(pSql); + + schema->tags = taosArrayInit(8, sizeof(SSchema)); + schema->fields = taosArrayInit(64, sizeof(SSchema)); + schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); + + uint32_t size = tscGetTableMetaMaxSize(); + STableMeta* tableMeta = calloc(1, size); + taosHashGetClone(tscTableMetaInfo, fullTableName, strlen(fullTableName), NULL, tableMeta, -1); + + tstrncpy(schema->sTableName, tableName, strlen(tableName)+1); + schema->precision = tableMeta->tableInfo.precision; + for (int i=0; itableInfo.numOfColumns; ++i) { + SSchema field; + tstrncpy(field.name, tableMeta->schema[i].name, strlen(tableMeta->schema[i].name)+1); + field.type = tableMeta->schema[i].type; + field.bytes = tableMeta->schema[i].bytes; + SSchema* pField = taosArrayPush(schema->fields, &field); + taosHashPut(schema->fieldHash, field.name, strlen(field.name), &pField, POINTER_BYTES); + } + + for (int i=0; itableInfo.numOfTags; ++i) { + int j = i + tableMeta->tableInfo.numOfColumns; + SSchema field; + tstrncpy(field.name, tableMeta->schema[j].name, strlen(tableMeta->schema[j].name)+1); + field.type = tableMeta->schema[j].type; + field.bytes = tableMeta->schema[j].bytes; + SSchema* pField = taosArrayPush(schema->tags, &field); + taosHashPut(schema->tagHash, field.name, strlen(field.name), &pField, POINTER_BYTES); + } + free(tableMeta); tableMeta = NULL; + return code; +} + +static int32_t reconcileDBSchemas(TAOS* taos, SArray* stableSchemas) { + int32_t code = 0; + size_t numStable = taosArrayGetSize(stableSchemas); + for (int i = 0; i < numStable; ++i) { + SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); + SSmlSTableSchema dbSchema = {0}; + + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + + if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { + SSchemaAction schemaAction = {0}; + schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; + memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); + memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); + schemaAction.createSTable.tags = pointSchema->tags; + schemaAction.createSTable.fields = pointSchema->fields; + applySchemaAction(taos, &schemaAction); + code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); + + pointSchema->precision = dbSchema.precision; + + destorySmlSTableSchema(&dbSchema); + } else if (code == TSDB_CODE_SUCCESS) { + size_t pointTagSize = taosArrayGetSize(pointSchema->tags); + size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); + + SHashObj* dbTagHash = dbSchema.tagHash; + SHashObj* dbFieldHash = dbSchema.fieldHash; + + for (int j = 0; j < pointTagSize; ++j) { + SSchema* pointTag = taosArrayGet(pointSchema->tags, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + applySchemaAction(taos, &schemaAction); + } + } + + SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); + SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); + memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); + + for (int j = 1; j < pointFieldSize; ++j) { + SSchema* pointCol = taosArrayGet(pointSchema->fields, j); + SSchemaAction schemaAction = {0}; + bool actionNeeded = false; + generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); + if (actionNeeded) { + applySchemaAction(taos, &schemaAction); + } + } + + pointSchema->precision = dbSchema.precision; + + destorySmlSTableSchema(&dbSchema); + } else { + return code; + } + } + return 0; +} + +static int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen) { qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv); SStringBuilder sb; memset(&sb, 0, sizeof(sb)); @@ -384,7 +504,7 @@ int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tabl return 0; } -int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind) { +static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName, SArray* tagsSchema, SArray* tagsBind) { size_t numTags = taosArrayGetSize(tagsSchema); char sql[TSDB_MAX_BINARY_LEN] = {0}; int freeBytes = TSDB_MAX_BINARY_LEN; @@ -428,10 +548,10 @@ int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const cha return 0; } -int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind) { +static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* rowsBind) { size_t numCols = taosArrayGetSize(colsSchema); - char sql[4096]; - int32_t freeBytes = 4096; + char sql[TSDB_MAX_BINARY_LEN]; + int32_t freeBytes = TSDB_MAX_BINARY_LEN; sprintf(sql, "insert into ? ("); for (int i = 0; i < numCols; ++i) { @@ -458,6 +578,7 @@ int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* r printf("%s", taos_stmt_errstr(stmt)); return code; } + size_t rows = taosArrayGetSize(rowsBind); for (int32_t i = 0; i < rows; ++i) { TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); @@ -479,13 +600,11 @@ int32_t insertBatch(TAOS* taos, char* cTableName, SArray* colsSchema, SArray* r return code; } - taos_stmt_close(stmt); return code; } -int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) { - SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int numPoints, SHashObj* cname2points) { for (int32_t i = 0; i < numPoints; ++i) { TAOS_SML_DATA_POINT * point = points + i; if (!point->childTableName) { @@ -526,6 +645,14 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) taosArrayPush(cTablePoints, &point); } + return 0; +} + +static int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) { + SHashObj* cname2points = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + true, false); + arrangePointsByChildTableName(points, numPoints, cname2points); + int isNullColBind = TSDB_TRUE; SArray** pCTablePoints = taosHashIterate(cname2points, NULL); while (pCTablePoints) { @@ -577,111 +704,44 @@ int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t numPoints) } creatChildTableIfNotExists(taos, point->childTableName, point->stableName, point->schema->tags, tagBinds); - insertBatch(taos, point->childTableName, point->schema->fields, rowsBind); + for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) { + TAOS_BIND* bind = taosArrayGet(tagBinds, i); + free(bind->length); + } + taosArrayDestroy(tagBinds); + + insertChildTableBatch(taos, point->childTableName, point->schema->fields, rowsBind); + for (int i = 0; i < rows; ++i) { + TAOS_BIND* colBinds = taosArrayGetP(rowsBind, i); + for (int j = 0; j < numCols; ++j) { + TAOS_BIND* bind = colBinds + j; + free(bind->length); + } + } + taosArrayDestroy(rowsBind); + taosArrayDestroy(cTablePoints); pCTablePoints = taosHashIterate(cname2points, pCTablePoints); } + + taosHashCleanup(cname2points); return 0; } int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { int32_t code = TSDB_CODE_SUCCESS; - SArray* stableArray = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray - SHashObj* sname2shema = taosHashInit(32, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - for (int i = 0; i < numPoint; ++i) { - TAOS_SML_DATA_POINT* point = &points[i]; - size_t stableNameLen = strlen(point->stableName); - SSmlSTableSchema** ppStableSchema = taosHashGet(sname2shema, point->stableName, stableNameLen); - SSmlSTableSchema* pStableSchema = NULL; - if (ppStableSchema) { - pStableSchema= *ppStableSchema; - } else { - SSmlSTableSchema schema; - strncpy(schema.sTableName, point->stableName, stableNameLen); - schema.sTableName[stableNameLen] = '\0'; - schema.fields = taosArrayInit(64, sizeof(SSchema)); - schema.tags = taosArrayInit(8, sizeof(SSchema)); - schema.tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - schema.fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - - pStableSchema = taosArrayPush(stableArray, &schema); - taosHashPut(sname2shema, schema.sTableName, stableNameLen, &pStableSchema, POINTER_BYTES); - } - - for (int j = 0; j < point->tagNum; ++j) { - TAOS_SML_KV* tagKv = point->tags + j; - addTaosFieldToHashAndArray(tagKv, pStableSchema->tagHash, pStableSchema->tags); - } - - for (int j = 0; j < point->fieldNum; ++j) { - TAOS_SML_KV* fieldKv = point->fields + j; - addTaosFieldToHashAndArray(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); - } - - point->schema = pStableSchema; - } - - size_t numStable = taosArrayGetSize(stableArray); - for (int i = 0; i < numStable; ++i) { - SSmlSTableSchema* pointSchema = taosArrayGet(stableArray, i); - SSmlSTableSchema dbSchema = {0}; - - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { - SSchemaAction schemaAction = {0}; - schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; - memset(&schemaAction.createSTable, 0, sizeof(SCreateSTableActionInfo)); - memcpy(schemaAction.createSTable.sTableName, pointSchema->sTableName, TSDB_TABLE_NAME_LEN); - schemaAction.createSTable.tags = pointSchema->tags; - schemaAction.createSTable.fields = pointSchema->fields; - applySchemaAction(taos, &schemaAction); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - pointSchema->precision = dbSchema.precision; - } else if (code == TSDB_CODE_SUCCESS) { - size_t pointTagSize = taosArrayGetSize(pointSchema->tags); - size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); - - SHashObj* dbTagHash = dbSchema.tagHash; - SHashObj* dbFieldHash = dbSchema.fieldHash; - - for (int j = 0; j < pointTagSize; ++j) { - SSchema* pointTag = taosArrayGet(pointSchema->tags, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointTag, dbTagHash, true, pointSchema->sTableName, &schemaAction, &actionNeeded); - if (actionNeeded) { - applySchemaAction(taos, &schemaAction); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - pointSchema->precision = dbSchema.precision; - } - } - - SSchema* pointColTs = taosArrayGet(pointSchema->fields, 0); - SSchema* dbColTs = taosArrayGet(dbSchema.fields, 0); - memcpy(pointColTs->name, dbColTs->name, TSDB_COL_NAME_LEN); - - for (int j = 1; j < pointFieldSize; ++j) { - SSchema* pointCol = taosArrayGet(pointSchema->fields, j); - SSchemaAction schemaAction = {0}; - bool actionNeeded = false; - generateSchemaAction(pointCol, dbFieldHash, false, pointSchema->sTableName, &schemaAction, &actionNeeded); - if (actionNeeded) { - applySchemaAction(taos, &schemaAction); - code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - pointSchema->precision = dbSchema.precision; - } - } - - pointSchema->precision = dbSchema.precision; - } else { - return code; - } - } + SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray + buildDataPointSchemas(points, numPoint, stableSchemas); + reconcileDBSchemas(taos, stableSchemas); insertPoints(taos, points, numPoint); + + for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + taosArrayDestroy(schema->fields); + taosArrayDestroy(schema->tags); + } + taosArrayDestroy(stableSchemas); return code; } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 75262d74d8..de7fcecc7e 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -6,6 +6,7 @@ #include #include #include +#include static void prepare_data(TAOS* taos) { TAOS_RES *result; @@ -965,23 +966,13 @@ int32_t verify_schema_less(TAOS* taos) { char* lines[] = { "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", - "st,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833640000000", + "st,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagin\",c2=true,c4=5,c5=5 1626006833640000000", "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", "ste,t2=5,t3=L\"ste\" c1=true,c2=4,c3=\"iam\" 1626056811823316532", "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532" }; -// int code = taos_insert_by_lines(taos, lines , 5); - int code = taos_insert_by_lines(taos, &(lines[0]), 1); - - code = taos_insert_by_lines(taos, &(lines[1]), 1); - -// code = taos_insert_by_lines(taos, &(lines[2]), 1); -// -// code = taos_insert_by_lines(taos, &(lines[3]), 1); -// -// code = taos_insert_by_lines(taos, &(lines[4]), 1); - + int code = taos_insert_by_lines(taos, lines , 5); return code; } @@ -992,7 +983,7 @@ int main(int argc, char *argv[]) { const char* passwd = "taosdata"; taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - taos_options(TSDB_OPTION_CONFIGDIR, "/etc/taos"); + taosDumpGlobalCfg(); TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); diff --git a/tests/script/general/parser/line_insert.sim b/tests/script/general/parser/line_insert.sim new file mode 100644 index 0000000000..cb6d228e4d --- /dev/null +++ b/tests/script/general/parser/line_insert.sim @@ -0,0 +1,32 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/exec.sh -n dnode1 -s start +sleep 2000 +sql connect + +print =============== step1 +$db = testlp +$mte = ste +$mt = st +sql drop database $db -x step1 +step1: +sql create database $db precision 'us' +sql use $db +sql create stable $mte (ts timestamp, f int) TAGS(t1 bigint) + +line_insert st,t1=3i,t2=4,t3="t3" c1=3i,c3=L"passit",c2=false,c4=4 1626006833639000000 +line_insert st,t1=4i,t3="t4",t2=5 c1=3i,c3=L"passitagain",c2=true,c4=5,c5=5 1626006833640000000 +line_insert st,t1=4i,t2=5,t3="t4" c1=3i,c3=L"passitagain",c2=true,c4=5 1626006833642000000 +line_insert ste,t2=5,t3=L"ste" c1=true,c2=4,c3="iam" 1626056811823316532 +line_insert ste,t2=5,t3=L"ste2" c3="iamszhou",c4=false 1626056811843316532 + +#print =============== clear +#sql drop database $db +#sql show databases +#if $rows != 0 then +# return -1 +#endi +# +#system sh/exec.sh -n dnode1 -s stop -x SIGINT From f9bfe6cc3d2bd91539941c799bdc8eb9066385fa Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 13 Jul 2021 10:55:26 +0800 Subject: [PATCH 28/43] for reconfigure table reproduction --- tests/examples/c/apitest.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index de7fcecc7e..b8f68636bd 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -972,7 +972,9 @@ int32_t verify_schema_less(TAOS* taos) { "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532" }; - int code = taos_insert_by_lines(taos, lines , 5); + //int code = taos_insert_by_lines(taos, lines , 5); + int code = taos_insert_by_lines(taos, &lines[0], 1); + code = taos_insert_by_lines(taos, &lines[1], 1); return code; } From ddb783f40720708b0e94e40caac30fdabaa41c64 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 13 Jul 2021 13:06:46 +0800 Subject: [PATCH 29/43] add simple tsim test --- tests/examples/c/apitest.c | 6 +-- tests/script/general/parser/line_insert.sim | 42 ++++++++++++++++----- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index b8f68636bd..f4e222c4dc 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -972,9 +972,9 @@ int32_t verify_schema_less(TAOS* taos) { "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532" }; - //int code = taos_insert_by_lines(taos, lines , 5); - int code = taos_insert_by_lines(taos, &lines[0], 1); - code = taos_insert_by_lines(taos, &lines[1], 1); + int code = taos_insert_by_lines(taos, lines , 5); + //int code = taos_insert_by_lines(taos, &lines[0], 1); + //code = taos_insert_by_lines(taos, &lines[1], 1); return code; } diff --git a/tests/script/general/parser/line_insert.sim b/tests/script/general/parser/line_insert.sim index cb6d228e4d..f3067a3bbe 100644 --- a/tests/script/general/parser/line_insert.sim +++ b/tests/script/general/parser/line_insert.sim @@ -17,16 +17,38 @@ sql use $db sql create stable $mte (ts timestamp, f int) TAGS(t1 bigint) line_insert st,t1=3i,t2=4,t3="t3" c1=3i,c3=L"passit",c2=false,c4=4 1626006833639000000 -line_insert st,t1=4i,t3="t4",t2=5 c1=3i,c3=L"passitagain",c2=true,c4=5,c5=5 1626006833640000000 -line_insert st,t1=4i,t2=5,t3="t4" c1=3i,c3=L"passitagain",c2=true,c4=5 1626006833642000000 +line_insert st,t1=4i,t3="t41",t2=5 c1=3i,c3=L"passiT",c2=true,c4=5 1626006833640000000 +line_insert stf,t1=4i,t2=5,t3="t4" c1=3i,c3=L"passitagain",c2=true,c4=5 1626006833642000000 line_insert ste,t2=5,t3=L"ste" c1=true,c2=4,c3="iam" 1626056811823316532 -line_insert ste,t2=5,t3=L"ste2" c3="iamszhou",c4=false 1626056811843316532 + +sql select * from st +if $rows != 2 then + return -1 +endi + +if $data00 != @21-07-11 20:33:53.639000@ then + return -1 +endi + +if $data03 != @passit@ then + return -1 +endi + +sql select * from stf +if $rows != 1 then + return -1 +endi + +sql select * from ste +if $rows != 1 then + return -1 +endi #print =============== clear -#sql drop database $db -#sql show databases -#if $rows != 0 then -# return -1 -#endi -# -#system sh/exec.sh -n dnode1 -s stop -x SIGINT +sql drop database $db +sql show databases +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From bcba45e38bcd2769057f3b631ee390a9289a8599 Mon Sep 17 00:00:00 2001 From: Shenglian Zhou Date: Tue, 13 Jul 2021 14:19:56 +0800 Subject: [PATCH 30/43] retry afer batch execute reconfigure table --- src/client/src/tscParseLineProtocol.c | 61 +++++++++++++++------------ tests/examples/c/apitest.c | 8 ++-- 2 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 85a6c41ccf..04b96dc52a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -324,7 +324,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { return code; } -static int32_t destorySmlSTableSchema(SSmlSTableSchema* schema) { +static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) { taosHashCleanup(schema->tagHash); taosHashCleanup(schema->fieldHash); taosArrayDestroy(schema->tags); @@ -435,7 +435,7 @@ static int32_t reconcileDBSchemas(TAOS* taos, SArray* stableSchemas) { pointSchema->precision = dbSchema.precision; - destorySmlSTableSchema(&dbSchema); + destroySmlSTableSchema(&dbSchema); } else if (code == TSDB_CODE_SUCCESS) { size_t pointTagSize = taosArrayGetSize(pointSchema->tags); size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); @@ -469,7 +469,7 @@ static int32_t reconcileDBSchemas(TAOS* taos, SArray* stableSchemas) { pointSchema->precision = dbSchema.precision; - destorySmlSTableSchema(&dbSchema); + destroySmlSTableSchema(&dbSchema); } else { return code; } @@ -565,42 +565,47 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); - TAOS_STMT* stmt = taos_stmt_init(taos); - int32_t code; - code = taos_stmt_prepare(stmt, sql, strlen(sql)); - if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); - return code; - } + int32_t code = 0; + int32_t try = 0; + do { + TAOS_STMT* stmt = taos_stmt_init(taos); - code = taos_stmt_set_tbname(stmt, cTableName); - if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); - return code; - } - - size_t rows = taosArrayGetSize(rowsBind); - for (int32_t i = 0; i < rows; ++i) { - TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); - code = taos_stmt_bind_param(stmt, colsBinds); + code = taos_stmt_prepare(stmt, sql, strlen(sql)); if (code != 0) { printf("%s", taos_stmt_errstr(stmt)); return code; } - code = taos_stmt_add_batch(stmt); + + code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { printf("%s", taos_stmt_errstr(stmt)); return code; } - } - code = taos_stmt_execute(stmt); - if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); - return code; - } + size_t rows = taosArrayGetSize(rowsBind); + for (int32_t i = 0; i < rows; ++i) { + TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); + code = taos_stmt_bind_param(stmt, colsBinds); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + code = taos_stmt_add_batch(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + } + + code = taos_stmt_execute(stmt); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + } else { + taos_stmt_close(stmt); + } + } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); - taos_stmt_close(stmt); return code; } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index f4e222c4dc..e7d4b087f7 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -6,7 +6,6 @@ #include #include #include -#include static void prepare_data(TAOS* taos) { TAOS_RES *result; @@ -972,9 +971,9 @@ int32_t verify_schema_less(TAOS* taos) { "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532" }; - int code = taos_insert_by_lines(taos, lines , 5); - //int code = taos_insert_by_lines(taos, &lines[0], 1); - //code = taos_insert_by_lines(taos, &lines[1], 1); + // int code = taos_insert_by_lines(taos, lines , 5); + int code = taos_insert_by_lines(taos, &lines[0], 1); + code = taos_insert_by_lines(taos, &lines[1], 1); return code; } @@ -985,7 +984,6 @@ int main(int argc, char *argv[]) { const char* passwd = "taosdata"; taos_options(TSDB_OPTION_TIMEZONE, "GMT-8"); - taosDumpGlobalCfg(); TAOS* taos = taos_connect(host, user, passwd, "", 0); if (taos == NULL) { printf("\033[31mfailed to connect to db, reason:%s\033[0m\n", taos_errstr(taos)); From f28c1a6097c90d6a58afe7ca00d172ab3a2c444f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 13 Jul 2021 15:20:39 +0800 Subject: [PATCH 31/43] [td-5176]: improve the twa query performance in case of interval query. --- src/client/src/tscSQLParser.c | 68 ++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ded334c69f..59807d1e2e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -96,7 +96,7 @@ static int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrTo static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding); static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, bool isStable); -static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem); +static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery); static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql); static int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); @@ -1809,8 +1809,8 @@ static bool hasNoneUserDefineExpr(SQueryInfo* pQueryInfo) { return false; } -int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool isSTable, bool joinQuery, - bool timeWindowQuery) { +int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool joinQuery, + bool timeWindowQuery, bool outerQuery) { assert(pSelNodeList != NULL && pCmd != NULL); const char* msg1 = "too many items in selection clause"; @@ -1852,7 +1852,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { // use the dynamic array list to decide if the function is valid or not // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { + if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem, outerQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } } else if (type == SQL_NODE_EXPR) { @@ -1988,14 +1988,15 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem) { +int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery) { const char* msg1 = "tag for normal table query is not allowed"; const char* msg2 = "invalid column name"; + const char* msg3 = "tbname not allowed in outer query"; int32_t startPos = (int32_t)tscNumOfExprs(pQueryInfo); - int32_t optr = pItem->pNode->tokenId; + int32_t tokenId = pItem->pNode->tokenId; - if (optr == TK_ALL) { // project on all fields + if (tokenId == TK_ALL) { // project on all fields TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -2019,7 +2020,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t if (pTableMeta->tableType != TSDB_TEMP_TABLE) { tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMeta->id.uid); } - } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query + } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; // user-specified constant value as a new result column @@ -2027,13 +2028,13 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t index.tableIndex = 0; SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); - SExprInfo* pExpr = - tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, getNewResColId(pCmd)); + SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, + getNewResColId(pCmd)); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->base.numOfParams = 2; tVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); - } else if (optr == TK_ID) { + } else if (tokenId == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(&pItem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { @@ -2041,12 +2042,40 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = *tGetTbnameColumnSchema(); - char name[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); + if (outerQuery) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG, getNewResColId(pCmd)); + bool existed = false; + SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; + for (int32_t i = 0; i < numOfCols; ++i) { + if (strncasecmp(pSchema[i].name, TSQL_TBNAME_L, tListLen(pSchema[i].name)) == 0) { + existed = true; + index.columnIndex = i; + break; + } + } + + if (!existed) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + SSchema colSchema = pSchema[index.columnIndex]; + char name[TSDB_COL_NAME_LEN] = {0}; + getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); + + tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); + /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, + TSDB_COL_NORMAL, getNewResColId(pCmd)); + } else { + SSchema colSchema = *tGetTbnameColumnSchema(); + char name[TSDB_COL_NAME_LEN] = {0}; + getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); + + tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); + /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, + TSDB_COL_TAG, getNewResColId(pCmd)); + } } else { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -7156,8 +7185,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return code; } - bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (validateSelectNodeList(&pSql->cmd, pQueryInfo, pSqlNode->pSelNodeList, isSTable, false, false) != TSDB_CODE_SUCCESS) { + if (validateSelectNodeList(&pSql->cmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -7945,7 +7973,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return TSDB_CODE_TSC_INVALID_OPERATION; } - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, timeWindowQuery, true) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -8085,7 +8113,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t timeWindowQuery = (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, isSTable, joinQuery, timeWindowQuery) != + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } From 8f2008664e24972fd730aa8d4547d13ad092c26d Mon Sep 17 00:00:00 2001 From: wpan Date: Tue, 13 Jul 2021 16:34:19 +0800 Subject: [PATCH 32/43] add retry for reconfigure table --- src/client/src/tscParseLineProtocol.c | 30 +++++++++++++++------------ src/client/src/tscPrepare.c | 10 +++++++-- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 04b96dc52a..5fba602670 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -565,16 +565,19 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); + int32_t code = 0; int32_t try = 0; - do { - TAOS_STMT* stmt = taos_stmt_init(taos); - code = taos_stmt_prepare(stmt, sql, strlen(sql)); - if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); - return code; - } + TAOS_STMT* stmt = taos_stmt_init(taos); + + code = taos_stmt_prepare(stmt, sql, strlen(sql)); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + return code; + } + + do { code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { @@ -598,14 +601,15 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } code = taos_stmt_execute(stmt); - if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); - taos_stmt_close(stmt); - } else { - taos_stmt_close(stmt); - } } while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA); + if (code != 0) { + printf("%s", taos_stmt_errstr(stmt)); + taos_stmt_close(stmt); + } else { + taos_stmt_close(stmt); + } + return code; } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 798e79dbcb..3a462c96d6 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -1200,9 +1200,11 @@ static int insertBatchStmtExecute(STscStmt* pStmt) { // wait for the callback function to post the semaphore tsem_wait(&pStmt->pSql->rspSem); + code = pStmt->pSql->res.code; + insertBatchClean(pStmt); - return pStmt->pSql->res.code; + return code; } int stmtParseInsertTbTags(SSqlObj* pSql, STscStmt* pStmt) { @@ -1647,7 +1649,11 @@ int taos_stmt_close(TAOS_STMT* stmt) { } else { if (pStmt->multiTbInsert) { taosHashCleanup(pStmt->mtb.pTableHash); - pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, false); + bool rmMeta = false; + if (pStmt->pSql && pStmt->pSql->res.code != 0) { + rmMeta = true; + } + pStmt->mtb.pTableBlockHashList = tscDestroyBlockHashTable(pStmt->mtb.pTableBlockHashList, rmMeta); taosHashCleanup(pStmt->pSql->cmd.insertParam.pTableBlockHashList); pStmt->pSql->cmd.insertParam.pTableBlockHashList = NULL; taosArrayDestroy(pStmt->mtb.tags); From 0d719ab43c3ef22932f7c99e5cffa4996ad25a31 Mon Sep 17 00:00:00 2001 From: Shenglian Zhou Date: Tue, 13 Jul 2021 16:37:07 +0800 Subject: [PATCH 33/43] add log and free memory --- src/client/src/tscParseLineProtocol.c | 165 +++++++++++++++++++++----- tests/examples/c/apitest.c | 9 +- 2 files changed, 142 insertions(+), 32 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 04b96dc52a..b657594db4 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -98,8 +98,12 @@ static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { if (kv->type == TSDB_DATA_TYPE_NCHAR) { char* ucs = malloc(kv->length * TSDB_NCHAR_SIZE + 1); int32_t bytesNeeded = 0; - //todo check conversion succeed - taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); + bool succ = taosMbsToUcs4(kv->value, kv->length, ucs, kv->length * TSDB_NCHAR_SIZE, &bytesNeeded); + if (!succ) { + free(ucs); + tscError("convert nchar string to UCS4_LE failed:%s", kv->value); + return TSDB_CODE_TSC_INVALID_VALUE; + } free(ucs); *bytes = bytesNeeded + VARSTR_HEADER_SIZE; } else if (kv->type == TSDB_DATA_TYPE_BINARY) { @@ -112,17 +116,20 @@ static int32_t getFieldBytesFromSmlKv(TAOS_SML_KV* kv, int32_t* bytes) { static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* array) { SSchema* pField = NULL; SSchema** ppField = taosHashGet(hash, smlKv->key, strlen(smlKv->key)); + int32_t code = 0; if (ppField) { pField = *ppField; if (pField->type != smlKv->type) { - //TODO: - tscError("type mismatch"); - return -1; + tscError("type mismatch. key %s, type %d. type before %d", smlKv->key, smlKv->type, pField->type); + return TSDB_CODE_TSC_INVALID_VALUE; } int32_t bytes = 0; - getFieldBytesFromSmlKv(smlKv, &bytes); + code = getFieldBytesFromSmlKv(smlKv, &bytes); + if (code != 0) { + return code; + } pField->bytes = MAX(pField->bytes, bytes); } else { @@ -133,7 +140,10 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra field.type = smlKv->type; int32_t bytes = 0; - getFieldBytesFromSmlKv(smlKv, &bytes); + code = getFieldBytesFromSmlKv(smlKv, &bytes); + if (code != 0) { + return code; + } field.bytes = bytes; pField = taosArrayPush(array, &field); @@ -146,6 +156,7 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra } static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas) { + int32_t code = 0; SHashObj* sname2shema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); @@ -171,12 +182,20 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* tagKv = point->tags + j; - buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags); + code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags); + if (code != 0) { + tscError("build data point schema failed. point no.: %d, tag key: %s", i, tagKv->key); + return code; + } } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* fieldKv = point->fields + j; - buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); + code = buildSmlKvSchema(fieldKv, pStableSchema->fieldHash, pStableSchema->fields); + if (code != 0) { + tscError("build data point schema failed. point no.: %d, tag key: %s", i, fieldKv->key); + return code; + } } point->schema = pStableSchema; @@ -190,6 +209,13 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, } taosHashCleanup(sname2shema); + tscDebug("build point schema succeed. num of super table: %zu", numStables); + for (int32_t i = 0; i < numStables; ++i) { + SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); + tscDebug("\ttable name: %s, tags number: %zu, fields number: %zu", schema->sTableName, + taosArrayGetSize(schema->tags), taosArrayGetSize(schema->fields)); + } + return 0; } @@ -199,8 +225,9 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash if (ppDbAttr) { SSchema* dbAttr = *ppDbAttr; if (pointColField->type != dbAttr->type) { - //todo error - return -5; + tscError("point type and db type mismatch. key: %s. point type: %d, db type: %d", pointColField->name, + pointColField->type, dbAttr->type); + return TSDB_CODE_TSC_INVALID_VALUE; } if (IS_VAR_DATA_TYPE(pointColField->type) && (pointColField->bytes > dbAttr->bytes)) { @@ -225,6 +252,7 @@ static int32_t generateSchemaAction(SSchema* pointColField, SHashObj* dbAttrHash action->alterSTable.field = pointColField; *actionNeeded = true; } + tscDebug("generate schema action. action needed: %d, action: %d", *actionNeeded, action->action); return 0; } @@ -256,12 +284,14 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { int32_t outBytes = 0; char *result = (char *)calloc(1, capacity); + tscDebug("apply schema action: %d", action->action); switch (action->action) { case SCHEMA_ACTION_ADD_COLUMN: { int n = sprintf(result, "alter stable %s add column ", action->alterSTable.sTableName); buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + taos_free_result(res); break; } case SCHEMA_ACTION_ADD_TAG: { @@ -270,6 +300,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { result+n, capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + taos_free_result(res); break; } case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { @@ -278,6 +309,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + taos_free_result(res); break; } case SCHEMA_ACTION_CHANGE_TAG_SIZE: { @@ -286,6 +318,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { capacity-n, &outBytes); TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery code = taos_errno(res); + taos_free_result(res); break; } case SCHEMA_ACTION_CREATE_STABLE: { @@ -314,13 +347,18 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { outBytes = snprintf(pos, freeBytes, ")"); TAOS_RES* res = taos_query(taos, result); code = taos_errno(res); + taos_free_result(res); break; } default: break; } + free(result); + if (code != 0) { + tscError("apply schema action failure. %s", tstrerror(code)); + } return code; } @@ -341,11 +379,14 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { return TSDB_CODE_TSC_DISCONNECTED; } + tscDebug("load table schema. super table name: %s", tableName); + char sql[256]; snprintf(sql, 256, "describe %s", tableName); TAOS_RES* res = taos_query(taos, sql); code = taos_errno(res); if (code != 0) { + tscError("describe table failure. %s", taos_errstr(res)); taos_free_result(res); return code; } @@ -369,16 +410,13 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) { return code; } - char fullTableName[TSDB_TABLE_FNAME_LEN] = {0}; memset(fullTableName, 0, tListLen(fullTableName)); tNameExtractFullName(&sname, fullTableName); - if (code != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pSql); return code; } - tscFreeSqlObj(pSql); schema->tags = taosArrayInit(8, sizeof(SSchema)); @@ -410,6 +448,8 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { SSchema* pField = taosArrayPush(schema->tags, &field); taosHashPut(schema->tagHash, field.name, strlen(field.name), &pField, POINTER_BYTES); } + tscDebug("load table meta succeed. %s, columns number: %d, tag number: %d, precision: %d", + tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision); free(tableMeta); tableMeta = NULL; return code; } @@ -422,7 +462,6 @@ static int32_t reconcileDBSchemas(TAOS* taos, SArray* stableSchemas) { SSmlSTableSchema dbSchema = {0}; code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { SSchemaAction schemaAction = {0}; schemaAction.action = SCHEMA_ACTION_CREATE_STABLE; @@ -432,10 +471,12 @@ static int32_t reconcileDBSchemas(TAOS* taos, SArray* stableSchemas) { schemaAction.createSTable.fields = pointSchema->fields; applySchemaAction(taos, &schemaAction); code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); - - pointSchema->precision = dbSchema.precision; - - destroySmlSTableSchema(&dbSchema); + if (code != 0) { + tscError("reconcile point schema failed. can not create %s", pointSchema->sTableName); + } else { + pointSchema->precision = dbSchema.precision; + destroySmlSTableSchema(&dbSchema); + } } else if (code == TSDB_CODE_SUCCESS) { size_t pointTagSize = taosArrayGetSize(pointSchema->tags); size_t pointFieldSize = taosArrayGetSize(pointSchema->fields); @@ -471,6 +512,7 @@ static int32_t reconcileDBSchemas(TAOS* taos, SArray* stableSchemas) { destroySmlSTableSchema(&dbSchema); } else { + tscError("load table meta error: %s", tstrerror(code)); return code; } } @@ -496,11 +538,12 @@ static int32_t getChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, in MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len); MD5Final(&context); *tableNameLen = snprintf(tableName, *tableNameLen, - "tbl_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], + "t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0], context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6], context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11], context.digest[12], context.digest[13], context.digest[14], context.digest[15]); taosStringBuilderDestroy(&sb); + tscDebug("child table name: %s", tableName); return 0; } @@ -524,23 +567,25 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co } snprintf(sql + strlen(sql) - 1, freeBytes-strlen(sql)+1, ")"); + tscDebug("create table : %s", sql); + TAOS_STMT* stmt = taos_stmt_init(taos); int32_t code; code = taos_stmt_prepare(stmt, sql, strlen(sql)); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); return code; } code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind)); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); return code; } code = taos_stmt_execute(stmt); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); return code; } @@ -565,6 +610,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols } snprintf(sql + strlen(sql)-1, freeBytes-strlen(sql)+1, ")"); + tscDebug("insert rows %zu into child table %s. ", taosArrayGetSize(rowsBind), cTableName); int32_t code = 0; int32_t try = 0; do { @@ -572,13 +618,13 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols code = taos_stmt_prepare(stmt, sql, strlen(sql)); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); return code; } code = taos_stmt_set_tbname(stmt, cTableName); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); return code; } @@ -587,19 +633,19 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i); code = taos_stmt_bind_param(stmt, colsBinds); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); return code; } code = taos_stmt_add_batch(stmt); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); return code; } } code = taos_stmt_execute(stmt); if (code != 0) { - printf("%s", taos_stmt_errstr(stmt)); + tscError("%s", taos_stmt_errstr(stmt)); taos_stmt_close(stmt); } else { taos_stmt_close(stmt); @@ -722,6 +768,7 @@ static int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t num TAOS_BIND* bind = colBinds + j; free(bind->length); } + free(colBinds); } taosArrayDestroy(rowsBind); taosArrayDestroy(cTablePoints); @@ -734,6 +781,8 @@ static int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t num } int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { + tscDebug("taos_sml_insert. number of points: %d", numPoint); + int32_t code = TSDB_CODE_SUCCESS; SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray @@ -846,6 +895,30 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { item->value = malloc(item->length); char* endptr = NULL; *(int64_t*)(item->value) = strtoll(sv, &endptr, 10); + } else if (*last == 'b') { + item->type = TSDB_DATA_TYPE_TINYINT; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(int8_t*)(item->value) = strtoll(sv, &endptr, 10); + } else if (*last == 's') { + item->type = TSDB_DATA_TYPE_SMALLINT; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(int16_t*)(item->value) = strtoll(sv, &endptr, 10); + } else if (*last == 'w') { + item->type = TSDB_DATA_TYPE_INT; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(int32_t*)(item->value) = strtoll(sv, &endptr, 10); + } else if (*last == 'f') { + item->type = TSDB_DATA_TYPE_FLOAT; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(float*)(item->value) = strtold(sv, &endptr); } else { item->type = TSDB_DATA_TYPE_DOUBLE; item->length = (int16_t)tDataTypes[item->type].bytes; @@ -1001,6 +1074,36 @@ int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* faile return 0; } +void destroyLPPoint(void* p) { + SLPPoint* lpPoint = p; + for (int i=0; ifields); ++i) { + SLPItem* item = taosArrayGet(lpPoint->fields, i); + free(item->value); + } + taosArrayDestroy(lpPoint->fields); + + for (int i=0; itags); ++i) { + SLPItem* item = taosArrayGet(lpPoint->tags, i); + free(item->value); + } + taosArrayDestroy(lpPoint->tags); +} + +void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { + for (int i=0; itagNum; ++i) { + free((point->tags+i)->key); + free((point->tags+i)->value); + } + free(point->tags); + for (int i=0; ifieldNum; ++i) { + free((point->fields+i)->key); + free((point->fields+i)->value); + } + free(point->fields); + free(point->stableName); + free(point->childTableName); +} + int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { SArray* lpPoints = taosArrayInit(numLines, sizeof(SLPPoint)); tscParseLines(lines, numLines, lpPoints, NULL); @@ -1065,6 +1168,12 @@ int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { } taos_sml_insert(taos, points, numPoints); + + for (int i=0; i Date: Tue, 13 Jul 2021 17:34:06 +0800 Subject: [PATCH 34/43] show crash --- tests/examples/c/apitest.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 4f9c890839..376b00ef71 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -972,9 +972,9 @@ int32_t verify_schema_less(TAOS* taos) { "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false,c5=32b,c6=64s,c7=32w,c8=88.88f 1626056812843316532" }; - int code = taos_insert_by_lines(taos, lines , 6); -// int code = taos_insert_by_lines(taos, &lines[0], 1); -// code = taos_insert_by_lines(taos, &lines[1], 1); + //int code = taos_insert_by_lines(taos, lines , 6); + int code = taos_insert_by_lines(taos, &lines[0], 1); + code = taos_insert_by_lines(taos, &lines[1], 1); return code; } From 153b5d7115e6eb157e3b8639e3b716ee1c1a0faa Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 13 Jul 2021 21:37:22 +0800 Subject: [PATCH 35/43] pass pr build check --- src/client/src/tscParseLineProtocol.c | 62 ++++++++++++++++----------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index a3bb7c6f15..a198a7b42a 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -324,7 +324,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { case SCHEMA_ACTION_CREATE_STABLE: { int n = sprintf(result, "create stable %s (", action->createSTable.sTableName); char* pos = result + n; int freeBytes = capacity - n; - int numCols = taosArrayGetSize(action->createSTable.fields); + size_t numCols = taosArrayGetSize(action->createSTable.fields); for (int32_t i = 0; i < numCols; ++i) { SSchema* field = taosArrayGet(action->createSTable.fields, i); buildColumnDescription(field, pos, freeBytes, &outBytes); @@ -336,7 +336,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action) { outBytes = snprintf(pos, freeBytes, ") tags ("); pos += outBytes; freeBytes -= outBytes; - int numTags = taosArrayGetSize(action->createSTable.tags); + size_t numTags = taosArrayGetSize(action->createSTable.tags); for (int32_t i = 0; i < numTags; ++i) { SSchema* field = taosArrayGet(action->createSTable.tags, i); buildColumnDescription(field, pos, freeBytes, &outBytes); @@ -397,7 +397,7 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema) { pSql->signature = pSql; pSql->fp = NULL; - SStrToken tableToken = {.z=tableName, .n=strlen(tableName), .type=TK_ID}; + SStrToken tableToken = {.z=tableName, .n=(uint32_t)strlen(tableName), .type=TK_ID}; tGetToken(tableName, &tableToken.type); // Check if the table name available or not if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { @@ -459,7 +459,8 @@ static int32_t reconcileDBSchemas(TAOS* taos, SArray* stableSchemas) { size_t numStable = taosArrayGetSize(stableSchemas); for (int i = 0; i < numStable; ++i) { SSmlSTableSchema* pointSchema = taosArrayGet(stableSchemas, i); - SSmlSTableSchema dbSchema = {0}; + SSmlSTableSchema dbSchema; + memset(&dbSchema, 0, sizeof(SSmlSTableSchema)); code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema); if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) { @@ -571,7 +572,7 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co TAOS_STMT* stmt = taos_stmt_init(taos); int32_t code; - code = taos_stmt_prepare(stmt, sql, strlen(sql)); + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); return code; @@ -617,7 +618,7 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols TAOS_STMT* stmt = taos_stmt_init(taos); - code = taos_stmt_prepare(stmt, sql, strlen(sql)); + code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql)); if (code != 0) { tscError("%s", taos_stmt_errstr(stmt)); return code; @@ -717,8 +718,8 @@ static int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t num SArray* cTablePoints = *pCTablePoints; TAOS_SML_DATA_POINT * point = taosArrayGetP(cTablePoints, 0); - int32_t numTags = taosArrayGetSize(point->schema->tags); - int32_t numCols = taosArrayGetSize(point->schema->fields); + size_t numTags = taosArrayGetSize(point->schema->tags); + size_t numCols = taosArrayGetSize(point->schema->fields); SArray* tagBinds = taosArrayInit(numTags, sizeof(TAOS_BIND)); taosArraySetSize(tagBinds, numTags); @@ -728,7 +729,7 @@ static int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t num } for (int j = 0; j < point->tagNum; ++j) { TAOS_SML_KV* kv = point->tags + j; - int32_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); + size_t idx = TARRAY_ELEM_IDX(point->schema->tags, kv->schema); TAOS_BIND* bind = taosArrayGet(tagBinds, idx); bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); @@ -750,7 +751,7 @@ static int32_t insertPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t num } for (int j = 0; j < point->fieldNum; ++j) { TAOS_SML_KV* kv = point->fields + j; - int32_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); + size_t idx = TARRAY_ELEM_IDX(point->schema->fields, kv->schema); TAOS_BIND* bind = colBinds + idx; bind->buffer_type = kv->type; bind->length = malloc(sizeof(uintptr_t*)); @@ -793,10 +794,24 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { int32_t code = TSDB_CODE_SUCCESS; SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray - buildDataPointSchemas(points, numPoint, stableSchemas); - reconcileDBSchemas(taos, stableSchemas); - insertPoints(taos, points, numPoint); + code = buildDataPointSchemas(points, numPoint, stableSchemas); + if (code != 0) { + tscError("error building data point schemas : %s", tstrerror(code)); + goto clean_up; + } + code = reconcileDBSchemas(taos, stableSchemas); + if (code != 0) { + tscError("error change db schema : %s", tstrerror(code)); + goto clean_up; + } + + code = insertPoints(taos, points, numPoint); + if (code != 0) { + tscError("error insert points : %s", tstrerror(code)); + } + +clean_up: for (int i = 0; i < taosArrayGetSize(stableSchemas); ++i) { SSmlSTableSchema* schema = taosArrayGet(stableSchemas, i); taosArrayDestroy(schema->fields); @@ -806,11 +821,6 @@ int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) { return code; } - -//todo: table/column length check -//todo: type check -//todo: taosmbs2ucs4 check - //========================================================================= typedef enum { @@ -907,25 +917,25 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(int8_t*)(item->value) = strtoll(sv, &endptr, 10); + *(int8_t*)(item->value) = (int8_t)strtoll(sv, &endptr, 10); } else if (*last == 's') { item->type = TSDB_DATA_TYPE_SMALLINT; item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(int16_t*)(item->value) = strtoll(sv, &endptr, 10); + *(int16_t*)(item->value) = (int16_t)strtoll(sv, &endptr, 10); } else if (*last == 'w') { item->type = TSDB_DATA_TYPE_INT; item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(int32_t*)(item->value) = strtoll(sv, &endptr, 10); + *(int32_t*)(item->value) = (int32_t)strtoll(sv, &endptr, 10); } else if (*last == 'f') { item->type = TSDB_DATA_TYPE_FLOAT; item->length = (int16_t)tDataTypes[item->type].bytes; item->value = malloc(item->length); char* endptr = NULL; - *(float*)(item->value) = strtold(sv, &endptr); + *(float*)(item->value) = (float)strtold(sv, &endptr); } else { item->type = TSDB_DATA_TYPE_DOUBLE; item->length = (int16_t)tDataTypes[item->type].bytes; @@ -1073,7 +1083,7 @@ int32_t tscParseLine(SStrToken line, SLPPoint* point) { int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { for (int32_t i = 0; i < numLines; ++i) { - SStrToken tkLine = {.z = lines[i], .n = strlen(lines[i])+1}; + SStrToken tkLine = {.z = lines[i], .n = (uint32_t)strlen(lines[i])+1}; SLPPoint point; tscParseLine(tkLine, &point); taosArrayPush(points, &point); @@ -1126,7 +1136,7 @@ int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { size_t lpTagSize = taosArrayGetSize(lpPoint->tags); point->tags = calloc(lpTagSize, sizeof(TAOS_SML_KV)); - point->tagNum = lpTagSize; + point->tagNum = (int)lpTagSize; for (int j=0; jtags, j); TAOS_SML_KV* tagKv = point->tags + j; @@ -1144,7 +1154,7 @@ int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { size_t lpFieldsSize = taosArrayGetSize(lpPoint->fields); point->fields = calloc(lpFieldsSize + 1, sizeof(TAOS_SML_KV)); - point->fieldNum = lpFieldsSize + 1; + point->fieldNum = (int)(lpFieldsSize + 1); TAOS_SML_KV* tsField = point->fields + 0; char tsKey[256]; @@ -1174,7 +1184,7 @@ int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { } } - taos_sml_insert(taos, points, numPoints); + taos_sml_insert(taos, points, (int)numPoints); for (int i=0; i Date: Tue, 13 Jul 2021 22:41:04 +0800 Subject: [PATCH 36/43] add more data type --- src/client/src/tscParseLineProtocol.c | 18 +++++++++++++++++- tests/examples/c/apitest.c | 9 ++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index a198a7b42a..9f86d780d3 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -912,6 +912,12 @@ int32_t parseItemValue(SLPItem* item, LPItemKind kind) { item->value = malloc(item->length); char* endptr = NULL; *(int64_t*)(item->value) = strtoll(sv, &endptr, 10); + } else if (*last == 'u') { + item->type = TSDB_DATA_TYPE_UBIGINT; + item->length = (int16_t)tDataTypes[item->type].bytes; + item->value = malloc(item->length); + char* endptr = NULL; + *(uint64_t*)(item->value) = (uint64_t)strtoull(sv, &endptr, 10); } else if (*last == 'b') { item->type = TSDB_DATA_TYPE_TINYINT; item->length = (int16_t)tDataTypes[item->type].bytes; @@ -989,6 +995,16 @@ int32_t parsePointTime(SLPPoint* point) { } else { char* endptr = NULL; point->ts = strtoll(point->tsToken.z, &endptr, 10); + char* last = point->tsToken.z + point->tsToken.n - 1; + if (*last == 's') { + point->ts *= (int64_t)1e9; + } else if (*last == 'a') { + point->ts *= (int64_t)1e6; + } else if (*last == 'u') { + point->ts *= (int64_t)1e3; + } else if (*last == 'b') { + point->ts *= 1; + } } return 0; } @@ -1083,7 +1099,7 @@ int32_t tscParseLine(SStrToken line, SLPPoint* point) { int32_t tscParseLines(char* lines[], int numLines, SArray* points, SArray* failedLines) { for (int32_t i = 0; i < numLines; ++i) { - SStrToken tkLine = {.z = lines[i], .n = (uint32_t)strlen(lines[i])+1}; + SStrToken tkLine = {.z = lines[i], .n = (uint32_t)strlen(lines[i])}; SLPPoint point; tscParseLine(tkLine, &point); taosArrayPush(points, &point); diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 376b00ef71..ed76e7b2b0 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -966,13 +966,16 @@ int32_t verify_schema_less(TAOS* taos) { char* lines[] = { "st,t1=3i,t2=4,t3=\"t3\" c1=3i,c3=L\"passit\",c2=false,c4=4 1626006833639000000", "st,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagin\",c2=true,c4=5,c5=5 1626006833640000000", - "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", "ste,t2=5,t3=L\"ste\" c1=true,c2=4,c3=\"iam\" 1626056811823316532", + "st,t1=4i,t2=5,t3=\"t4\" c1=3i,c3=L\"passitagain\",c2=true,c4=5 1626006833642000000", "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false 1626056811843316532", - "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false,c5=32b,c6=64s,c7=32w,c8=88.88f 1626056812843316532" + "ste,t2=5,t3=L\"ste2\" c3=\"iamszhou\",c4=false,c5=32b,c6=64s,c7=32w,c8=88.88f 1626056812843316532", + "st,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagin\",c2=true,c4=5,c5=5,c6=7u 1626006933640000000", + "stf,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagin\",c2=true,c4=5,c5=5,c6=7u 1626006933640000000", + "stf,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagin_stf\",c2=false,c5=5,c6=7u 1626006933641a" }; - //int code = taos_insert_by_lines(taos, lines , 6); +// int code = taos_insert_by_lines(taos, lines , sizeof(lines)/sizeof(char*)); int code = taos_insert_by_lines(taos, &lines[0], 1); code = taos_insert_by_lines(taos, &lines[1], 1); From d95f27acf684a8fbb0cec4e001342b36fb472fbe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 13 Jul 2021 23:03:07 +0800 Subject: [PATCH 37/43] [td-225]refactor the log --- src/client/src/tscSubquery.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 8ab3512cba..f10646e3a3 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2542,7 +2542,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlObj* pSub = pSql->pSubs[j]; SRetrieveSupport* pSupport = pSub->param; - tscDebug("0x%"PRIx64" sub:%p launch subquery, orderOfSub:%d.", pSql->self, pSub, pSupport->subqueryIndex); + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); tscBuildAndSendRequest(pSub, NULL); } @@ -2861,8 +2861,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscDebug("0x%"PRIx64" sub:%p retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", - pParentSql->self, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", + pParentSql->self, pSql->self, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd)->distinctTag)) { tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, From 6b404cd1cee243b63f5a32324f1c277094f73f20 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Wed, 14 Jul 2021 07:48:55 +0800 Subject: [PATCH 38/43] change api function name taos_insert_by_lines to taos_insert_lines --- src/client/src/tscParseLineProtocol.c | 12 +++++++----- src/inc/taos.h | 2 +- tests/examples/c/apitest.c | 6 +++--- tests/tsim/src/simExe.c | 4 ++-- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscParseLineProtocol.c b/src/client/src/tscParseLineProtocol.c index 9f86d780d3..37264e8eaa 100644 --- a/src/client/src/tscParseLineProtocol.c +++ b/src/client/src/tscParseLineProtocol.c @@ -2,19 +2,21 @@ #include #include #include + #include "os.h" #include "osString.h" #include "ttype.h" #include "tmd5.h" #include "tstrbuild.h" #include "tname.h" -#include "taos.h" -#include "tsclient.h" -#include "tscLog.h" #include "hash.h" #include "tskiplist.h" -#include "tscUtil.h" +#include "tscUtil.h" +#include "tsclient.h" +#include "tscLog.h" + +#include "taos.h" typedef struct { char sTableName[TSDB_TABLE_NAME_LEN]; SHashObj* tagHash; @@ -1137,7 +1139,7 @@ void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) { free(point->childTableName); } -int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines) { +int taos_insert_lines(TAOS* taos, char* lines[], int numLines) { SArray* lpPoints = taosArrayInit(numLines, sizeof(SLPPoint)); tscParseLines(lines, numLines, lpPoints, NULL); diff --git a/src/inc/taos.h b/src/inc/taos.h index ca18c4fb93..a62f387924 100644 --- a/src/inc/taos.h +++ b/src/inc/taos.h @@ -169,7 +169,7 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); -DLL_EXPORT int taos_insert_by_lines(TAOS* taos, char* lines[], int numLines); +DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines); #ifdef __cplusplus } diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index ed76e7b2b0..a377bbc7b4 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -975,9 +975,9 @@ int32_t verify_schema_less(TAOS* taos) { "stf,t1=4i,t3=\"t4\",t2=5,t4=5 c1=3i,c3=L\"passitagin_stf\",c2=false,c5=5,c6=7u 1626006933641a" }; -// int code = taos_insert_by_lines(taos, lines , sizeof(lines)/sizeof(char*)); - int code = taos_insert_by_lines(taos, &lines[0], 1); - code = taos_insert_by_lines(taos, &lines[1], 1); +// int code = taos_insert_lines(taos, lines , sizeof(lines)/sizeof(char*)); + int code = taos_insert_lines(taos, &lines[0], 1); + code = taos_insert_lines(taos, &lines[1], 1); return code; } diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 2615448e4b..a05f46ce0d 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -1079,7 +1079,7 @@ bool simExecuteLineInsertCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); char * lines[] = {rest}; - int32_t ret = taos_insert_by_lines(script->taos, lines, 1); + int32_t ret = taos_insert_lines(script->taos, lines, 1); if (ret == TSDB_CODE_SUCCESS) { simDebug("script:%s, taos:%p, %s executed. success.", script->fileName, script->taos, rest); script->linePos++; @@ -1102,7 +1102,7 @@ bool simExecuteLineInsertErrorCmd(SScript *script, char *rest) { simInfo("script:%s, %s", script->fileName, rest); simLogSql(buf, true); char * lines[] = {rest}; - int32_t ret = taos_insert_by_lines(script->taos, lines, 1); + int32_t ret = taos_insert_lines(script->taos, lines, 1); if (ret == TSDB_CODE_SUCCESS) { sprintf(script->error, "script:%s, taos:%p, %s executed. expect failed, but success.", script->fileName, script->taos, rest); script->linePos++; From 420a1dfe05a5737ded2bc00a634422003f961efb Mon Sep 17 00:00:00 2001 From: glzhao89 Date: Wed, 14 Jul 2021 09:16:00 +0800 Subject: [PATCH 39/43] [TD-5150]: fix unitialized semaphore when running test suites on MacOS --- src/client/src/tscUtil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 173db05e10..02aeb3e33d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -3608,6 +3608,7 @@ void executeQuery(SSqlObj* pSql, SQueryInfo* pQueryInfo) { pNew->signature = pNew; pNew->sqlstr = strdup(pSql->sqlstr); // todo refactor pNew->fp = tscSubqueryCompleteCallback; + tsem_init(&pNew->rspSem, 0, 0); SRetrieveSupport* ps = calloc(1, sizeof(SRetrieveSupport)); // todo use object id ps->pParentSql = pSql; From 76e28e294cd2ae7acdb884cc28cb6d0907ec6c6c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 14 Jul 2021 10:31:57 +0800 Subject: [PATCH 40/43] [td-225]update the test script. --- .../general/parser/select_with_tags.sim | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index 9d5214bfaa..514aabe56f 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -169,32 +169,32 @@ if $rows != 12800 then return -1 endi -sql select top(c1, 100), tbname, t1, t2 from select_tags_mt0; -if $rows != 100 then +sql select top(c1, 80), tbname, t1, t2 from select_tags_mt0; +if $rows != 80 then return -1 endi -if $data00 != @70-01-01 08:03:30.100@ then +if $data00 != @70-01-01 08:03:40.100@ then return -1 endi -if $data10 != @70-01-01 08:03:30.200@ then +if $data10 != @70-01-01 08:03:40.200@ then return -1 endi -if $data01 != 110 then +if $data01 != 111 then return -1 endi -if $data02 != @select_tags_tb11@ then +if $data02 != @select_tags_tb12@ then return -1 endi -if $data03 != 11 then +if $data03 != 12 then return -1 endi -if $data04 != @abc11@ then +if $data04 != @abc12@ then return -1 endi @@ -227,8 +227,8 @@ if $data04 != @abc12@ then return -1 endi -sql select bottom(c1, 100), tbname, t1, t2 from select_tags_mt0; -if $rows != 100 then +sql select bottom(c1, 72), tbname, t1, t2 from select_tags_mt0; +if $rows != 72 then return -1 endi From 44cd80b3e1224c59f89954c199fa02b13a59e4fd Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 14 Jul 2021 11:32:53 +0800 Subject: [PATCH 41/43] [TD-5255] : update getting-start doc to fit latest taosdemo feature. --- documentation20/cn/02.getting-started/docs.md | 98 +++++++++---------- 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index 6eb58a1433..fa36481646 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -2,25 +2,25 @@ ## 快捷安装 -TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 +TDengine 软件分为服务器、客户端和报警模块三部分，目前 2.0 版服务器仅能在 Linux 系统上安装和运行，后续会支持 Windows、Mac OS 等系统。客户端可以在 Windows 或 Linux 上安装和运行。任何 OS 的应用也可以选择 RESTful 接口连接服务器 taosd。CPU 支持 X64/ARM64/MIPS64/Alpha64，后续会支持 ARM32、RISC-V 等 CPU 架构。用户可根据需求选择通过 [源码](https://www.taosdata.com/cn/getting-started/#通过源码安装) 或者 [安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装) 来安装。 ### 通过源码安装 -请参考我们的[TDengine github主页](https://github.com/taosdata/TDengine)下载源码并安装. +请参考我们的 [TDengine github 主页](https://github.com/taosdata/TDengine) 下载源码并安装. -### 通过Docker容器运行 +### 通过 Docker 容器运行 -暂时不建议生产环境采用 Docker 来部署 TDengine 的客户端或服务端，但在开发环境下或初次尝试时，使用 Docker 方式部署是十分方便的。特别是，利用 Docker，可以方便地在 Mac OSX 和 Windows 环境下尝试 TDengine。 +暂时不建议生产环境采用 Docker 来部署 TDengine 的客户端或服务端，但在开发环境下或初次尝试时，使用 Docker 方式部署是十分方便的。特别是，利用 Docker，可以方便地在 Mac OS X 和 Windows 环境下尝试 TDengine。 -详细操作方法请参照 [通过Docker快速体验TDengine](https://www.taosdata.com/cn/documentation/getting-started/docker)。 +详细操作方法请参照 [通过 Docker 快速体验 TDengine](https://www.taosdata.com/cn/documentation/getting-started/docker)。 ### 通过安装包安装 -TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择： +TDengine 的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择： -安装包下载在[这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。 +安装包下载在 [这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。 -具体的安装过程，请参见[TDengine多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html)以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。 +具体的安装过程，请参见 [TDengine 多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html) 以及 [视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。 ## 轻松启动 @@ -53,21 +53,21 @@ $ systemctl status taosd 如果系统中不支持 systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 -## TDengine命令行程序 +## TDengine 命令行程序 -执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。 +执行 TDengine 命令行程序，您只要在 Linux 终端执行 `taos` 即可。 ```bash $ taos ``` -如果TDengine终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/documentation/faq/)来解决终端连接服务端失败的问题）。TDengine终端的提示符号如下： +如果 TDengine 终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](https://www.taosdata.com/cn/documentation/faq/) 来解决终端连接服务端失败的问题）。TDengine 终端的提示符号如下： ```cmd taos> ``` -在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例： +在 TDengine 终端中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例： ```mysql create database demo; @@ -76,24 +76,24 @@ create table t (ts timestamp, speed int); insert into t values ('2019-07-15 00:00:00', 10); insert into t values ('2019-07-15 01:00:00', 20); select * from t; - ts | speed | -=================================== - 19-07-15 00:00:00.000| 10| - 19-07-15 01:00:00.000| 20| -Query OK, 2 row(s) in set (0.001700s) + ts | speed | +======================================== + 2019-07-15 00:00:00.000 | 10 | + 2019-07-15 01:00:00.000 | 20 | +Query OK, 2 row(s) in set (0.003128s) ``` -除执行SQL语句外，系统管理员还可以从TDengine终端检查系统运行状态，添加删除用户账号等。 +除执行 SQL 语句外，系统管理员还可以从 TDengine 终端检查系统运行状态，添加删除用户账号等。 ### 命令行参数 -您可通过配置命令行参数来改变TDengine终端的行为。以下为常用的几个命令行参数： +您可通过配置命令行参数来改变 TDengine 终端的行为。以下为常用的几个命令行参数： -- -c, --config-dir: 指定配置文件目录，默认为_/etc/taos_ -- -h, --host: 指定服务的IP地址，默认为本地服务 -- -s, --commands: 在不进入终端的情况下运行TDengine命令 -- -u, -- user: 连接TDengine服务器的用户名，缺省为root -- -p, --password: 连接TDengine服务器的密码，缺省为taosdata +- -c, --config-dir: 指定配置文件目录，默认为 _/etc/taos_ +- -h, --host: 指定服务的 FQDN 地址（也可以使用 IP），默认为连接本地服务 +- -s, --commands: 在不进入终端的情况下运行 TDengine 命令 +- -u, --user: 连接 TDengine 服务器的用户名，缺省为 root +- -p, --password: 连接TDengine服务器的密码，缺省为 taosdata - -?, --help: 打印出所有命令行参数 示例： @@ -102,7 +102,7 @@ Query OK, 2 row(s) in set (0.001700s) $ taos -h 192.168.0.1 -s "use db; show tables;" ``` -### 运行SQL命令脚本 +### 运行 SQL 命令脚本 TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本. @@ -110,27 +110,27 @@ TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本. taos> source ; ``` -### Shell小技巧 +### Shell 小技巧 - 可以使用上下光标键查看历史输入的指令 -- 修改用户密码。在 shell 中使用 alter user 指令 +- 修改用户密码，在 shell 中使用 alter user 指令 - ctrl+c 中止正在进行中的查询 - 执行 `RESET QUERY CACHE` 清空本地缓存的表 schema ## TDengine 极速体验 -启动TDengine的服务，在Linux终端执行taosdemo +启动 TDengine 的服务，在 Linux 终端执行 taosdemo ```bash $ taosdemo ``` -该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。 +该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "t0" 到 "t9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。 -执行这条命令大概需要10分钟，最后共插入10亿条记录。 +执行这条命令大概需要几分钟，最后共插入 1 亿条记录。 -在TDengine客户端输入查询命令，体验查询速度。 +在 TDengine 客户端输入查询命令，体验查询速度。 - 查询超级表下记录总条数： @@ -138,49 +138,43 @@ $ taosdemo taos> select count(*) from test.meters; ``` -- 查询10亿条记录的平均值、最大值、最小值等： +- 查询 1 亿条记录的平均值、最大值、最小值等： ```mysql -taos> select avg(f1), max(f2), min(f3) from test.meters; +taos> select avg(current), max(voltage), min(phase) from test.meters; ``` -- 查询loc="beijing"的记录总条数： +- 查询 location="beijing" 的记录总条数： ```mysql -taos> select count(*) from test.meters where loc="beijing"; +taos> select count(*) from test.meters where location="beijing"; ``` -- 查询areaid=10的所有记录的平均值、最大值、最小值等： +- 查询 groupdId=10 的所有记录的平均值、最大值、最小值等： ```mysql -taos> select avg(f1), max(f2), min(f3) from test.meters where areaid=10; +taos> select avg(current), max(voltage), min(phase) from test.meters where groupdId=10; ``` -- 对表t10按10s进行平均值、最大值和最小值聚合统计： +- 对表 t10 按 10s 进行平均值、最大值和最小值聚合统计： ```mysql -taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); +taos> select avg(current), max(voltage), min(phase) from test.t10 interval(10s); ``` -**Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。 +**Note:** taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。 ## 客户端和报警模块 -如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux和Windows安装包如下： +如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux 和 Windows 安装包可以在 [这里](https://www.taosdata.com/cn/getting-started/#客户端) 下载。 -- TDengine-client-2.0.10.0-Linux-x64.tar.gz(3.0M) -- TDengine-client-2.0.10.0-Windows-x64.exe(2.8M) -- TDengine-client-2.0.10.0-Windows-x86.exe(2.8M) - -报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）： - -- TDengine-alert-2.0.10.0-Linux-x64.tar.gz (8.1M) +报警模块的 Linux 和 Windows 安装包请在 [所有下载链接](https://www.taosdata.com/cn/all-downloads/) 页面搜索“TDengine Alert Linux”章节或“TDengine Alert Windows”章节进行下载。使用方法请参考 [报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)。 ## 支持平台列表 -### TDengine服务器支持的平台列表 +### TDengine 服务器支持的平台列表 | | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | | -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | --------------------- | @@ -201,9 +195,9 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); -### TDengine客户端和连接器支持的平台列表 +### TDengine 客户端和连接器支持的平台列表 -目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。 +目前 TDengine 的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha 等硬件平台，以及 Linux/Win64/Win32 等开发环境。 对照矩阵如下： @@ -220,5 +214,5 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); 注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 -请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。 +请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector) 查看更详细的信息。 From a1f1315de382125aba443f47062c25b5945d166e Mon Sep 17 00:00:00 2001 From: weicz1221 <49259303+weicz1221@users.noreply.github.com> Date: Wed, 14 Jul 2021 16:13:27 +0800 Subject: [PATCH 42/43] fix sql regex (#6838) Co-authored-by: weicz --- .../jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index e3179bd317..efe3303bd9 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -110,7 +110,7 @@ public class Utils { return rawSql; // toLowerCase String preparedSql = rawSql.trim().toLowerCase(); - String[] clause = new String[]{"values\\s*\\(.*?\\)", "tags\\s*\\(.*?\\)", "where\\s*.*"}; + String[] clause = new String[]{"values\\s*\\([\\s\\S]*?\\)", "tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"}; Map placeholderPositions = new HashMap<>(); RangeSet clauseRangeSet = TreeRangeSet.create(); findPlaceholderPosition(preparedSql, placeholderPositions); From 80faa86c952dd5110862add4be9e4ecce1735def Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 14 Jul 2021 17:25:05 +0800 Subject: [PATCH 43/43] Feature/sangshuduo/td 5242 taosdemo support 4096 columns (#6856) * [TD-5136]: taosdemo simulate real senario. * update test case according to taosdemo change * adjust range of semi-random data. * make demo mode use different tag name and value. * [TD-5242]: taosdemo support max columns align with TSDB defines. --- src/kit/taosdemo/taosdemo.c | 107 ++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 42 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index fa3d263678..d088d015d5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -75,7 +75,7 @@ enum TEST_MODE { #define MAX_RECORDS_PER_REQ 32766 -#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. +#define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) @@ -84,26 +84,23 @@ enum TEST_MODE { #define MAX_PASSWORD_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 -#define MAX_DATA_SIZE (16*1024)+20 // max record len: 16*1024, timestamp string and ,('') need extra space -#define MAX_NUM_DATATYPE 10 +#define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. -#define MAX_SAMPLES_ONCE_FROM_FILE 10000 -#define MAX_NUM_DATATYPE 10 +#define MAX_SAMPLES_ONCE_FROM_FILE 10000 +#define MAX_NUM_COLUMNS (TSDB_MAX_COLUMNS - 1) // exclude first column timestamp -#define MAX_DB_COUNT 8 -#define MAX_SUPER_TABLE_COUNT 200 -#define MAX_COLUMN_COUNT 1024 -#define MAX_TAG_COUNT 128 +#define MAX_DB_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 200 -#define MAX_QUERY_SQL_COUNT 100 -#define MAX_QUERY_SQL_LENGTH 1024 +#define MAX_QUERY_SQL_COUNT 100 +#define MAX_QUERY_SQL_LENGTH 1024 -#define MAX_DATABASE_COUNT 256 -#define INPUT_BUF_LEN 256 +#define MAX_DATABASE_COUNT 256 +#define INPUT_BUF_LEN 256 #define DEFAULT_TIMESTAMP_STEP 1 @@ -218,7 +215,7 @@ typedef struct SArguments_S { bool performance_print; char * output_file; bool async_mode; - char * datatype[MAX_NUM_DATATYPE + 1]; + char * datatype[MAX_NUM_COLUMNS + 1]; uint32_t len_of_binary; uint32_t num_of_CPR; uint32_t num_of_threads; @@ -274,9 +271,9 @@ typedef struct SSuperTable_S { char tagsFile[MAX_FILE_NAME_LEN]; uint32_t columnCount; - StrColumn columns[MAX_COLUMN_COUNT]; + StrColumn columns[TSDB_MAX_COLUMNS]; uint32_t tagCount; - StrColumn tags[MAX_TAG_COUNT]; + StrColumn tags[TSDB_MAX_TAGS]; char* childTblName; char* colsOfCreateChildTable; @@ -565,6 +562,8 @@ double randdouble[MAX_PREPARED_RAND]; char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; +#define DEFAULT_DATATYPE_NUM 3 + SArguments g_args = { NULL, // metaFile 0, // test_mode @@ -595,7 +594,7 @@ SArguments g_args = { { "FLOAT", // datatype "INT", // datatype - "FLOAT", // datatype + "FLOAT", // datatype. DEFAULT_DATATYPE_NUM is 3 }, 16, // len_of_binary 4, // num_of_CPR @@ -725,9 +724,13 @@ static void printHelp() { "The data_type of columns, default: FLOAT, INT, FLOAT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s%d\n", indent, "-l", indent, - "The number of columns per record. Default is 3. Max values is ", - MAX_NUM_DATATYPE); + printf("%s%s%s%s%d%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is ", + DEFAULT_DATATYPE_NUM, + ". Max values is ", + MAX_NUM_COLUMNS); + printf("%s%s%s%s\n", indent, indent, indent, + "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -931,13 +934,16 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->num_of_CPR = atoi(argv[++i]); - if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { - printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + if (arguments->num_of_CPR > MAX_NUM_COLUMNS) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_COLUMNS); prompt(); - arguments->num_of_CPR = MAX_NUM_DATATYPE; + arguments->num_of_CPR = MAX_NUM_COLUMNS; } - for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + for (int col = DEFAULT_DATATYPE_NUM; col < arguments->num_of_CPR; col ++) { + arguments->datatype[col] = "INT"; + } + for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) { arguments->datatype[col] = NULL; } @@ -990,7 +996,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->datatype[index++] = token; token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; + if (index >= MAX_NUM_COLUMNS) break; } arguments->datatype[index] = NULL; } @@ -1086,7 +1092,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } int columnCount; - for (columnCount = 0; columnCount < MAX_NUM_DATATYPE; columnCount ++) { + for (columnCount = 0; columnCount < MAX_NUM_COLUMNS; columnCount ++) { if (g_args.datatype[columnCount] == NULL) { break; } @@ -1111,7 +1117,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->use_metric ? "true" : "false"); if (*(arguments->datatype)) { printf("# Specified data type: "); - for (int i = 0; i < MAX_NUM_DATATYPE; i++) + for (int i = 0; i < MAX_NUM_COLUMNS; i++) if (arguments->datatype[i]) printf("%s,", arguments->datatype[i]); else @@ -2389,8 +2395,15 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { tmfree(buf); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", strlen("int"))) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + if ((g_args.demo_mode) && (i == 0)) { + dataLen += snprintf(dataBuf + dataLen, + TSDB_MAX_SQL_LEN - dataLen, + "%d, ", tableSeq % 10); + } else { + dataLen += snprintf(dataBuf + dataLen, + TSDB_MAX_SQL_LEN - dataLen, "%d, ", tableSeq); + } } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", strlen("bigint"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, @@ -2787,16 +2800,26 @@ static int createSuperTable( char* dataType = superTbl->tags[tagIndex].dataType; if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, - "BINARY", superTbl->tags[tagIndex].dataLen); + if ((g_args.demo_mode) && (tagIndex == 1)) { + len += snprintf(tags + len, STRING_LEN - len, + "loction BINARY(%d), ", + superTbl->tags[tagIndex].dataLen); + } else { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", + tagIndex, "BINARY", superTbl->tags[tagIndex].dataLen); + } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "NCHAR", superTbl->tags[tagIndex].dataLen); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + if ((g_args.demo_mode) && (tagIndex == 0)) { + len += snprintf(tags + len, STRING_LEN - len, "groupId INT, "); + } else { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "INT"); + } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, @@ -3352,9 +3375,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } int columnSize = cJSON_GetArraySize(columns); - if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) { + if ((columnSize + 1/* ts */) > TSDB_MAX_COLUMNS) { errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", - __func__, __LINE__, MAX_COLUMN_COUNT); + __func__, __LINE__, TSDB_MAX_COLUMNS); goto PARSE_OVER; } @@ -3410,9 +3433,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } } - if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) { + if ((index + 1 /* ts */) > MAX_NUM_COLUMNS) { errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n", - __func__, __LINE__, MAX_COLUMN_COUNT); + __func__, __LINE__, MAX_NUM_COLUMNS); goto PARSE_OVER; } @@ -3429,9 +3452,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } int tagSize = cJSON_GetArraySize(tags); - if (tagSize > MAX_TAG_COUNT) { + if (tagSize > TSDB_MAX_TAGS) { errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", - __func__, __LINE__, MAX_TAG_COUNT); + __func__, __LINE__, TSDB_MAX_TAGS); goto PARSE_OVER; } @@ -3481,17 +3504,17 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } } - if (index > MAX_TAG_COUNT) { + if (index > TSDB_MAX_TAGS) { errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n", - __func__, __LINE__, MAX_TAG_COUNT); + __func__, __LINE__, TSDB_MAX_TAGS); goto PARSE_OVER; } superTbls->tagCount = index; - if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) { + if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > TSDB_MAX_COLUMNS) { errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n", - __func__, __LINE__, MAX_COLUMN_COUNT); + __func__, __LINE__, TSDB_MAX_COLUMNS); goto PARSE_OVER; } ret = true; @@ -7919,7 +7942,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; g_Dbs.db[0].superTbls[0].columnCount = 0; - for (int i = 0; i < MAX_NUM_DATATYPE; i++) { + for (int i = 0; i < MAX_NUM_COLUMNS; i++) { if (data_type[i] == NULL) { break; }