From bc2c2c07a174630a6d5a70cf3c7c740800a21e15 Mon Sep 17 00:00:00 2001 From: xywang Date: Tue, 1 Jun 2021 11:54:22 +0000 Subject: [PATCH 01/31] [TD-2574]: refactored algorithms for top and bottom functions. --- src/query/src/qAggMain.c | 201 +++++++++++++++++++++------------------ 1 file changed, 107 insertions(+), 94 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 4935633b9c..a7106a14a9 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2201,6 +2201,101 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ } while (0) +static void heapSwap(tValuePair *a, tValuePair *b, const int16_t tagLen) { + char tag[32768]; + tValuePair temp; + + memset(tag, 0, sizeof(tag)); + temp.pTags = tag; + + VALUEPAIRASSIGN(&temp, a, tagLen); + VALUEPAIRASSIGN(a, b, tagLen); + VALUEPAIRASSIGN(b, &temp, tagLen); +} + +static void heapAdjust(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t start, int32_t end, bool minRoot) { + int32_t parent = start; + int32_t child = 2 * parent + 1; + + while (child <= end) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (minRoot) { + if (child + 1 <= end && pList[child]->v.i64 < pList[child + 1]->v.i64) { + child++; + } + + if (pList[parent]->v.i64 > pList[child]->v.i64) { + break; + } + } else { + if (child + 1 <= end && pList[child]->v.i64 >= pList[child + 1]->v.i64) { + child++; + } + + if (pList[parent]->v.i64 <= pList[child]->v.i64) { + break; + } + } + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (minRoot) { + if (child + 1 <= end && pList[child]->v.u64 < pList[child + 1]->v.u64) { + child++; + } + + if (pList[parent]->v.u64 > pList[child]->v.u64) { + break; + } + } else { + if (child + 1 <= end && pList[child]->v.u64 >= pList[child + 1]->v.u64) { + child++; + } + + if (pList[parent]->v.u64 <= pList[child]->v.u64) { + break; + } + } + } else { + if (minRoot) { + if (child + 1 <= end && pList[child]->v.dKey < pList[child + 1]->v.dKey) { + child++; + } + + if (pList[parent]->v.dKey > pList[child]->v.dKey) { + break; + } + } else { + if (child + 1 <= end && pList[child]->v.dKey >= pList[child + 1]->v.dKey) { + child++; + } + + if (pList[parent]->v.dKey <= pList[child]->v.dKey) { + break; + } + } + } + + heapSwap(pList[parent], pList[child], tagLen); + + parent = child; + child = parent * 2 + 1; + } +} + +void heapSort(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t len, bool minRoot) { + int32_t i; + + for (i = len / 2 - 1; i >= 0; i--) { + heapAdjust(pList, type, i, tagLen, len - 1, minRoot); + } + +/* + for (i = len - 1; i > 0; i--) { + heapSwap(pList[0], pList[i], tagsLen); + heapAdjust(pList, type, tagsLen, i - 1, minRoot); + } +*/ +} + static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, SExtTagsInfo *pTagInfo, char *pTags, int16_t stage) { tVariant val = {0}; @@ -2210,59 +2305,17 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, assert(pList != NULL); if (pInfo->num < maxLen) { - if (pInfo->num == 0 || - (IS_SIGNED_NUMERIC_TYPE(type) && val.i64 >= pList[pInfo->num - 1]->v.i64) || - (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 >= pList[pInfo->num - 1]->v.u64) || - (IS_FLOAT_TYPE(type) && val.dKey >= pList[pInfo->num - 1]->v.dKey)) { - valuePairAssign(pList[pInfo->num], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); - } else { - int32_t i = pInfo->num - 1; - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.i64 > val.i64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.u64 > val.u64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else { - while (i >= 0 && pList[i]->v.dKey > val.dKey) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } - - valuePairAssign(pList[i + 1], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage); - } - + valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + + heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 0); + pInfo->num++; } else { - int32_t i = 0; - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 > pList[0]->v.i64) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { - // find the appropriate the slot position - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.i64 < val.i64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.u64 < val.u64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } else { - while (i + 1 < maxLen && pList[i + 1]->v.dKey < val.dKey) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } - - valuePairAssign(pList[i], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 0); } } } @@ -2276,57 +2329,17 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa assert(pList != NULL); if (pInfo->num < maxLen) { - if (pInfo->num == 0) { - valuePairAssign(pList[pInfo->num], type, (const char*) &val.i64, ts, pTags, pTagInfo, stage); - } else { - int32_t i = pInfo->num - 1; - - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.i64 < val.i64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i >= 0 && pList[i]->v.u64 < val.u64) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } else { - while (i >= 0 && pList[i]->v.dKey < val.dKey) { - VALUEPAIRASSIGN(pList[i + 1], pList[i], pTagInfo->tagsLen); - i -= 1; - } - } - - valuePairAssign(pList[i + 1], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); - } - + valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + + heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 1); + pInfo->num++; } else { - int32_t i = 0; - if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i64 < pList[0]->v.i64) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { - // find the appropriate the slot position - if (IS_SIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.i64 > val.i64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - while (i + 1 < maxLen && pList[i + 1]->v.u64 > val.u64) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } else { - while (i + 1 < maxLen && pList[i + 1]->v.dKey > val.dKey) { - VALUEPAIRASSIGN(pList[i], pList[i + 1], pTagInfo->tagsLen); - i += 1; - } - } - - valuePairAssign(pList[i], type, (const char*)&val.i64, ts, pTags, pTagInfo, stage); + valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); + heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 1); } } } From 0ef76e0c1dba1203297e1b70c11922c5ebc27f9a Mon Sep 17 00:00:00 2001 From: xywang Date: Wed, 2 Jun 2021 05:50:11 +0000 Subject: [PATCH 02/31] [TD-2574]: fixed a bug in heapSort. --- src/query/src/qAggMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index a7106a14a9..b0329c845d 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2285,13 +2285,13 @@ void heapSort(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t len, bo int32_t i; for (i = len / 2 - 1; i >= 0; i--) { - heapAdjust(pList, type, i, tagLen, len - 1, minRoot); + heapAdjust(pList, type, tagLen, i, len - 1, minRoot); } /* for (i = len - 1; i > 0; i--) { heapSwap(pList[0], pList[i], tagsLen); - heapAdjust(pList, type, tagsLen, i - 1, minRoot); + heapAdjust(pList, type, 0, tagsLen, i - 1, minRoot); } */ } From e7753404758bf8c9a3896d4936ae4b0aab54a21e Mon Sep 17 00:00:00 2001 From: xywang Date: Sat, 19 Jun 2021 16:25:04 +0800 Subject: [PATCH 03/31] [TD-2574]: added general heapsort algorithm --- src/query/src/qAggMain.c | 124 ++++++++++----------------------------- src/util/inc/talgo.h | 33 +++++++++++ src/util/src/talgo.c | 86 +++++++++++++++++++++++++++ 3 files changed, 149 insertions(+), 94 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index b0329c845d..2c7242b26a 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2201,99 +2201,35 @@ static void valuePairAssign(tValuePair *dst, int16_t type, const char *val, int6 memcpy((dst)->pTags, (src)->pTags, (size_t)(__l)); \ } while (0) -static void heapSwap(tValuePair *a, tValuePair *b, const int16_t tagLen) { - char tag[32768]; - tValuePair temp; +static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) +{ + uint16_t type = *(uint16_t *) param; + tValuePair *val1 = *(tValuePair **) p1; + tValuePair *val2 = *(tValuePair **) p2; + + if (IS_SIGNED_NUMERIC_TYPE(type)) { + return val1->v.i64 - val2->v.i64; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + return val1->v.u64 - val2->v.u64; + } + + return val1->v.dKey - val2->v.dKey; +} + +static void topBotSwapFn(void *dst, void *src, const void *param) +{ + char tag[32768]; + tValuePair temp; + uint16_t tagLen = *(uint16_t *) param; + tValuePair *vdst = *(tValuePair **) dst; + tValuePair *vsrc = *(tValuePair **) src; memset(tag, 0, sizeof(tag)); temp.pTags = tag; - VALUEPAIRASSIGN(&temp, a, tagLen); - VALUEPAIRASSIGN(a, b, tagLen); - VALUEPAIRASSIGN(b, &temp, tagLen); -} - -static void heapAdjust(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t start, int32_t end, bool minRoot) { - int32_t parent = start; - int32_t child = 2 * parent + 1; - - while (child <= end) { - if (IS_SIGNED_NUMERIC_TYPE(type)) { - if (minRoot) { - if (child + 1 <= end && pList[child]->v.i64 < pList[child + 1]->v.i64) { - child++; - } - - if (pList[parent]->v.i64 > pList[child]->v.i64) { - break; - } - } else { - if (child + 1 <= end && pList[child]->v.i64 >= pList[child + 1]->v.i64) { - child++; - } - - if (pList[parent]->v.i64 <= pList[child]->v.i64) { - break; - } - } - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - if (minRoot) { - if (child + 1 <= end && pList[child]->v.u64 < pList[child + 1]->v.u64) { - child++; - } - - if (pList[parent]->v.u64 > pList[child]->v.u64) { - break; - } - } else { - if (child + 1 <= end && pList[child]->v.u64 >= pList[child + 1]->v.u64) { - child++; - } - - if (pList[parent]->v.u64 <= pList[child]->v.u64) { - break; - } - } - } else { - if (minRoot) { - if (child + 1 <= end && pList[child]->v.dKey < pList[child + 1]->v.dKey) { - child++; - } - - if (pList[parent]->v.dKey > pList[child]->v.dKey) { - break; - } - } else { - if (child + 1 <= end && pList[child]->v.dKey >= pList[child + 1]->v.dKey) { - child++; - } - - if (pList[parent]->v.dKey <= pList[child]->v.dKey) { - break; - } - } - } - - heapSwap(pList[parent], pList[child], tagLen); - - parent = child; - child = parent * 2 + 1; - } -} - -void heapSort(tValuePair **pList, uint16_t type, int16_t tagLen, int32_t len, bool minRoot) { - int32_t i; - - for (i = len / 2 - 1; i >= 0; i--) { - heapAdjust(pList, type, tagLen, i, len - 1, minRoot); - } - -/* - for (i = len - 1; i > 0; i--) { - heapSwap(pList[0], pList[i], tagsLen); - heapAdjust(pList, type, 0, tagsLen, i - 1, minRoot); - } -*/ + VALUEPAIRASSIGN(&temp, vdst, tagLen); + VALUEPAIRASSIGN(vdst, vsrc, tagLen); + VALUEPAIRASSIGN(vsrc, &temp, tagLen); } static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, int64_t ts, uint16_t type, @@ -2303,11 +2239,11 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, tValuePair **pList = pInfo->res; assert(pList != NULL); - + if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 0); + taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); pInfo->num++; } else { @@ -2315,7 +2251,7 @@ static void do_top_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pData, (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 > pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey > pList[0]->v.dKey)) { valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 0); + taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 0); } } } @@ -2331,7 +2267,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa if (pInfo->num < maxLen) { valuePairAssign(pList[pInfo->num], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapSort(pList, type, pTagInfo->tagsLen, pInfo->num + 1, 1); + taosheapsort((void *) pList, sizeof(tValuePair **), pInfo->num + 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); pInfo->num++; } else { @@ -2339,7 +2275,7 @@ static void do_bottom_function_add(STopBotInfo *pInfo, int32_t maxLen, void *pDa (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u64 < pList[0]->v.u64) || (IS_FLOAT_TYPE(type) && val.dKey < pList[0]->v.dKey)) { valuePairAssign(pList[0], type, (const char *)&val.i64, ts, pTags, pTagInfo, stage); - heapAdjust(pList, type, pTagInfo->tagsLen, 0, maxLen - 1, 1); + taosheapadjust((void *) pList, sizeof(tValuePair **), 0, maxLen - 1, (const void *) &type, topBotComparFn, (const void *) &pTagInfo->tagsLen, topBotSwapFn, 1); } } } diff --git a/src/util/inc/talgo.h b/src/util/inc/talgo.h index 9e3692225b..4aa5430605 100644 --- a/src/util/inc/talgo.h +++ b/src/util/inc/talgo.h @@ -34,6 +34,7 @@ typedef int (*__compar_fn_t) (const void *, const void *); #define elePtrAt(base, size, idx) (void *)((char *)(base) + (size) * (idx)) typedef int32_t (*__ext_compar_fn_t)(const void *p1, const void *p2, const void *param); +typedef void (*__ext_swap_fn_t)(void *p1, void *p2, const void *param); /** * quick sort, with the compare function requiring additional parameters support @@ -59,6 +60,38 @@ void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ex */ void *taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t fn, int flags); +/** + * adjust heap + * + * @param base: the start address of array + * @param size: size of every item in array + * @param start: the first index + * @param end: the last index + * @param parcompar: parameters for compare function + * @param compar: user defined compare function + * @param parswap: parameters for swap function + * @param swap: user defined swap function, the default swap function doswap will be used if swap is NULL + * @param maxroot: if heap is max root heap + * @return + */ +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); + +/** + * sort heap to make sure it is a max/min root heap + * + * @param base: the start address of array + * @param size: size of every item in array + * @param len: the length of array + * @param parcompar: parameters for compare function + * @param compar: user defined compare function + * @param parswap: parameters for swap function + * @param swap: user defined swap function, the default swap function doswap will be used if swap is NULL + * @param maxroot: if heap is max root heap + * @return + */ +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot); + + #ifdef __cplusplus } #endif diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 278683539e..54b7e00eb7 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -225,3 +225,89 @@ void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, return NULL; } + +void taosheapadjust(void *base, int32_t size, int32_t start, int32_t end, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) +{ + int32_t parent; + int32_t child; + char *buf; + + if (base && size > 0 && compar) { + parent = start; + child = 2 * parent + 1; + + if (swap == NULL) { + buf = calloc(1, size); + if (buf == NULL) { + return; + } + } + + if (maxroot) { + while (child <= end) { + if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) < 0) { + child++; + } + + if ((*compar)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parcompar) > 0) { + break; + } + + if (swap == NULL) { + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); + } else { + (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); + } + + parent = child; + child = 2 * parent + 1; + } + } else { + while (child <= end) { + if (child + 1 <= end && (*compar)(elePtrAt(base, size, child), elePtrAt(base, size, child + 1), parcompar) > 0) { + child++; + } + + if ((*compar)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parcompar) < 0) { + break; + } + + if (swap == NULL) { + doswap(elePtrAt(base, size, parent), elePtrAt(base, size, child), size, buf); + } else { + (*swap)(elePtrAt(base, size, parent), elePtrAt(base, size, child), parswap); + } + + parent = child; + child = 2 * parent + 1; + } + } + + if (swap == NULL) { + tfree(buf); + } + } +} + +void taosheapsort(void *base, int32_t size, int32_t len, const void *parcompar, __ext_compar_fn_t compar, const void *parswap, __ext_swap_fn_t swap, bool maxroot) +{ + int32_t i; + + if (base && size > 0) { + for (i = len / 2 - 1; i >= 0; i--) { + taosheapadjust(base, size, i, len - 1, parcompar, compar, parswap, swap, maxroot); + } + } + +/* + char *buf = calloc(1, size); + + for (i = len - 1; i > 0; i--) { + doswap(elePtrAt(base, size, 0), elePtrAt(base, size, i)); + taosheapadjust(base, size, 0, i - 1, parcompar, compar, parswap, swap, maxroot); + } + + tfree(buf); +*/ +} + From fe3cc4facd7306153bf41587ed77874d2fe3c46b Mon Sep 17 00:00:00 2001 From: xywang Date: Mon, 28 Jun 2021 19:32:10 +0800 Subject: [PATCH 04/31] [TD-2574]: fixed compilation errors. --- src/query/src/qAggMain.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 2c7242b26a..c8f4cf8dbd 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -2208,12 +2208,24 @@ static int32_t topBotComparFn(const void *p1, const void *p2, const void *param) tValuePair *val2 = *(tValuePair **) p2; if (IS_SIGNED_NUMERIC_TYPE(type)) { - return val1->v.i64 - val2->v.i64; - } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { - return val1->v.u64 - val2->v.u64; + if (val1->v.i64 == val2->v.i64) { + return 0; + } + + return (val1->v.i64 > val2->v.i64) ? 1 : -1; + } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { + if (val1->v.u64 == val2->v.u64) { + return 0; + } + + return (val1->v.u64 > val2->v.u64) ? 1 : -1; } - return val1->v.dKey - val2->v.dKey; + if (val1->v.dKey == val2->v.dKey) { + return 0; + } + + return (val1->v.dKey > val2->v.dKey) ? 1 : -1; } static void topBotSwapFn(void *dst, void *src, const void *param) From f28c1a6097c90d6a58afe7ca00d172ab3a2c444f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 13 Jul 2021 15:20:39 +0800 Subject: [PATCH 05/31] [td-5176]: improve the twa query performance in case of interval query. --- src/client/src/tscSQLParser.c | 68 ++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index ded334c69f..59807d1e2e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -96,7 +96,7 @@ static int32_t parseIntervalOffset(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrTo static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SStrToken* pSliding); static int32_t validateStateWindowNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode, bool isStable); -static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem); +static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery); static int32_t validateWhereNode(SQueryInfo* pQueryInfo, tSqlExpr** pExpr, SSqlObj* pSql); static int32_t validateFillNode(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSqlNode* pSqlNode); @@ -1809,8 +1809,8 @@ static bool hasNoneUserDefineExpr(SQueryInfo* pQueryInfo) { return false; } -int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool isSTable, bool joinQuery, - bool timeWindowQuery) { +int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pSelNodeList, bool joinQuery, + bool timeWindowQuery, bool outerQuery) { assert(pSelNodeList != NULL && pCmd != NULL); const char* msg1 = "too many items in selection clause"; @@ -1852,7 +1852,7 @@ int32_t validateSelectNodeList(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SArray* pS } else if (type == SQL_NODE_TABLE_COLUMN || type == SQL_NODE_VALUE) { // use the dynamic array list to decide if the function is valid or not // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 - if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { + if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem, outerQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } } else if (type == SQL_NODE_EXPR) { @@ -1988,14 +1988,15 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem) { +int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSqlExprItem* pItem, bool outerQuery) { const char* msg1 = "tag for normal table query is not allowed"; const char* msg2 = "invalid column name"; + const char* msg3 = "tbname not allowed in outer query"; int32_t startPos = (int32_t)tscNumOfExprs(pQueryInfo); - int32_t optr = pItem->pNode->tokenId; + int32_t tokenId = pItem->pNode->tokenId; - if (optr == TK_ALL) { // project on all fields + if (tokenId == TK_ALL) { // project on all fields TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); SColumnIndex index = COLUMN_INDEX_INITIALIZER; @@ -2019,7 +2020,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t if (pTableMeta->tableType != TSDB_TEMP_TABLE) { tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMeta->id.uid); } - } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query + } else if (tokenId == TK_STRING || tokenId == TK_INTEGER || tokenId == TK_FLOAT) { // simple column projection query SColumnIndex index = COLUMN_INDEX_INITIALIZER; // user-specified constant value as a new result column @@ -2027,13 +2028,13 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t index.tableIndex = 0; SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->value, &pItem->pNode->exprToken, pItem->aliasName); - SExprInfo* pExpr = - tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, getNewResColId(pCmd)); + SExprInfo* pExpr = tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC, + getNewResColId(pCmd)); // NOTE: the first parameter is reserved for the tag column id during join query process. pExpr->base.numOfParams = 2; tVariantAssign(&pExpr->base.param[1], &pItem->pNode->value); - } else if (optr == TK_ID) { + } else if (tokenId == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(&pItem->pNode->columnName, pQueryInfo, &index, tscGetErrorMsgPayload(pCmd)) != TSDB_CODE_SUCCESS) { @@ -2041,12 +2042,40 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema colSchema = *tGetTbnameColumnSchema(); - char name[TSDB_COL_NAME_LEN] = {0}; - getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); + if (outerQuery) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); + int32_t numOfCols = tscGetNumOfColumns(pTableMetaInfo->pTableMeta); - tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); - /*SExprInfo* pExpr = */tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG, getNewResColId(pCmd)); + bool existed = false; + SSchema* pSchema = pTableMetaInfo->pTableMeta->schema; + for (int32_t i = 0; i < numOfCols; ++i) { + if (strncasecmp(pSchema[i].name, TSQL_TBNAME_L, tListLen(pSchema[i].name)) == 0) { + existed = true; + index.columnIndex = i; + break; + } + } + + if (!existed) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3); + } + + SSchema colSchema = pSchema[index.columnIndex]; + char name[TSDB_COL_NAME_LEN] = {0}; + getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); + + tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); + /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, + TSDB_COL_NORMAL, getNewResColId(pCmd)); + } else { + SSchema colSchema = *tGetTbnameColumnSchema(); + char name[TSDB_COL_NAME_LEN] = {0}; + getColumnName(pItem, name, colSchema.name, sizeof(colSchema.name) - 1); + + tstrncpy(colSchema.name, name, TSDB_COL_NAME_LEN); + /*SExprInfo* pExpr = */ tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, + TSDB_COL_TAG, getNewResColId(pCmd)); + } } else { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -7156,8 +7185,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return code; } - bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (validateSelectNodeList(&pSql->cmd, pQueryInfo, pSqlNode->pSelNodeList, isSTable, false, false) != TSDB_CODE_SUCCESS) { + if (validateSelectNodeList(&pSql->cmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -7945,7 +7973,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf return TSDB_CODE_TSC_INVALID_OPERATION; } - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, timeWindowQuery, true) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -8085,7 +8113,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf int32_t timeWindowQuery = (TPARSER_HAS_TOKEN(pSqlNode->interval.interval) || TPARSER_HAS_TOKEN(pSqlNode->sessionVal.gap)); - if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, isSTable, joinQuery, timeWindowQuery) != + if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, joinQuery, timeWindowQuery, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } From d95f27acf684a8fbb0cec4e001342b36fb472fbe Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 13 Jul 2021 23:03:07 +0800 Subject: [PATCH 06/31] [td-225]refactor the log --- src/client/src/tscSubquery.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 8ab3512cba..f10646e3a3 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2542,7 +2542,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlObj* pSub = pSql->pSubs[j]; SRetrieveSupport* pSupport = pSub->param; - tscDebug("0x%"PRIx64" sub:%p launch subquery, orderOfSub:%d.", pSql->self, pSub, pSupport->subqueryIndex); + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" launch subquery, orderOfSub:%d.", pSql->self, pSub->self, pSupport->subqueryIndex); tscBuildAndSendRequest(pSub, NULL); } @@ -2861,8 +2861,8 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(pRes->numOfRows == numOfRows); int64_t num = atomic_add_fetch_64(&pState->numOfRetrievedRows, numOfRows); - tscDebug("0x%"PRIx64" sub:%p retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", - pParentSql->self, pSql, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); + tscDebug("0x%"PRIx64" sub:0x%"PRIx64" retrieve numOfRows:%d totalNumOfRows:%" PRIu64 " from ep:%s, orderOfSub:%d", + pParentSql->self, pSql->self, pRes->numOfRows, pState->numOfRetrievedRows, pSql->epSet.fqdn[pSql->epSet.inUse], idx); if (num > tsMaxNumOfOrderedResults && tscIsProjectionQueryOnSTable(pQueryInfo, 0) && !(tscGetQueryInfo(&pParentSql->cmd)->distinctTag)) { tscError("0x%"PRIx64" sub:0x%"PRIx64" num of OrderedRes is too many, max allowed:%" PRId32 " , current:%" PRId64, From 76e28e294cd2ae7acdb884cc28cb6d0907ec6c6c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 14 Jul 2021 10:31:57 +0800 Subject: [PATCH 07/31] [td-225]update the test script. --- .../general/parser/select_with_tags.sim | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/script/general/parser/select_with_tags.sim b/tests/script/general/parser/select_with_tags.sim index 9d5214bfaa..514aabe56f 100644 --- a/tests/script/general/parser/select_with_tags.sim +++ b/tests/script/general/parser/select_with_tags.sim @@ -169,32 +169,32 @@ if $rows != 12800 then return -1 endi -sql select top(c1, 100), tbname, t1, t2 from select_tags_mt0; -if $rows != 100 then +sql select top(c1, 80), tbname, t1, t2 from select_tags_mt0; +if $rows != 80 then return -1 endi -if $data00 != @70-01-01 08:03:30.100@ then +if $data00 != @70-01-01 08:03:40.100@ then return -1 endi -if $data10 != @70-01-01 08:03:30.200@ then +if $data10 != @70-01-01 08:03:40.200@ then return -1 endi -if $data01 != 110 then +if $data01 != 111 then return -1 endi -if $data02 != @select_tags_tb11@ then +if $data02 != @select_tags_tb12@ then return -1 endi -if $data03 != 11 then +if $data03 != 12 then return -1 endi -if $data04 != @abc11@ then +if $data04 != @abc12@ then return -1 endi @@ -227,8 +227,8 @@ if $data04 != @abc12@ then return -1 endi -sql select bottom(c1, 100), tbname, t1, t2 from select_tags_mt0; -if $rows != 100 then +sql select bottom(c1, 72), tbname, t1, t2 from select_tags_mt0; +if $rows != 72 then return -1 endi From 2acc0194c667ea04d740fb61ddaf94f8432fd6b4 Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 14 Jul 2021 11:17:01 +0800 Subject: [PATCH 08/31] fix bug --- src/query/inc/qUtil.h | 9 +++++++++ src/query/src/qExecutor.c | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 0756e41785..e20a9268d3 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -24,6 +24,15 @@ memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ } while (0) +#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \ + do { \ + assert(sizeof(_uid) == sizeof(uint64_t)); \ + *(void **)(_k) = (_buf); \ + *(uint64_t *)((_k) + POINTER_BYTES) = (_uid); \ + memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \ + } while (0) + + #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3d585afb87..2b2f1efceb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -433,7 +433,7 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult existed = (pResultRowInfo->pResult[0] == (*p1)); pResultRowInfo->curPos = 0; } else { // check if current pResultRowInfo contains the existed pResultRow - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid); + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (index != NULL) { pResultRowInfo->curPos = (int32_t) *index; @@ -471,7 +471,7 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; int64_t index = pResultRowInfo->curPos; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid); + SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); } @@ -1790,7 +1790,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf pRuntimeEnv->pResultRowHashTable = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pRuntimeEnv->pResultRowListSet = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t)); + pRuntimeEnv->keyBuf = malloc(pQueryAttr->maxTableColumnWidth + sizeof(int64_t) + POINTER_BYTES); pRuntimeEnv->pool = initResultRowPool(getResultRowSize(pRuntimeEnv)); pRuntimeEnv->prevRow = malloc(POINTER_BYTES * pQueryAttr->numOfCols + pQueryAttr->srcRowSize); From 72a36b080d57b8c0e1785a6e8c04d459f9b7e2af Mon Sep 17 00:00:00 2001 From: Jun Li Date: Tue, 13 Jul 2021 20:30:19 -0700 Subject: [PATCH 09/31] Remove unused code --- src/vnode/src/vnodeMain.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 979e4e4cdd..f826c1aecd 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -47,9 +47,6 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return terrno; } - char rootDir[TSDB_FILENAME_LEN] = {0}; - sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId); - char vnodeDir[TSDB_FILENAME_LEN] = "\0"; snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d", pVnodeCfg->cfg.vgId); if (tfsMkdir(vnodeDir) < 0) { @@ -63,23 +60,6 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) { return code; } - // STsdbCfg tsdbCfg = {0}; - // tsdbCfg.tsdbId = pVnodeCfg->cfg.vgId; - // tsdbCfg.cacheBlockSize = pVnodeCfg->cfg.cacheBlockSize; - // tsdbCfg.totalBlocks = pVnodeCfg->cfg.totalBlocks; - // tsdbCfg.daysPerFile = pVnodeCfg->cfg.daysPerFile; - // tsdbCfg.keep = pVnodeCfg->cfg.daysToKeep; - // tsdbCfg.keep1 = pVnodeCfg->cfg.daysToKeep1; - // tsdbCfg.keep2 = pVnodeCfg->cfg.daysToKeep2; - // tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock; - // tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; - // tsdbCfg.precision = pVnodeCfg->cfg.precision; - // tsdbCfg.compression = pVnodeCfg->cfg.compression; - // tsdbCfg.update = pVnodeCfg->cfg.update; - // tsdbCfg.cacheLastRow = pVnodeCfg->cfg.cacheLastRow; - - // char tsdbDir[TSDB_FILENAME_LEN] = {0}; - // sprintf(tsdbDir, "vnode/vnode%d/tsdb", pVnodeCfg->cfg.vgId); if (tsdbCreateRepo(pVnodeCfg->cfg.vgId) < 0) { vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno)); return TSDB_CODE_VND_INIT_FAILED; From 44cd80b3e1224c59f89954c199fa02b13a59e4fd Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Wed, 14 Jul 2021 11:32:53 +0800 Subject: [PATCH 10/31] [TD-5255] : update getting-start doc to fit latest taosdemo feature. --- documentation20/cn/02.getting-started/docs.md | 98 +++++++++---------- 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/documentation20/cn/02.getting-started/docs.md b/documentation20/cn/02.getting-started/docs.md index 6eb58a1433..fa36481646 100644 --- a/documentation20/cn/02.getting-started/docs.md +++ b/documentation20/cn/02.getting-started/docs.md @@ -2,25 +2,25 @@ ## 快捷安装 -TDengine软件分为服务器、客户端和报警模块三部分，目前2.0版服务器仅能在Linux系统上安装和运行，后续会支持Windows、mac OS等系统。客户端可以在Windows或Linux上安装和运行。任何OS的应用也可以选择RESTful接口连接服务器taosd。CPU支持X64/ARM64/MIPS64/Alpha64，后续会支持ARM32、RISC-V等CPU架构。用户可根据需求选择通过[源码](https://www.taosdata.com/cn/getting-started/#通过源码安装)或者[安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装)来安装。 +TDengine 软件分为服务器、客户端和报警模块三部分，目前 2.0 版服务器仅能在 Linux 系统上安装和运行，后续会支持 Windows、Mac OS 等系统。客户端可以在 Windows 或 Linux 上安装和运行。任何 OS 的应用也可以选择 RESTful 接口连接服务器 taosd。CPU 支持 X64/ARM64/MIPS64/Alpha64，后续会支持 ARM32、RISC-V 等 CPU 架构。用户可根据需求选择通过 [源码](https://www.taosdata.com/cn/getting-started/#通过源码安装) 或者 [安装包](https://www.taosdata.com/cn/getting-started/#通过安装包安装) 来安装。 ### 通过源码安装 -请参考我们的[TDengine github主页](https://github.com/taosdata/TDengine)下载源码并安装. +请参考我们的 [TDengine github 主页](https://github.com/taosdata/TDengine) 下载源码并安装. -### 通过Docker容器运行 +### 通过 Docker 容器运行 -暂时不建议生产环境采用 Docker 来部署 TDengine 的客户端或服务端，但在开发环境下或初次尝试时，使用 Docker 方式部署是十分方便的。特别是，利用 Docker，可以方便地在 Mac OSX 和 Windows 环境下尝试 TDengine。 +暂时不建议生产环境采用 Docker 来部署 TDengine 的客户端或服务端，但在开发环境下或初次尝试时，使用 Docker 方式部署是十分方便的。特别是，利用 Docker，可以方便地在 Mac OS X 和 Windows 环境下尝试 TDengine。 -详细操作方法请参照 [通过Docker快速体验TDengine](https://www.taosdata.com/cn/documentation/getting-started/docker)。 +详细操作方法请参照 [通过 Docker 快速体验 TDengine](https://www.taosdata.com/cn/documentation/getting-started/docker)。 ### 通过安装包安装 -TDengine的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择： +TDengine 的安装非常简单，从下载到安装成功仅仅只要几秒钟。服务端安装包包含客户端和连接器，我们提供三种安装包，您可以根据需要选择： -安装包下载在[这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。 +安装包下载在 [这里](https://www.taosdata.com/cn/getting-started/#通过安装包安装)。 -具体的安装过程，请参见[TDengine多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html)以及[视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。 +具体的安装过程，请参见 [TDengine 多种安装包的安装和卸载](https://www.taosdata.com/blog/2019/08/09/566.html) 以及 [视频教程](https://www.taosdata.com/blog/2020/11/11/1941.html)。 ## 轻松启动 @@ -53,21 +53,21 @@ $ systemctl status taosd 如果系统中不支持 systemd，也可以用手动运行 /usr/local/taos/bin/taosd 方式启动 TDengine 服务。 -## TDengine命令行程序 +## TDengine 命令行程序 -执行TDengine命令行程序，您只要在Linux终端执行`taos`即可。 +执行 TDengine 命令行程序，您只要在 Linux 终端执行 `taos` 即可。 ```bash $ taos ``` -如果TDengine终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考[FAQ](https://www.taosdata.com/cn/documentation/faq/)来解决终端连接服务端失败的问题）。TDengine终端的提示符号如下： +如果 TDengine 终端连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](https://www.taosdata.com/cn/documentation/faq/) 来解决终端连接服务端失败的问题）。TDengine 终端的提示符号如下： ```cmd taos> ``` -在TDengine终端中，用户可以通过SQL命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的SQL语句需要以分号结束来运行。示例： +在 TDengine 终端中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例： ```mysql create database demo; @@ -76,24 +76,24 @@ create table t (ts timestamp, speed int); insert into t values ('2019-07-15 00:00:00', 10); insert into t values ('2019-07-15 01:00:00', 20); select * from t; - ts | speed | -=================================== - 19-07-15 00:00:00.000| 10| - 19-07-15 01:00:00.000| 20| -Query OK, 2 row(s) in set (0.001700s) + ts | speed | +======================================== + 2019-07-15 00:00:00.000 | 10 | + 2019-07-15 01:00:00.000 | 20 | +Query OK, 2 row(s) in set (0.003128s) ``` -除执行SQL语句外，系统管理员还可以从TDengine终端检查系统运行状态，添加删除用户账号等。 +除执行 SQL 语句外，系统管理员还可以从 TDengine 终端检查系统运行状态，添加删除用户账号等。 ### 命令行参数 -您可通过配置命令行参数来改变TDengine终端的行为。以下为常用的几个命令行参数： +您可通过配置命令行参数来改变 TDengine 终端的行为。以下为常用的几个命令行参数： -- -c, --config-dir: 指定配置文件目录，默认为_/etc/taos_ -- -h, --host: 指定服务的IP地址，默认为本地服务 -- -s, --commands: 在不进入终端的情况下运行TDengine命令 -- -u, -- user: 连接TDengine服务器的用户名，缺省为root -- -p, --password: 连接TDengine服务器的密码，缺省为taosdata +- -c, --config-dir: 指定配置文件目录，默认为 _/etc/taos_ +- -h, --host: 指定服务的 FQDN 地址（也可以使用 IP），默认为连接本地服务 +- -s, --commands: 在不进入终端的情况下运行 TDengine 命令 +- -u, --user: 连接 TDengine 服务器的用户名，缺省为 root +- -p, --password: 连接TDengine服务器的密码，缺省为 taosdata - -?, --help: 打印出所有命令行参数 示例： @@ -102,7 +102,7 @@ Query OK, 2 row(s) in set (0.001700s) $ taos -h 192.168.0.1 -s "use db; show tables;" ``` -### 运行SQL命令脚本 +### 运行 SQL 命令脚本 TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本. @@ -110,27 +110,27 @@ TDengine 终端可以通过 `source` 命令来运行 SQL 命令脚本. taos> source ; ``` -### Shell小技巧 +### Shell 小技巧 - 可以使用上下光标键查看历史输入的指令 -- 修改用户密码。在 shell 中使用 alter user 指令 +- 修改用户密码，在 shell 中使用 alter user 指令 - ctrl+c 中止正在进行中的查询 - 执行 `RESET QUERY CACHE` 清空本地缓存的表 schema ## TDengine 极速体验 -启动TDengine的服务，在Linux终端执行taosdemo +启动 TDengine 的服务，在 Linux 终端执行 taosdemo ```bash $ taosdemo ``` -该命令将在数据库test下面自动创建一张超级表meters，该超级表下有1万张表，表名为"t0" 到"t9999"，每张表有10万条记录，每条记录有 （f1, f2， f3）三个字段，时间戳从"2017-07-14 10:40:00 000" 到"2017-07-14 10:41:39 999"，每张表带有标签areaid和loc, areaid被设置为1到10, loc被设置为"beijing"或者“shanghai"。 +该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "t0" 到 "t9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupdId，groupdId 被设置为 1 到 10， location 被设置为 "beijing" 或者 "shanghai"。 -执行这条命令大概需要10分钟，最后共插入10亿条记录。 +执行这条命令大概需要几分钟，最后共插入 1 亿条记录。 -在TDengine客户端输入查询命令，体验查询速度。 +在 TDengine 客户端输入查询命令，体验查询速度。 - 查询超级表下记录总条数： @@ -138,49 +138,43 @@ $ taosdemo taos> select count(*) from test.meters; ``` -- 查询10亿条记录的平均值、最大值、最小值等： +- 查询 1 亿条记录的平均值、最大值、最小值等： ```mysql -taos> select avg(f1), max(f2), min(f3) from test.meters; +taos> select avg(current), max(voltage), min(phase) from test.meters; ``` -- 查询loc="beijing"的记录总条数： +- 查询 location="beijing" 的记录总条数： ```mysql -taos> select count(*) from test.meters where loc="beijing"; +taos> select count(*) from test.meters where location="beijing"; ``` -- 查询areaid=10的所有记录的平均值、最大值、最小值等： +- 查询 groupdId=10 的所有记录的平均值、最大值、最小值等： ```mysql -taos> select avg(f1), max(f2), min(f3) from test.meters where areaid=10; +taos> select avg(current), max(voltage), min(phase) from test.meters where groupdId=10; ``` -- 对表t10按10s进行平均值、最大值和最小值聚合统计： +- 对表 t10 按 10s 进行平均值、最大值和最小值聚合统计： ```mysql -taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); +taos> select avg(current), max(voltage), min(phase) from test.t10 interval(10s); ``` -**Note:** taosdemo命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help`详细列出。您可以设置不同参数进行体验。 +**Note:** taosdemo 命令本身带有很多选项，配置表的数目、记录条数等等，请执行 `taosdemo --help` 详细列出。您可以设置不同参数进行体验。 ## 客户端和报警模块 -如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux和Windows安装包如下： +如果客户端和服务端运行在不同的电脑上，可以单独安装客户端。Linux 和 Windows 安装包可以在 [这里](https://www.taosdata.com/cn/getting-started/#客户端) 下载。 -- TDengine-client-2.0.10.0-Linux-x64.tar.gz(3.0M) -- TDengine-client-2.0.10.0-Windows-x64.exe(2.8M) -- TDengine-client-2.0.10.0-Windows-x86.exe(2.8M) - -报警模块的Linux安装包如下（请参考[报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)）： - -- TDengine-alert-2.0.10.0-Linux-x64.tar.gz (8.1M) +报警模块的 Linux 和 Windows 安装包请在 [所有下载链接](https://www.taosdata.com/cn/all-downloads/) 页面搜索“TDengine Alert Linux”章节或“TDengine Alert Windows”章节进行下载。使用方法请参考 [报警模块的使用方法](https://github.com/taosdata/TDengine/blob/master/alert/README_cn.md)。 ## 支持平台列表 -### TDengine服务器支持的平台列表 +### TDengine 服务器支持的平台列表 | | **CentOS 6/7/8** | **Ubuntu 16/18/20** | **Other Linux** | **统信 UOS** | **银河/中标麒麟** | **凝思 V60/V80** | **华为 EulerOS** | | -------------- | --------------------- | ------------------------ | --------------- | --------------- | ------------------------- | --------------------- | --------------------- | @@ -201,9 +195,9 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); -### TDengine客户端和连接器支持的平台列表 +### TDengine 客户端和连接器支持的平台列表 -目前TDengine的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha等硬件平台，以及Linux/Win64/Win32等开发环境。 +目前 TDengine 的连接器可支持的平台广泛，目前包括：X64/X86/ARM64/ARM32/MIPS/Alpha 等硬件平台，以及 Linux/Win64/Win32 等开发环境。 对照矩阵如下： @@ -220,5 +214,5 @@ taos> select avg(f1), max(f2), min(f3) from test.t10 interval(10s); 注： ● 表示经过官方测试验证， ○ 表示非官方测试验证。 -请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector)查看更详细的信息。 +请跳转到 [连接器](https://www.taosdata.com/cn/documentation/connector) 查看更详细的信息。 From 37e86d57e3024535552cf7ef736fb156e0c24a1a Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 14 Jul 2021 13:02:14 +0800 Subject: [PATCH 11/31] add debug info --- src/query/src/qExecutor.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2b2f1efceb..0f8bcc0712 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -437,6 +437,11 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (index != NULL) { pResultRowInfo->curPos = (int32_t) *index; + if (pResultRowInfo->size > 1) { + for(int32_t k = 0; k size - 1; ++k) { + assert(pResultRowInfo->pResult[k]->win.skey < pResultRowInfo->pResult[k+1]->win.skey); + } + } existed = true; } else { existed = false; @@ -470,6 +475,10 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult pResultRowInfo->curPos = pResultRowInfo->size; pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; + if (pResultRowInfo->curPos > 0) { + assert(pResultRowInfo->pResult[pResultRowInfo->curPos]->win.skey > pResultRowInfo->pResult[pResultRowInfo->curPos-1]->win.skey); + } + int64_t index = pResultRowInfo->curPos; SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); From e17466980806a73306ae16204f1aea5121db6e62 Mon Sep 17 00:00:00 2001 From: xywang Date: Wed, 14 Jul 2021 13:54:45 +0800 Subject: [PATCH 12/31] [TD-5169]: fixed a potential crash bug --- src/plugins/http/inc/httpSql.h | 3 ++ src/plugins/http/src/httpGcHandle.c | 37 +++++++++++++++-- src/plugins/http/src/httpSql.c | 62 +++++++++++++++++++++++++++++ src/plugins/http/src/httpTgHandle.c | 17 +++++++- 4 files changed, 115 insertions(+), 4 deletions(-) diff --git a/src/plugins/http/inc/httpSql.h b/src/plugins/http/inc/httpSql.h index db3e3a3b16..325545af47 100644 --- a/src/plugins/http/inc/httpSql.h +++ b/src/plugins/http/inc/httpSql.h @@ -35,4 +35,7 @@ void httpTrimTableName(char *name); int32_t httpShrinkTableName(HttpContext *pContext, int32_t pos, char *name); char * httpGetCmdsString(HttpContext *pContext, int32_t pos); +int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql); +void httpCheckFreeEscapedSql(char *oldSql, char *newSql); + #endif diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index 925c74e7cd..ed3a28567e 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -176,6 +176,20 @@ bool gcProcessQueryRequest(HttpContext* pContext) { return false; } +#define ESCAPE_ERROR_PROC(code, context, root) \ + do { \ + if (code != 0) { \ + if (code == 1) { \ + httpSendErrorResp(context, TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR); \ + } else { \ + httpSendErrorResp(context, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); \ + } \ + \ + cJSON_Delete(root); \ + return false; \ + } \ + } while (0) + for (int32_t i = 0; i < size; ++i) { cJSON* query = cJSON_GetArrayItem(root, i); if (query == NULL) continue; @@ -186,7 +200,14 @@ bool gcProcessQueryRequest(HttpContext* pContext) { continue; } - int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, refId->valuestring); + char *newStr = NULL; + int32_t retCode = 0; + + retCode = httpCheckAllocEscapeSql(refId->valuestring, &newStr); + ESCAPE_ERROR_PROC(retCode, pContext, root); + + int32_t refIdBuffer = httpAddToSqlCmdBuffer(pContext, newStr); + httpCheckFreeEscapedSql(refId->valuestring, newStr); if (refIdBuffer == -1) { httpWarn("context:%p, fd:%d, user:%s, refId buffer is full", pContext, pContext->fd, pContext->user); break; @@ -195,7 +216,11 @@ bool gcProcessQueryRequest(HttpContext* pContext) { cJSON* alias = cJSON_GetObjectItem(query, "alias"); int32_t aliasBuffer = -1; if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) { - aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring); + retCode = httpCheckAllocEscapeSql(alias->valuestring, &newStr); + ESCAPE_ERROR_PROC(retCode, pContext, root); + + aliasBuffer = httpAddToSqlCmdBuffer(pContext, newStr); + httpCheckFreeEscapedSql(alias->valuestring, newStr); if (aliasBuffer == -1) { httpWarn("context:%p, fd:%d, user:%s, alias buffer is full", pContext, pContext->fd, pContext->user); break; @@ -211,7 +236,11 @@ bool gcProcessQueryRequest(HttpContext* pContext) { continue; } - int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, sql->valuestring); + retCode = httpCheckAllocEscapeSql(sql->valuestring, &newStr); + ESCAPE_ERROR_PROC(retCode, pContext, root); + + int32_t sqlBuffer = httpAddToSqlCmdBuffer(pContext, newStr); + httpCheckFreeEscapedSql(sql->valuestring, newStr); if (sqlBuffer == -1) { httpWarn("context:%p, fd:%d, user:%s, sql buffer is full", pContext, pContext->fd, pContext->user); break; @@ -237,6 +266,8 @@ bool gcProcessQueryRequest(HttpContext* pContext) { } } +#undef ESCAPE_ERROR_PROC + pContext->reqType = HTTP_REQTYPE_MULTI_SQL; pContext->encodeMethod = &gcQueryMethod; pContext->multiCmds->pos = 0; diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 5a0480b694..20d1e159b3 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -423,3 +423,65 @@ void httpProcessRequest(HttpContext *pContext) { httpExecCmd(pContext); } } + +int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql) +{ + char *pos; + + if (oldSql == NULL || newSql == NULL) { + return 0; + } + + /* bad sql clause */ + pos = strstr(oldSql, "%%"); + if (pos) { + httpError("bad sql:%s", oldSql); + return 1; + } + + pos = strchr(oldSql, '%'); + if (pos == NULL) { + httpDebug("sql:%s", oldSql); + *newSql = oldSql; + return 0; + } + + *newSql = (char *) calloc(1, (strlen(oldSql) << 1) + 1); + if (newSql == NULL) { + httpError("failed to allocate for new sql, old sql:%s", oldSql); + return -1; + } + + char *src = oldSql; + char *dst = *newSql; + size_t sqlLen = strlen(src); + + while (1) { + memcpy(dst, src, pos - src + 1); + dst += pos - src + 1; + *dst++ = '%'; + + if (pos + 1 >= oldSql + sqlLen) { + break; + } + + src = ++pos; + pos = strchr(pos, '%'); + if (pos == NULL) { + memcpy(dst, src, sqlLen - strlen(src)); + break; + } + } + + return 0; +} + +void httpCheckFreeEscapedSql(char *oldSql, char *newSql) +{ + if (oldSql && newSql) { + if (oldSql != newSql) { + free(newSql); + } + } +} + diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index 69ac3e19c5..8aa156b84a 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -610,7 +610,22 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { // stable tag for detail for (int32_t i = 0; i < orderTagsLen; ++i) { cJSON *tag = orderedTags[i]; - stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tag->string); + + char *tagStr = NULL; + int32_t retCode = httpCheckAllocEscapeSql(tag->string, &tagStr); + if (retCode != 0) { + if (retCode == 1) { + httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_INVALID_JSON); + } else { + httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); + } + + return false; + } + + stable_cmd->tagNames[i] = table_cmd->tagNames[i] = httpAddToSqlCmdBuffer(pContext, tagStr); + + httpCheckFreeEscapedSql(tag->string, tagStr); if (tag->type == cJSON_String) stable_cmd->tagValues[i] = table_cmd->tagValues[i] = httpAddToSqlCmdBuffer(pContext, "'%s'", tag->valuestring); From 23545c2b29f73f71e1cca39f8ab8f819907aec1b Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 14 Jul 2021 14:28:30 +0800 Subject: [PATCH 13/31] fix bug --- src/query/inc/qUtil.h | 4 +++- src/query/src/qExecutor.c | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index e20a9268d3..c8741030c0 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -27,13 +27,15 @@ #define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \ do { \ assert(sizeof(_uid) == sizeof(uint64_t)); \ - *(void **)(_k) = (_buf); \ + *(void **)(_k) = (_buf); \ *(uint64_t *)((_k) + POINTER_BYTES) = (_uid); \ memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \ } while (0) #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) +#define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) + #define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0f8bcc0712..5214d52589 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -434,7 +434,7 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult pResultRowInfo->curPos = 0; } else { // check if current pResultRowInfo contains the existed pResultRow SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { pResultRowInfo->curPos = (int32_t) *index; if (pResultRowInfo->size > 1) { @@ -481,7 +481,7 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult int64_t index = pResultRowInfo->curPos; SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); - taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); + taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); } // too many time window in query From 83153f6a3e2f91bdb17b99258e4c0fd4e6e5fd59 Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 14 Jul 2021 14:39:20 +0800 Subject: [PATCH 14/31] fix bug --- src/query/src/qExecutor.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 5214d52589..9016737c40 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -475,10 +475,6 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult pResultRowInfo->curPos = pResultRowInfo->size; pResultRowInfo->pResult[pResultRowInfo->size++] = pResult; - if (pResultRowInfo->curPos > 0) { - assert(pResultRowInfo->pResult[pResultRowInfo->curPos]->win.skey > pResultRowInfo->pResult[pResultRowInfo->curPos-1]->win.skey); - } - int64_t index = pResultRowInfo->curPos; SET_RES_EXT_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, tid, pResultRowInfo); taosHashPut(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes), &index, POINTER_BYTES); From c0c7043d20b658808a5e4bc5afa6ac1b462aaf01 Mon Sep 17 00:00:00 2001 From: wpan Date: Wed, 14 Jul 2021 14:53:18 +0800 Subject: [PATCH 15/31] fix bug --- src/query/src/qExecutor.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9016737c40..fa2ddb05b8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -437,11 +437,6 @@ static SResultRow* doSetResultOutBufByKey(SQueryRuntimeEnv* pRuntimeEnv, SResult int64_t* index = taosHashGet(pRuntimeEnv->pResultRowListSet, pRuntimeEnv->keyBuf, GET_RES_EXT_WINDOW_KEY_LEN(bytes)); if (index != NULL) { pResultRowInfo->curPos = (int32_t) *index; - if (pResultRowInfo->size > 1) { - for(int32_t k = 0; k size - 1; ++k) { - assert(pResultRowInfo->pResult[k]->win.skey < pResultRowInfo->pResult[k+1]->win.skey); - } - } existed = true; } else { existed = false; From ff9341e2e3838f9c108857a45c5b8e9a16a95cbc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 14 Jul 2021 16:12:58 +0800 Subject: [PATCH 16/31] [td-225]code refactor. --- src/client/src/tscSql.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 1e93892876..60261368f4 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -948,8 +948,6 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); pSql->pTscObj = taos; pSql->signature = pSql; - - pSql->fp = NULL; // todo set the correct callback function pointer pSql->cmd.pTableMetaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); int32_t length = (int32_t)strlen(tableNameList); From a1f1315de382125aba443f47062c25b5945d166e Mon Sep 17 00:00:00 2001 From: weicz1221 <49259303+weicz1221@users.noreply.github.com> Date: Wed, 14 Jul 2021 16:13:27 +0800 Subject: [PATCH 17/31] fix sql regex (#6838) Co-authored-by: weicz --- .../jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java index e3179bd317..efe3303bd9 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/Utils.java @@ -110,7 +110,7 @@ public class Utils { return rawSql; // toLowerCase String preparedSql = rawSql.trim().toLowerCase(); - String[] clause = new String[]{"values\\s*\\(.*?\\)", "tags\\s*\\(.*?\\)", "where\\s*.*"}; + String[] clause = new String[]{"values\\s*\\([\\s\\S]*?\\)", "tags\\s*\\([\\s\\S]*?\\)", "where[\\s\\S]*"}; Map placeholderPositions = new HashMap<>(); RangeSet clauseRangeSet = TreeRangeSet.create(); findPlaceholderPosition(preparedSql, placeholderPositions); From cf76ed44fe396003a970f79c6c9ca882a85efe0f Mon Sep 17 00:00:00 2001 From: xywang Date: Wed, 14 Jul 2021 16:45:32 +0800 Subject: [PATCH 18/31] [TD-5169]: fixed a parsing bug --- src/plugins/http/src/httpSql.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 20d1e159b3..b2480dcad8 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -468,7 +468,7 @@ int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql) src = ++pos; pos = strchr(pos, '%'); if (pos == NULL) { - memcpy(dst, src, sqlLen - strlen(src)); + memcpy(dst, src, strlen(src)); break; } } From 80faa86c952dd5110862add4be9e4ecce1735def Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 14 Jul 2021 17:25:05 +0800 Subject: [PATCH 19/31] Feature/sangshuduo/td 5242 taosdemo support 4096 columns (#6856) * [TD-5136]: taosdemo simulate real senario. * update test case according to taosdemo change * adjust range of semi-random data. * make demo mode use different tag name and value. * [TD-5242]: taosdemo support max columns align with TSDB defines. --- src/kit/taosdemo/taosdemo.c | 107 ++++++++++++++++++++++-------------- 1 file changed, 65 insertions(+), 42 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index fa3d263678..d088d015d5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -75,7 +75,7 @@ enum TEST_MODE { #define MAX_RECORDS_PER_REQ 32766 -#define HEAD_BUFF_LEN 1024*24 // 16*1024 + (192+32)*2 + insert into .. +#define HEAD_BUFF_LEN TSDB_MAX_COLUMNS*24 // 16*MAX_COLUMNS + (192+32)*2 + insert into .. #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) @@ -84,26 +84,23 @@ enum TEST_MODE { #define MAX_PASSWORD_SIZE 64 #define MAX_HOSTNAME_SIZE 64 #define MAX_TB_NAME_SIZE 64 -#define MAX_DATA_SIZE (16*1024)+20 // max record len: 16*1024, timestamp string and ,('') need extra space -#define MAX_NUM_DATATYPE 10 +#define MAX_DATA_SIZE (16*TSDB_MAX_COLUMNS)+20 // max record len: 16*MAX_COLUMNS, timestamp string and ,('') need extra space #define OPT_ABORT 1 /* –abort */ #define STRING_LEN 60000 #define MAX_PREPARED_RAND 1000000 #define MAX_FILE_NAME_LEN 256 // max file name length on linux is 255. -#define MAX_SAMPLES_ONCE_FROM_FILE 10000 -#define MAX_NUM_DATATYPE 10 +#define MAX_SAMPLES_ONCE_FROM_FILE 10000 +#define MAX_NUM_COLUMNS (TSDB_MAX_COLUMNS - 1) // exclude first column timestamp -#define MAX_DB_COUNT 8 -#define MAX_SUPER_TABLE_COUNT 200 -#define MAX_COLUMN_COUNT 1024 -#define MAX_TAG_COUNT 128 +#define MAX_DB_COUNT 8 +#define MAX_SUPER_TABLE_COUNT 200 -#define MAX_QUERY_SQL_COUNT 100 -#define MAX_QUERY_SQL_LENGTH 1024 +#define MAX_QUERY_SQL_COUNT 100 +#define MAX_QUERY_SQL_LENGTH 1024 -#define MAX_DATABASE_COUNT 256 -#define INPUT_BUF_LEN 256 +#define MAX_DATABASE_COUNT 256 +#define INPUT_BUF_LEN 256 #define DEFAULT_TIMESTAMP_STEP 1 @@ -218,7 +215,7 @@ typedef struct SArguments_S { bool performance_print; char * output_file; bool async_mode; - char * datatype[MAX_NUM_DATATYPE + 1]; + char * datatype[MAX_NUM_COLUMNS + 1]; uint32_t len_of_binary; uint32_t num_of_CPR; uint32_t num_of_threads; @@ -274,9 +271,9 @@ typedef struct SSuperTable_S { char tagsFile[MAX_FILE_NAME_LEN]; uint32_t columnCount; - StrColumn columns[MAX_COLUMN_COUNT]; + StrColumn columns[TSDB_MAX_COLUMNS]; uint32_t tagCount; - StrColumn tags[MAX_TAG_COUNT]; + StrColumn tags[TSDB_MAX_TAGS]; char* childTblName; char* colsOfCreateChildTable; @@ -565,6 +562,8 @@ double randdouble[MAX_PREPARED_RAND]; char *aggreFunc[] = {"*", "count(*)", "avg(col0)", "sum(col0)", "max(col0)", "min(col0)", "first(col0)", "last(col0)"}; +#define DEFAULT_DATATYPE_NUM 3 + SArguments g_args = { NULL, // metaFile 0, // test_mode @@ -595,7 +594,7 @@ SArguments g_args = { { "FLOAT", // datatype "INT", // datatype - "FLOAT", // datatype + "FLOAT", // datatype. DEFAULT_DATATYPE_NUM is 3 }, 16, // len_of_binary 4, // num_of_CPR @@ -725,9 +724,13 @@ static void printHelp() { "The data_type of columns, default: FLOAT, INT, FLOAT."); printf("%s%s%s%s\n", indent, "-w", indent, "The length of data_type 'BINARY' or 'NCHAR'. Default is 16"); - printf("%s%s%s%s%d\n", indent, "-l", indent, - "The number of columns per record. Default is 3. Max values is ", - MAX_NUM_DATATYPE); + printf("%s%s%s%s%d%s%d\n", indent, "-l", indent, + "The number of columns per record. Default is ", + DEFAULT_DATATYPE_NUM, + ". Max values is ", + MAX_NUM_COLUMNS); + printf("%s%s%s%s\n", indent, indent, indent, + "All of the new column(s) type is INT. If use -b to specify column type, -l will be ignored."); printf("%s%s%s%s\n", indent, "-T", indent, "The number of threads. Default is 10."); printf("%s%s%s%s\n", indent, "-i", indent, @@ -931,13 +934,16 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->num_of_CPR = atoi(argv[++i]); - if (arguments->num_of_CPR > MAX_NUM_DATATYPE) { - printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_DATATYPE); + if (arguments->num_of_CPR > MAX_NUM_COLUMNS) { + printf("WARNING: max acceptible columns count is %d\n", MAX_NUM_COLUMNS); prompt(); - arguments->num_of_CPR = MAX_NUM_DATATYPE; + arguments->num_of_CPR = MAX_NUM_COLUMNS; } - for (int col = arguments->num_of_CPR; col < MAX_NUM_DATATYPE; col++) { + for (int col = DEFAULT_DATATYPE_NUM; col < arguments->num_of_CPR; col ++) { + arguments->datatype[col] = "INT"; + } + for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) { arguments->datatype[col] = NULL; } @@ -990,7 +996,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->datatype[index++] = token; token = strsep(&running, ","); - if (index >= MAX_NUM_DATATYPE) break; + if (index >= MAX_NUM_COLUMNS) break; } arguments->datatype[index] = NULL; } @@ -1086,7 +1092,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } int columnCount; - for (columnCount = 0; columnCount < MAX_NUM_DATATYPE; columnCount ++) { + for (columnCount = 0; columnCount < MAX_NUM_COLUMNS; columnCount ++) { if (g_args.datatype[columnCount] == NULL) { break; } @@ -1111,7 +1117,7 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { arguments->use_metric ? "true" : "false"); if (*(arguments->datatype)) { printf("# Specified data type: "); - for (int i = 0; i < MAX_NUM_DATATYPE; i++) + for (int i = 0; i < MAX_NUM_COLUMNS; i++) if (arguments->datatype[i]) printf("%s,", arguments->datatype[i]); else @@ -2389,8 +2395,15 @@ static char* generateTagVaulesForStb(SSuperTable* stbInfo, int32_t tableSeq) { tmfree(buf); } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "int", strlen("int"))) { - dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, + if ((g_args.demo_mode) && (i == 0)) { + dataLen += snprintf(dataBuf + dataLen, + TSDB_MAX_SQL_LEN - dataLen, + "%d, ", tableSeq % 10); + } else { + dataLen += snprintf(dataBuf + dataLen, + TSDB_MAX_SQL_LEN - dataLen, "%d, ", tableSeq); + } } else if (0 == strncasecmp(stbInfo->tags[i].dataType, "bigint", strlen("bigint"))) { dataLen += snprintf(dataBuf + dataLen, TSDB_MAX_SQL_LEN - dataLen, @@ -2787,16 +2800,26 @@ static int createSuperTable( char* dataType = superTbl->tags[tagIndex].dataType; if (strcasecmp(dataType, "BINARY") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, - "BINARY", superTbl->tags[tagIndex].dataLen); + if ((g_args.demo_mode) && (tagIndex == 1)) { + len += snprintf(tags + len, STRING_LEN - len, + "loction BINARY(%d), ", + superTbl->tags[tagIndex].dataLen); + } else { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", + tagIndex, "BINARY", superTbl->tags[tagIndex].dataLen); + } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "NCHAR") == 0) { len += snprintf(tags + len, STRING_LEN - len, "t%d %s(%d), ", tagIndex, "NCHAR", superTbl->tags[tagIndex].dataLen); lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 3; } else if (strcasecmp(dataType, "INT") == 0) { - len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, + if ((g_args.demo_mode) && (tagIndex == 0)) { + len += snprintf(tags + len, STRING_LEN - len, "groupId INT, "); + } else { + len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, "INT"); + } lenOfTagOfOneRow += superTbl->tags[tagIndex].dataLen + 11; } else if (strcasecmp(dataType, "BIGINT") == 0) { len += snprintf(tags + len, STRING_LEN - len, "t%d %s, ", tagIndex, @@ -3352,9 +3375,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } int columnSize = cJSON_GetArraySize(columns); - if ((columnSize + 1/* ts */) > MAX_COLUMN_COUNT) { + if ((columnSize + 1/* ts */) > TSDB_MAX_COLUMNS) { errorPrint("%s() LN%d, failed to read json, column size overflow, max column size is %d\n", - __func__, __LINE__, MAX_COLUMN_COUNT); + __func__, __LINE__, TSDB_MAX_COLUMNS); goto PARSE_OVER; } @@ -3410,9 +3433,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } } - if ((index + 1 /* ts */) > MAX_COLUMN_COUNT) { + if ((index + 1 /* ts */) > MAX_NUM_COLUMNS) { errorPrint("%s() LN%d, failed to read json, column size overflow, allowed max column size is %d\n", - __func__, __LINE__, MAX_COLUMN_COUNT); + __func__, __LINE__, MAX_NUM_COLUMNS); goto PARSE_OVER; } @@ -3429,9 +3452,9 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } int tagSize = cJSON_GetArraySize(tags); - if (tagSize > MAX_TAG_COUNT) { + if (tagSize > TSDB_MAX_TAGS) { errorPrint("%s() LN%d, failed to read json, tags size overflow, max tag size is %d\n", - __func__, __LINE__, MAX_TAG_COUNT); + __func__, __LINE__, TSDB_MAX_TAGS); goto PARSE_OVER; } @@ -3481,17 +3504,17 @@ static bool getColumnAndTagTypeFromInsertJsonFile( } } - if (index > MAX_TAG_COUNT) { + if (index > TSDB_MAX_TAGS) { errorPrint("%s() LN%d, failed to read json, tags size overflow, allowed max tag count is %d\n", - __func__, __LINE__, MAX_TAG_COUNT); + __func__, __LINE__, TSDB_MAX_TAGS); goto PARSE_OVER; } superTbls->tagCount = index; - if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > MAX_COLUMN_COUNT) { + if ((superTbls->columnCount + superTbls->tagCount + 1 /* ts */) > TSDB_MAX_COLUMNS) { errorPrint("%s() LN%d, columns + tags is more than allowed max columns count: %d\n", - __func__, __LINE__, MAX_COLUMN_COUNT); + __func__, __LINE__, TSDB_MAX_COLUMNS); goto PARSE_OVER; } ret = true; @@ -7919,7 +7942,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].maxSqlLen = g_args.max_sql_len; g_Dbs.db[0].superTbls[0].columnCount = 0; - for (int i = 0; i < MAX_NUM_DATATYPE; i++) { + for (int i = 0; i < MAX_NUM_COLUMNS; i++) { if (data_type[i] == NULL) { break; } From cc13cd44c37d65e32f372be04299164ecd4278f4 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 14 Jul 2021 17:25:49 +0800 Subject: [PATCH 20/31] [TD-5260]add case for a potential crash --- tests/script/general/http/grafana_bug.sim | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/script/general/http/grafana_bug.sim b/tests/script/general/http/grafana_bug.sim index 0816e88f3f..ed184e17c6 100644 --- a/tests/script/general/http/grafana_bug.sim +++ b/tests/script/general/http/grafana_bug.sim @@ -247,4 +247,25 @@ if $system_content != @[{"refId":"A","target":"{val1:nil, val2:nil}","datapoints return -1 endi +sql create table tt (ts timestamp ,i int) tags(j binary(20),k binary(20)); +sql insert into t1 using tt tags('jnetworki','t1') values('2020-01-01 00:00:00.000',1)('2020-01-01 00:01:00.000',2)('2020-01-01 00:02:00.000',3)('2020-01-01 00:03:00.000',4)('2020-01-01 00:04:00.000',5); + +system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select max(i) from db.tt where j like \u0027%network%\u0027 and ts >= \u00272020-01-01 00:00:00.000\u0027 and ts < \u00272020-01-01 00:05:00.000\u0027 interval(5m) group by k "} ]' 127.0.0.1:7111/grafana/query +print step1-> $system_content +if $system_content != @[{"refId":"A","target":"{k:t1}","datapoints":[[5,1577808000000]]}]@ then + return -1 +endi + +system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select max(i) from db.tt where j like \u0027jnetwo%\u0027 and ts >= \u00272020-01-01 00:00:00.000\u0027 and ts < \u00272020-01-01 00:05:00.000\u0027 interval(5m) group by k "} ]' 127.0.0.1:7111/grafana/query +print step1-> $system_content +if $system_content != @[{"refId":"A","target":"{k:t1}","datapoints":[[5,1577808000000]]}]@ then + return -1 +endi + +system_content curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d '[ {"refId":"A","alias":"","sql":"select max(i) from db.tt where j like \u0027%networki\u0027 and ts >= \u00272020-01-01 00:00:00.000\u0027 and ts < \u00272020-01-01 00:05:00.000\u0027 interval(5m) group by k "} ]' 127.0.0.1:7111/grafana/query +print step1-> $system_content +if $system_content != @[{"refId":"A","target":"{k:t1}","datapoints":[[5,1577808000000]]}]@ then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 09b58e7e8e3d1156529ed7f0fdc5884ff7424887 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 14 Jul 2021 18:32:43 +0800 Subject: [PATCH 21/31] [TD-5259]: JDBC vulnerable packages. (#6858) --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 4 ++-- src/connector/jdbc/pom.xml | 6 +++--- .../comparisonTest/cassandra/cassandratest/pom.xml | 2 +- tests/comparisonTest/opentsdb/opentsdbtest/pom.xml | 4 ++-- tests/examples/JDBC/SpringJdbcTemplate/pom.xml | 2 +- tests/examples/JDBC/connectionPools/pom.xml | 13 +++++++++---- .../com/taosdata/example/ConnectionPoolDemo.java | 5 +++-- .../com/taosdata/example/common/InsertTask.java | 5 +++-- tests/examples/JDBC/mybatisplus-demo/pom.xml | 2 +- tests/examples/JDBC/taosdemo/pom.xml | 12 ++++++------ .../com/taosdata/taosdemo/TaosDemoApplication.java | 5 +++-- .../taosdata/taosdemo/dao/DatabaseMapperImpl.java | 5 +++-- .../taosdata/taosdemo/dao/SubTableMapperImpl.java | 5 +++-- .../taosdata/taosdemo/dao/SuperTableMapperImpl.java | 5 +++-- .../com/taosdata/taosdemo/dao/TableMapperImpl.java | 5 +++-- .../taosdata/taosdemo/service/SubTableService.java | 5 +++-- 17 files changed, 50 insertions(+), 37 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 30aa801122..fced638966 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.32-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-*-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 81af0ec144..7791317969 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,8 +8,8 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.32-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-*-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) -ENDIF () \ No newline at end of file +ENDIF () diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 61d7fb85ef..a1aa41b351 100644 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.32 + 2.0.33 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -40,7 +40,7 @@ junit junit - 4.13 + 4.13.1 test @@ -57,7 +57,7 @@ com.google.guava guava - 29.0-jre + 30.0-jre diff --git a/tests/comparisonTest/cassandra/cassandratest/pom.xml b/tests/comparisonTest/cassandra/cassandratest/pom.xml index 8eeb5c3aa0..00630d93d1 100644 --- a/tests/comparisonTest/cassandra/cassandratest/pom.xml +++ b/tests/comparisonTest/cassandra/cassandratest/pom.xml @@ -75,7 +75,7 @@ junit junit - 4.11 + 4.13.1 test diff --git a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml index e0ada8b763..b55a136c73 100644 --- a/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml +++ b/tests/comparisonTest/opentsdb/opentsdbtest/pom.xml @@ -87,14 +87,14 @@ junit junit - 4.11 + 4.13.1 test com.google.guava guava - 29.0-jre + 30.0-jre diff --git a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml index 64a91b951b..eac3dec0a9 100644 --- a/tests/examples/JDBC/SpringJdbcTemplate/pom.xml +++ b/tests/examples/JDBC/SpringJdbcTemplate/pom.xml @@ -40,7 +40,7 @@ junit junit - 4.13 + 4.13.1 test diff --git a/tests/examples/JDBC/connectionPools/pom.xml b/tests/examples/JDBC/connectionPools/pom.xml index 045e9d336c..34518900ed 100644 --- a/tests/examples/JDBC/connectionPools/pom.xml +++ b/tests/examples/JDBC/connectionPools/pom.xml @@ -4,6 +4,11 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + + 1.8 + 1.8 + + com.taosdata.demo connectionPools 1.0-SNAPSHOT @@ -46,9 +51,9 @@ - log4j - log4j - 1.2.17 + org.apache.logging.log4j + log4j-core + 2.14.1 @@ -108,4 +113,4 @@ - \ No newline at end of file + diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java index bd57d138b2..96ad65aa4f 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/ConnectionPoolDemo.java @@ -5,7 +5,8 @@ import com.taosdata.example.pool.C3p0Builder; import com.taosdata.example.pool.DbcpBuilder; import com.taosdata.example.pool.DruidPoolBuilder; import com.taosdata.example.pool.HikariCpBuilder; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import javax.sql.DataSource; import java.sql.Connection; @@ -17,7 +18,7 @@ import java.util.concurrent.TimeUnit; public class ConnectionPoolDemo { - private static Logger logger = Logger.getLogger(DruidPoolBuilder.class); + private static Logger logger = LogManager.getLogger(DruidPoolBuilder.class); private static final String dbName = "pool_test"; private static String poolType = "hikari"; diff --git a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java index da7c9a22b5..f8f1555c08 100644 --- a/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java +++ b/tests/examples/JDBC/connectionPools/src/main/java/com/taosdata/example/common/InsertTask.java @@ -1,6 +1,7 @@ package com.taosdata.example.common; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import javax.sql.DataSource; import java.sql.Connection; @@ -10,7 +11,7 @@ import java.util.Random; public class InsertTask implements Runnable { private final Random random = new Random(System.currentTimeMillis()); - private static final Logger logger = Logger.getLogger(InsertTask.class); + private static final Logger logger = LogManager.getLogger(InsertTask.class); private final DataSource ds; private final String dbName; diff --git a/tests/examples/JDBC/mybatisplus-demo/pom.xml b/tests/examples/JDBC/mybatisplus-demo/pom.xml index a83d0a00e6..ad6a63e800 100644 --- a/tests/examples/JDBC/mybatisplus-demo/pom.xml +++ b/tests/examples/JDBC/mybatisplus-demo/pom.xml @@ -68,7 +68,7 @@ junit junit - 4.12 + 4.13.1 test diff --git a/tests/examples/JDBC/taosdemo/pom.xml b/tests/examples/JDBC/taosdemo/pom.xml index 22c2f3b63e..91b976c2ae 100644 --- a/tests/examples/JDBC/taosdemo/pom.xml +++ b/tests/examples/JDBC/taosdemo/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.taosdata taosdemo - 2.0 + 2.0.1 taosdemo jar Demo project for TDengine @@ -81,20 +81,20 @@ mysql mysql-connector-java - 5.1.47 + 8.0.16 test - log4j - log4j - 1.2.17 + org.apache.logging.log4j + log4j-core + 2.14.1 junit junit - 4.12 + 4.13.1 test diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java index c361df82b0..d4f5ff2688 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/TaosDemoApplication.java @@ -8,7 +8,8 @@ import com.taosdata.taosdemo.service.SqlExecuteTask; import com.taosdata.taosdemo.service.SubTableService; import com.taosdata.taosdemo.service.SuperTableService; import com.taosdata.taosdemo.service.data.SuperTableMetaGenerator; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import javax.sql.DataSource; import java.io.IOException; @@ -20,7 +21,7 @@ import java.util.Map; public class TaosDemoApplication { - private static final Logger logger = Logger.getLogger(TaosDemoApplication.class); + private static final Logger logger = LogManager.getLogger(TaosDemoApplication.class); public static void main(String[] args) throws IOException { // 读配置参数 diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java index 421a2dea1f..9340fc3fdd 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/DatabaseMapperImpl.java @@ -1,14 +1,15 @@ package com.taosdata.taosdemo.dao; import com.taosdata.taosdemo.utils.SqlSpeller; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.jdbc.core.JdbcTemplate; import javax.sql.DataSource; import java.util.Map; public class DatabaseMapperImpl implements DatabaseMapper { - private static final Logger logger = Logger.getLogger(DatabaseMapperImpl.class); + private static final Logger logger = LogManager.getLogger(DatabaseMapperImpl.class); private final JdbcTemplate jdbcTemplate; diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java index 90b0990a2b..db0d43ff05 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SubTableMapperImpl.java @@ -3,7 +3,8 @@ package com.taosdata.taosdemo.dao; import com.taosdata.taosdemo.domain.SubTableMeta; import com.taosdata.taosdemo.domain.SubTableValue; import com.taosdata.taosdemo.utils.SqlSpeller; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.jdbc.core.JdbcTemplate; import javax.sql.DataSource; @@ -11,7 +12,7 @@ import java.util.List; public class SubTableMapperImpl implements SubTableMapper { - private static final Logger logger = Logger.getLogger(SubTableMapperImpl.class); + private static final Logger logger = LogManager.getLogger(SubTableMapperImpl.class); private final JdbcTemplate jdbcTemplate; public SubTableMapperImpl(DataSource dataSource) { diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java index efa9a1f39e..658a403a0c 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/SuperTableMapperImpl.java @@ -2,13 +2,14 @@ package com.taosdata.taosdemo.dao; import com.taosdata.taosdemo.domain.SuperTableMeta; import com.taosdata.taosdemo.utils.SqlSpeller; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.jdbc.core.JdbcTemplate; import javax.sql.DataSource; public class SuperTableMapperImpl implements SuperTableMapper { - private static final Logger logger = Logger.getLogger(SuperTableMapperImpl.class); + private static final Logger logger = LogManager.getLogger(SuperTableMapperImpl.class); private JdbcTemplate jdbcTemplate; public SuperTableMapperImpl(DataSource dataSource) { diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java index b049fbe197..16bc094848 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/dao/TableMapperImpl.java @@ -3,13 +3,14 @@ package com.taosdata.taosdemo.dao; import com.taosdata.taosdemo.domain.TableMeta; import com.taosdata.taosdemo.domain.TableValue; import com.taosdata.taosdemo.utils.SqlSpeller; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import org.springframework.jdbc.core.JdbcTemplate; import java.util.List; public class TableMapperImpl implements TableMapper { - private static final Logger logger = Logger.getLogger(TableMapperImpl.class); + private static final Logger logger = LogManager.getLogger(TableMapperImpl.class); private JdbcTemplate template; @Override diff --git a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java index cea98a1c5d..b0a79dea78 100644 --- a/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java +++ b/tests/examples/JDBC/taosdemo/src/main/java/com/taosdata/taosdemo/service/SubTableService.java @@ -8,7 +8,8 @@ import com.taosdata.taosdemo.domain.SubTableValue; import com.taosdata.taosdemo.domain.SuperTableMeta; import com.taosdata.taosdemo.service.data.SubTableMetaGenerator; import com.taosdata.taosdemo.service.data.SubTableValueGenerator; -import org.apache.log4j.Logger; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.LogManager; import javax.sql.DataSource; import java.util.ArrayList; @@ -20,7 +21,7 @@ import java.util.stream.IntStream; public class SubTableService extends AbstractService { private SubTableMapper mapper; - private static final Logger logger = Logger.getLogger(SubTableService.class); + private static final Logger logger = LogManager.getLogger(SubTableService.class); public SubTableService(DataSource datasource) { this.mapper = new SubTableMapperImpl(datasource); From 954393e7706a37ace31e2fb55670ba23e454b672 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 14 Jul 2021 22:25:03 +0800 Subject: [PATCH 22/31] [td-5271]: fix a bug of max query in outer query. --- src/query/src/qAggMain.c | 13 ++++++------ tests/script/general/parser/nestquery.sim | 26 +++++++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 80285f498b..98ffebe616 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -58,7 +58,7 @@ for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ - __ctx->tag.i64 = (ts); \ + __ctx->tag.i64 = (ts); \ __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ } \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ @@ -520,7 +520,7 @@ int32_t noDataRequired(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId) { if ((ctx)->hasNull && isNull((char *)&(list)[i], tsdbType)) { \ continue; \ } \ - TSKEY key = GET_TS_DATA(ctx, i); \ + TSKEY key = (ctx)->ptsList != NULL? GET_TS_DATA(ctx, i):0; \ UPDATE_DATA(ctx, val, (list)[i], num, sign, key); \ } @@ -1463,10 +1463,11 @@ static void first_function(SQLFunctionCtx *pCtx) { } memcpy(pCtx->pOutput, data, pCtx->inputBytes); - - TSKEY k = GET_TS_DATA(pCtx, i); - DO_UPDATE_TAG_COLUMNS(pCtx, k); - + if (pCtx->ptsList != NULL) { + TSKEY k = GET_TS_DATA(pCtx, i); + DO_UPDATE_TAG_COLUMNS(pCtx, k); + } + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index 6035992d30..3c1ba03369 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -334,10 +334,6 @@ sql select top(x, 20) from (select c1 x from nest_tb0); sql select bottom(x, 20) from (select c1 x from nest_tb0) -print ===================> complex query - - - print ===================> group by + having @@ -464,6 +460,28 @@ if $data01 != 0.000083333 then return -1 endi +print ======================>TD-5271 +sql select min(val),max(val),first(val),last(val),count(val),sum(val),avg(val) from (select count(*) val from nest_mt0 group by tbname) +if $rows != 1 then + return -1 +endi + +if $data00 != 10000 then + return -1 +endi + +if $data01 != 10000 then + return -1 +endi + +if $data04 != 10 then + return -1 +endi + +if $data05 != 100000 then + return -1 +endi + print =================>us database interval query, TD-5039 sql create database test precision 'us'; sql use test; From c62ebe7dc270ef59f700cb5693e419137138d4d4 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 15 Jul 2021 09:53:11 +0800 Subject: [PATCH 23/31] Feature/sangshuduo/td 5136 taosdemo rework (#6852) * [TD-5136]: taosdemo simulate real senario. * update test case according to taosdemo change * adjust range of semi-random data. * make demo mode use different tag name and value. * change malloc to calloc for pid allocation. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index d088d015d5..9c547ff755 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -946,7 +946,6 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { for (int col = arguments->num_of_CPR; col < MAX_NUM_COLUMNS; col++) { arguments->datatype[col] = NULL; } - } else if (strcmp(argv[i], "-b") == 0) { arguments->demo_mode = false; if (argc == i+1) { @@ -3104,7 +3103,7 @@ static int startMultiThreadCreateChildTable( char* cols, int threads, uint64_t tableFrom, int64_t ntables, char* db_name, SSuperTable* superTblInfo) { - pthread_t *pids = malloc(threads * sizeof(pthread_t)); + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { @@ -6602,7 +6601,7 @@ static void startMultiThreadInsertData(int threads, char* db_name, } } - pthread_t *pids = malloc(threads * sizeof(pthread_t)); + pthread_t *pids = calloc(1, threads * sizeof(pthread_t)); assert(pids != NULL); threadInfo *infos = calloc(1, threads * sizeof(threadInfo)); @@ -7261,8 +7260,8 @@ static int queryTestProcess() { if ((nSqlCount > 0) && (nConcurrent > 0)) { - pids = malloc(nConcurrent * nSqlCount * sizeof(pthread_t)); - infos = malloc(nConcurrent * nSqlCount * sizeof(threadInfo)); + pids = calloc(1, nConcurrent * nSqlCount * sizeof(pthread_t)); + infos = calloc(1, nConcurrent * nSqlCount * sizeof(threadInfo)); if ((NULL == pids) || (NULL == infos)) { taos_close(taos); @@ -7307,8 +7306,8 @@ static int queryTestProcess() { //==== create sub threads for query from all sub table of the super table if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfSub = malloc(g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); + pidsOfSub = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); + infosOfSub = calloc(1, g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); if ((NULL == pidsOfSub) || (NULL == infosOfSub)) { free(infos); @@ -7741,11 +7740,13 @@ static int subscribeTestProcess() { exit(-1); } - pids = malloc( + pids = calloc( + 1, g_queryInfo.specifiedQueryInfo.sqlCount * g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc( + infos = calloc( + 1, g_queryInfo.specifiedQueryInfo.sqlCount * g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); @@ -7774,11 +7775,13 @@ static int subscribeTestProcess() { } else { if ((g_queryInfo.superQueryInfo.sqlCount > 0) && (g_queryInfo.superQueryInfo.threadCnt > 0)) { - pidsOfStable = malloc( + pidsOfStable = calloc( + 1, g_queryInfo.superQueryInfo.sqlCount * g_queryInfo.superQueryInfo.threadCnt * sizeof(pthread_t)); - infosOfStable = malloc( + infosOfStable = calloc( + 1, g_queryInfo.superQueryInfo.sqlCount * g_queryInfo.superQueryInfo.threadCnt * sizeof(threadInfo)); @@ -8095,7 +8098,7 @@ static void queryResult() { // query data pthread_t read_id; - threadInfo *pThreadInfo = malloc(sizeof(threadInfo)); + threadInfo *pThreadInfo = calloc(1, sizeof(threadInfo)); assert(pThreadInfo); pThreadInfo->start_time = 1500000000000; // 2017-07-14 10:40:00.000 pThreadInfo->start_table_from = 0; From d2941c06088c20e150f9faf459cf000e054ebf8e Mon Sep 17 00:00:00 2001 From: xywang Date: Thu, 15 Jul 2021 13:17:10 +0800 Subject: [PATCH 24/31] [TD-5169]: simplified function implementation --- src/inc/taoserror.h | 2 ++ src/plugins/http/src/httpGcHandle.c | 8 ++------ src/plugins/http/src/httpSql.c | 10 +++++----- src/plugins/http/src/httpTgHandle.c | 8 ++------ src/util/src/terror.c | 2 ++ 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 1e996be889..b18aa2c2d9 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -395,6 +395,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_HTTP_OP_VALUE_NULL TAOS_DEF_ERROR_CODE(0, 0x11A5) //"value not find") #define TSDB_CODE_HTTP_OP_VALUE_TYPE TAOS_DEF_ERROR_CODE(0, 0x11A6) //"value type should be boolean number or string") +#define TSDB_CODE_HTTP_REQUEST_JSON_ERROR TAOS_DEF_ERROR_CODE(0, 0x1F00) //"http request json error") + // odbc #define TSDB_CODE_ODBC_OOM TAOS_DEF_ERROR_CODE(0, 0x2100) //"out of memory") #define TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM TAOS_DEF_ERROR_CODE(0, 0x2101) //"convertion not a valid literal input") diff --git a/src/plugins/http/src/httpGcHandle.c b/src/plugins/http/src/httpGcHandle.c index ed3a28567e..883afcc4ec 100644 --- a/src/plugins/http/src/httpGcHandle.c +++ b/src/plugins/http/src/httpGcHandle.c @@ -178,12 +178,8 @@ bool gcProcessQueryRequest(HttpContext* pContext) { #define ESCAPE_ERROR_PROC(code, context, root) \ do { \ - if (code != 0) { \ - if (code == 1) { \ - httpSendErrorResp(context, TSDB_CODE_HTTP_GC_REQ_PARSE_ERROR); \ - } else { \ - httpSendErrorResp(context, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); \ - } \ + if (code != TSDB_CODE_SUCCESS) { \ + httpSendErrorResp(context, code); \ \ cJSON_Delete(root); \ return false; \ diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index b2480dcad8..c2e723732a 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -429,27 +429,27 @@ int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql) char *pos; if (oldSql == NULL || newSql == NULL) { - return 0; + return TSDB_CODE_SUCCESS; } /* bad sql clause */ pos = strstr(oldSql, "%%"); if (pos) { httpError("bad sql:%s", oldSql); - return 1; + return TSDB_CODE_HTTP_REQUEST_JSON_ERROR; } pos = strchr(oldSql, '%'); if (pos == NULL) { httpDebug("sql:%s", oldSql); *newSql = oldSql; - return 0; + return TSDB_CODE_SUCCESS; } *newSql = (char *) calloc(1, (strlen(oldSql) << 1) + 1); if (newSql == NULL) { httpError("failed to allocate for new sql, old sql:%s", oldSql); - return -1; + return TSDB_CODE_HTTP_NO_ENOUGH_MEMORY; } char *src = oldSql; @@ -473,7 +473,7 @@ int32_t httpCheckAllocEscapeSql(char *oldSql, char **newSql) } } - return 0; + return TSDB_CODE_SUCCESS; } void httpCheckFreeEscapedSql(char *oldSql, char *newSql) diff --git a/src/plugins/http/src/httpTgHandle.c b/src/plugins/http/src/httpTgHandle.c index 8aa156b84a..32516b9fd1 100644 --- a/src/plugins/http/src/httpTgHandle.c +++ b/src/plugins/http/src/httpTgHandle.c @@ -613,12 +613,8 @@ bool tgProcessSingleMetric(HttpContext *pContext, cJSON *metric, char *db) { char *tagStr = NULL; int32_t retCode = httpCheckAllocEscapeSql(tag->string, &tagStr); - if (retCode != 0) { - if (retCode == 1) { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_TG_INVALID_JSON); - } else { - httpSendErrorResp(pContext, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY); - } + if (retCode != TSDB_CODE_SUCCESS) { + httpSendErrorResp(pContext, retCode); return false; } diff --git a/src/util/src/terror.c b/src/util/src/terror.c index 27a08d8e9e..6cb508ebae 100644 --- a/src/util/src/terror.c +++ b/src/util/src/terror.c @@ -403,6 +403,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, "tag value can not mor TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, "value not find") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, "value type should be boolean, number or string") +TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_REQUEST_JSON_ERROR, "http request json error") + // odbc TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, "out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, "convertion not a valid literal input") From b68bb2a84a9444166abca2e1d06424a7c5a78396 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 15 Jul 2021 15:28:55 +0800 Subject: [PATCH 25/31] [td-5037]: fix the last query performance worse then previous problem. --- src/query/inc/qExecutor.h | 2 +- src/query/src/qExecutor.c | 40 +++++++++++++++++++++------------------ src/query/src/queryMain.c | 2 +- src/tsdb/src/tsdbRead.c | 16 +++++++++++++--- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 9348606d0c..c4276bfe37 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -589,7 +589,7 @@ int32_t createIndirectQueryFuncExprFromMsg(SQueryTableMsg *pQueryMsg, int32_t nu SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableMsg *pQueryMsg, SColIndex *pColIndex, int32_t *code); SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SGroupbyExpr *pGroupbyExpr, SExprInfo *pExprs, - SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t *qId); + SExprInfo *pSecExprs, STableGroupInfo *pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, char* sql, uint64_t qId); int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, SQueryParam* param, char* start, int32_t prevResultLen, void* merger); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fa2ddb05b8..dbcea4e90c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2268,10 +2268,11 @@ static int32_t updateBlockLoadStatus(SQueryAttr *pQuery, int32_t status) { return status; } -static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) { - SQueryAttr* pQueryAttr = &pQInfo->query; - size_t t = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); - for(int32_t i = 0; i < t; ++i) { +static void doUpdateLastKey(SQueryAttr* pQueryAttr) { + STimeWindow* win = &pQueryAttr->window; + + size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); + for(int32_t i = 0; i < num; ++i) { SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); size_t len = taosArrayGetSize(p1); @@ -2286,7 +2287,7 @@ static void doExchangeTimeWindow(SQInfo* pQInfo, STimeWindow* win) { } } -static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { +static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool stableQuery) { SQueryAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // in case of point-interpolation query, use asc order scan @@ -2303,6 +2304,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } + pQueryAttr->needReverseScan = false; return; } @@ -2312,7 +2314,8 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); } - doExchangeTimeWindow(pQInfo, &pQueryAttr->window); + pQueryAttr->needReverseScan = false; + doUpdateLastKey(pQueryAttr); return; } @@ -2333,7 +2336,7 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doExchangeTimeWindow(pQInfo, &pQueryAttr->window); + doUpdateLastKey(pQueryAttr); } pQueryAttr->order.order = TSDB_ORDER_ASC; @@ -2343,12 +2346,13 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doExchangeTimeWindow(pQInfo, &pQueryAttr->window); + doUpdateLastKey(pQueryAttr); } pQueryAttr->order.order = TSDB_ORDER_DESC; } + pQueryAttr->needReverseScan = false; } else { // interval query if (stableQuery) { if (onlyFirstQuery(pQueryAttr)) { @@ -2357,20 +2361,22 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doExchangeTimeWindow(pQInfo, &pQueryAttr->window); + doUpdateLastKey(pQueryAttr); } pQueryAttr->order.order = TSDB_ORDER_ASC; + pQueryAttr->needReverseScan = false; } else if (onlyLastQuery(pQueryAttr)) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) { qDebug(msg, pQInfo, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); SWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey, TSKEY); - doExchangeTimeWindow(pQInfo, &pQueryAttr->window); + doUpdateLastKey(pQueryAttr); } pQueryAttr->order.order = TSDB_ORDER_DESC; + pQueryAttr->needReverseScan = false; } } } @@ -2388,9 +2394,6 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i while(((*rowsize) * MIN_ROWS_PER_PAGE) > (*ps) - overhead) { *ps = ((*ps) << 1u); } - -// pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); -// assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -4382,7 +4385,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr break; } case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), 1); + pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); break; } case OP_TableScan: { @@ -4420,8 +4423,10 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr if (pQInfo->summary.queryProfEvents == NULL) { qDebug("QInfo:0x%"PRIx64" failed to allocate query prof events array", pQInfo->qId); } + pQInfo->summary.operatorProfResults = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); + if (pQInfo->summary.operatorProfResults == NULL) { qDebug("QInfo:0x%"PRIx64" failed to allocate operator prof results hash", pQInfo->qId); } @@ -4814,7 +4819,6 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime pInfo->reverseTimes = reverseTime; pInfo->current = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; -// pInfo->prevGroupId = -1; SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); pOptr->name = "DataBlocksOptimizedScanOperator"; @@ -7366,7 +7370,7 @@ FORCE_INLINE bool checkQIdEqual(void *qHandle, uint64_t qId) { SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, SExprInfo* pExprs, SExprInfo* pSecExprs, STableGroupInfo* pTableGroupInfo, SColumnInfo* pTagCols, int32_t vgId, - char* sql, uint64_t *qId) { + char* sql, uint64_t qId) { int16_t numOfCols = pQueryMsg->numOfCols; int16_t numOfOutput = pQueryMsg->numOfOutput; @@ -7375,7 +7379,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S goto _cleanup_qinfo; } - pQInfo->qId = *qId; + pQInfo->qId = qId; // to make sure third party won't overwrite this structure pQInfo->signature = pQInfo; @@ -7485,7 +7489,7 @@ SQInfo* createQInfoImpl(SQueryTableMsg* pQueryMsg, SGroupbyExpr* pGroupbyExpr, S tsem_init(&pQInfo->ready, 0, 0); pQueryAttr->window = pQueryMsg->window; - changeExecuteScanOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); + updateDataCheckOrder(pQInfo, pQueryMsg, pQueryAttr->stableQuery); SQueryRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; STimeWindow window = pQueryAttr->window; diff --git a/src/query/src/queryMain.c b/src/query/src/queryMain.c index 787cb2f7d1..d4aa523bf8 100644 --- a/src/query/src/queryMain.c +++ b/src/query/src/queryMain.c @@ -162,7 +162,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi assert(pQueryMsg->stableQuery == isSTableQuery); (*pQInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, vgId, param.sql, qId); + param.pTagColumnInfo, vgId, param.sql, *qId); param.sql = NULL; param.pExprs = NULL; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index b17aa755a5..1eafb5e233 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -98,6 +98,8 @@ typedef struct SIOCostSummary { int64_t blockLoadTime; int64_t statisInfoLoadTime; int64_t checkForNextTime; + int64_t headFileLoad; + int64_t headFileLoadTime; } SIOCostSummary; typedef struct STsdbQueryHandle { @@ -1045,15 +1047,21 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo int32_t code = TSDB_CODE_SUCCESS; *numOfBlocks = 0; + pQueryHandle->cost.headFileLoad += 1; + int64_t s = taosGetTimestampUs(); + size_t numOfTables = 0; if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); + code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); for (int32_t i = 0; i < numOfTables; ++i) { code = loadBlockInfo(pQueryHandle, i, numOfBlocks); if (code != TSDB_CODE_SUCCESS) { + int64_t e = taosGetTimestampUs(); + + pQueryHandle->cost.headFileLoadTime += (e - s); return code; } } @@ -1061,6 +1069,8 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo assert(0); } + int64_t e = taosGetTimestampUs(); + pQueryHandle->cost.headFileLoadTime += (e - s); return code; } @@ -3731,8 +3741,8 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, - pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); + tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, + pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); tfree(pQueryHandle); } From 84da4d656f12921af36be6f001e53fa3c687aa02 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 15 Jul 2021 17:34:22 +0800 Subject: [PATCH 26/31] [td-5250]: compress the tableMeta info msg. --- src/client/src/tscServer.c | 22 +++++++++++++++++++--- src/inc/taosmsg.h | 10 ++++++---- src/mnode/src/mnodeTable.c | 36 ++++++++++++++++++++++++++++-------- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d30ee32d67..657d8b7848 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -13,7 +13,10 @@ * along with this program. If not, see . */ +#include #include "os.h" +#include "qPlan.h" +#include "qTableMeta.h" #include "tcmdtype.h" #include "tlockfree.h" #include "trpc.h" @@ -21,10 +24,8 @@ #include "tscLog.h" #include "tscProfile.h" #include "tscUtil.h" -#include "qTableMeta.h" #include "tsclient.h" #include "ttimer.h" -#include "qPlan.h" int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -2048,16 +2049,27 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { } SSqlCmd *pParentCmd = &pParentSql->cmd; - SHashObj *pSet = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); char* pMsg = pMultiMeta->meta; + char* buf = NULL; + if (pMultiMeta->compressed) { + buf = malloc(pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + int32_t len = tsDecompressString(pMultiMeta->meta, pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, + buf, pMultiMeta->rawLen - sizeof(SMultiTableMeta), ONE_STAGE_COMP, NULL, 0); + assert(len == pMultiMeta->rawLen - sizeof(SMultiTableMeta)); + + pMsg = buf; + } + for (int32_t i = 0; i < pMultiMeta->numOfTables; i++) { STableMetaMsg *pMetaMsg = (STableMetaMsg *)pMsg; int32_t code = tableMetaMsgConvert(pMetaMsg); if (code != TSDB_CODE_SUCCESS) { taosHashCleanup(pSet); taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); return code; } @@ -2066,6 +2078,8 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { tscError("0x%"PRIx64" invalid table meta from mnode, name:%s", pSql->self, pMetaMsg->tableFname); taosHashCleanup(pSet); taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -2115,6 +2129,8 @@ int tscProcessMultiTableMetaRsp(SSqlObj *pSql) { taosHashCleanup(pSet); taosReleaseRef(tscObjRef, pParentSql->self); + + tfree(buf); return TSDB_CODE_SUCCESS; } diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index f20e1535ba..4e76b6dcc1 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -760,10 +760,12 @@ typedef struct STableMetaMsg { } STableMetaMsg; typedef struct SMultiTableMeta { - int32_t numOfTables; - int32_t numOfVgroup; - int32_t contLen; - char meta[]; + int32_t numOfTables; + int32_t numOfVgroup; + uint32_t contLen:31; + uint8_t compressed:1; // denote if compressed or not + uint32_t rawLen; // size before compress + char meta[]; } SMultiTableMeta; typedef struct { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index beeff372aa..ea5611e683 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2892,7 +2892,7 @@ static SMultiTableMeta* ensureMsgBufferSpace(SMultiTableMeta *pMultiMeta, SArray (*totalMallocLen) *= 2; } - pMultiMeta = rpcReallocCont(pMultiMeta, *totalMallocLen); + pMultiMeta = realloc(pMultiMeta, *totalMallocLen); if (pMultiMeta == NULL) { return NULL; } @@ -2923,8 +2923,8 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { } // first malloc 80KB, subsequent reallocation will expand the size as twice of the original size - int32_t totalMallocLen = sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16); - pMultiMeta = rpcMallocCont(totalMallocLen); + int32_t totalMallocLen = sizeof(SMultiTableMeta) + sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16); + pMultiMeta = calloc(1, totalMallocLen); if (pMultiMeta == NULL) { code = TSDB_CODE_MND_OUT_OF_MEMORY; goto _end; @@ -2957,7 +2957,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { int remain = totalMallocLen - pMultiMeta->contLen; if (remain <= sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)) { totalMallocLen *= 2; - pMultiMeta = rpcReallocCont(pMultiMeta, totalMallocLen); + pMultiMeta = realloc(pMultiMeta, totalMallocLen); if (pMultiMeta == NULL) { mnodeDecTableRef(pMsg->pTable); code = TSDB_CODE_MND_OUT_OF_MEMORY; @@ -3027,16 +3027,36 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { pMsg->rpcRsp.len = pMultiMeta->contLen; code = TSDB_CODE_SUCCESS; + char* tmp = rpcMallocCont(pMultiMeta->contLen + 2); + int32_t len = tsCompressString(pMultiMeta->meta, (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, + tmp + sizeof(SMultiTableMeta), (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta) + 2, ONE_STAGE_COMP, NULL, 0); + + pMultiMeta->rawLen = pMultiMeta->contLen; + if (len == -1 || len + sizeof(SMultiTableMeta) >= pMultiMeta->contLen + 2) { // compress failed, do not compress this binary data + pMultiMeta->compressed = 0; + memcpy(tmp, pMultiMeta, sizeof(SMultiTableMeta) + pMultiMeta->contLen); + } else { + pMultiMeta->compressed = 1; + pMultiMeta->contLen = sizeof(SMultiTableMeta) + len; + + // copy the header and the compressed payload + memcpy(tmp, pMultiMeta, sizeof(SMultiTableMeta)); + } + + pMsg->rpcRsp.rsp = tmp; + pMsg->rpcRsp.len = pMultiMeta->contLen; + + SMultiTableMeta* p = (SMultiTableMeta*) tmp; + + mDebug("multiTable info build completed, original:%d, compressed:%d, comp:%d", p->rawLen, p->contLen, p->compressed); + _end: tfree(str); tfree(nameList); taosArrayDestroy(pList); pMsg->pTable = NULL; pMsg->pVgroup = NULL; - - if (code != TSDB_CODE_SUCCESS) { - rpcFreeCont(pMultiMeta); - } + tfree(pMultiMeta); return code; } From c8308dbafd0f5212e944a47cec9e18f13ac99328 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 15 Jul 2021 17:36:16 +0800 Subject: [PATCH 27/31] [td-225]add malloc buffer check. --- src/mnode/src/mnodeTable.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index ea5611e683..6dc2f8ad28 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -3028,6 +3028,11 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { code = TSDB_CODE_SUCCESS; char* tmp = rpcMallocCont(pMultiMeta->contLen + 2); + if (tmp == NULL) { + code = TSDB_CODE_MND_OUT_OF_MEMORY; + goto _end; + } + int32_t len = tsCompressString(pMultiMeta->meta, (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta), 1, tmp + sizeof(SMultiTableMeta), (int32_t)pMultiMeta->contLen - sizeof(SMultiTableMeta) + 2, ONE_STAGE_COMP, NULL, 0); From eabbe88f0e19a7da34e4a403924871879eda6647 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 15 Jul 2021 17:45:42 +0800 Subject: [PATCH 28/31] [td-255] --- src/query/src/qExecutor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index dbcea4e90c..ecbfe36ee3 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2340,6 +2340,7 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool } pQueryAttr->order.order = TSDB_ORDER_ASC; + pQueryAttr->needReverseScan = false; } else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) { if (QUERY_IS_ASC_QUERY(pQueryAttr)) { qDebug(msg, pQInfo, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, @@ -2350,9 +2351,9 @@ static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bool } pQueryAttr->order.order = TSDB_ORDER_DESC; + pQueryAttr->needReverseScan = false; } - pQueryAttr->needReverseScan = false; } else { // interval query if (stableQuery) { if (onlyFirstQuery(pQueryAttr)) { From 4b233396c3aef9d7ba0dd940060a1bffc0186dec Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Thu, 15 Jul 2021 20:28:07 +0800 Subject: [PATCH 29/31] [TD-5286]: fix test case (#6871) * [TD-5286]: test insert and query, the ts should be the same * print timestamp * fix the setBytes test cases --- .../java/com/taosdata/jdbc/TSDBPreparedStatementTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java index 40ff5c23ef..e48237755f 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/TSDBPreparedStatementTest.java @@ -841,13 +841,13 @@ public class TSDBPreparedStatementTest { } @Test - public void setBytes() throws SQLException, IOException { + public void setBytes() throws SQLException { // given long ts = System.currentTimeMillis(); byte[] f8 = "{\"name\": \"john\", \"age\": 10, \"address\": \"192.168.1.100\"}".getBytes(); // when - pstmt_insert.setTimestamp(1, new Timestamp(System.currentTimeMillis())); + pstmt_insert.setTimestamp(1, new Timestamp(ts)); pstmt_insert.setBytes(9, f8); int result = pstmt_insert.executeUpdate(); From 81bc57224a99c69165960ad7bd3b0a1875a65664 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 15 Jul 2021 23:49:39 +0800 Subject: [PATCH 30/31] [TD-5300]: taosdemo stmt debug print. (#6873) * [TD-5300]: taosdemo stmt debug print. * fix default iface is unknown. --- src/kit/taosdemo/taosdemo.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 9c547ff755..0327df5a62 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -569,7 +569,7 @@ SArguments g_args = { 0, // test_mode "127.0.0.1", // host 6030, // port - TAOSC_IFACE, // iface + INTERFACE_BUT, // iface "root", // user #ifdef _TD_POWER_ "powerdb", // password @@ -1429,8 +1429,13 @@ static int printfInsertMeta() { else printf("\ntaosdemo is simulating random data as you request..\n\n"); - printf("interface: \033[33m%s\033[0m\n", - (g_args.iface==TAOSC_IFACE)?"taosc":(g_args.iface==REST_IFACE)?"rest":"stmt"); + if (g_args.iface != INTERFACE_BUT) { + // first time if no iface specified + printf("interface: \033[33m%s\033[0m\n", + (g_args.iface==TAOSC_IFACE)?"taosc": + (g_args.iface==REST_IFACE)?"rest":"stmt"); + } + printf("host: \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port); printf("user: \033[33m%s\033[0m\n", g_Dbs.user); @@ -5038,13 +5043,17 @@ static int32_t execInsert(threadInfo *pThreadInfo, uint32_t k) uint16_t iface; if (superTblInfo) iface = superTblInfo->iface; - else - iface = g_args.iface; + else { + if (g_args.iface == INTERFACE_BUT) + iface = TAOSC_IFACE; + else + iface = g_args.iface; + } debugPrint("[%d] %s() LN%d %s\n", pThreadInfo->threadID, __func__, __LINE__, - (g_args.iface==TAOSC_IFACE)? - "taosc":(g_args.iface==REST_IFACE)?"rest":"stmt"); + (iface==TAOSC_IFACE)? + "taosc":(iface==REST_IFACE)?"rest":"stmt"); switch(iface) { case TAOSC_IFACE: @@ -5884,7 +5893,7 @@ static void printStatPerThread(threadInfo *pThreadInfo) pThreadInfo->threadID, pThreadInfo->totalInsertRows, pThreadInfo->totalAffectedRows, - (double)(pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay/1000.0))); + (pThreadInfo->totalDelay)?(double)((pThreadInfo->totalAffectedRows / (pThreadInfo->totalDelay)/1000.0)): FLT_MAX); } // sync write interlace data @@ -6463,7 +6472,7 @@ static int convertHostToServAddr(char *host, uint16_t port, struct sockaddr_in * } static void startMultiThreadInsertData(int threads, char* db_name, - char* precision,SSuperTable* superTblInfo) { + char* precision, SSuperTable* superTblInfo) { int32_t timePrec = TSDB_TIME_PRECISION_MILLI; if (0 != precision[0]) { @@ -7936,7 +7945,12 @@ static void setParaFromArg(){ tstrncpy(g_Dbs.db[0].superTbls[0].childTblPrefix, g_args.tb_prefix, TSDB_TABLE_NAME_LEN - 20); tstrncpy(g_Dbs.db[0].superTbls[0].dataSource, "rand", MAX_TB_NAME_SIZE); - g_Dbs.db[0].superTbls[0].iface = g_args.iface; + + if (g_args.iface == INTERFACE_BUT) { + g_Dbs.db[0].superTbls[0].iface = TAOSC_IFACE; + } else { + g_Dbs.db[0].superTbls[0].iface = g_args.iface; + } tstrncpy(g_Dbs.db[0].superTbls[0].startTimestamp, "2017-07-14 10:40:00.000", MAX_TB_NAME_SIZE); g_Dbs.db[0].superTbls[0].timeStampStep = DEFAULT_TIMESTAMP_STEP; From 9d89c37d70e1f3c417bd9aaede91515e77c07477 Mon Sep 17 00:00:00 2001 From: Zhiyu Yang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Fri, 16 Jul 2021 09:35:43 +0800 Subject: [PATCH 31/31] Feature/td 5265 (#6878) * [TD-5265]: add test case for executeBatch in Statement * [TD-5265]: support continue process in Statement.executeBatch method * change * change * change * change --- src/connector/jdbc/readme.md | 492 ++++++++++++++---- .../com/taosdata/jdbc/AbstractStatement.java | 42 +- .../java/com/taosdata/jdbc/TSDBConstants.java | 2 + .../java/com/taosdata/jdbc/TSDBDriver.java | 5 + .../jdbc/cases/BatchErrorIgnoreTest.java | 144 +++++ 5 files changed, 567 insertions(+), 118 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java diff --git a/src/connector/jdbc/readme.md b/src/connector/jdbc/readme.md index e81f078c15..3c52ebb00a 100644 --- a/src/connector/jdbc/readme.md +++ b/src/connector/jdbc/readme.md @@ -1,54 +1,62 @@ +# Java Connector -## TAOS-JDBCDriver 概述 +TDengine 提供了遵循 JDBC 标准（3.0）API 规范的 `taos-jdbcdriver` 实现，可在 maven 的中央仓库 [Sonatype Repository][1] 搜索下载。 -TDengine 为了方便 Java 应用使用，提供了遵循 JDBC 标准(3.0)API 规范的 `taos-jdbcdriver` 实现。目前可以通过 [Sonatype Repository][1] 搜索并下载。 +`taos-jdbcdriver` 的实现包括 2 种形式： JDBC-JNI 和 JDBC-RESTful（taos-jdbcdriver-2.0.18 开始支持 JDBC-RESTful）。 JDBC-JNI 通过调用客户端 libtaos.so（或 taos.dll ）的本地方法实现， JDBC-RESTful 则在内部封装了 RESTful 接口实现。 -由于 TDengine 是使用 c 语言开发的，使用 taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 +![tdengine-connector](https://www.taosdata.com/cn/documentation/user/pages/images/tdengine-jdbc-connector.png) -* libtaos.so - 在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 - -* taos.dll - 在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 - -> 注意：在 windows 环境开发时需要安装 TDengine 对应的 windows 版本客户端，由于目前没有提供 Linux 环境单独的客户端，需要安装 TDengine 才能使用。 +上图显示了 3 种 Java 应用使用连接器访问 TDengine 的方式： -TDengine 的 JDBC 驱动实现尽可能的与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征的差异导致 taos-jdbcdriver 并未完全实现 JDBC 标准规范。在使用时需要注意以下几点： +* JDBC-JNI：Java 应用在物理节点1（pnode1）上使用 JDBC-JNI 的 API ，直接调用客户端 API（libtaos.so 或 taos.dll）将写入和查询请求发送到位于物理节点2（pnode2）上的 taosd 实例。 +* RESTful：应用将 SQL 发送给位于物理节点2（pnode2）上的 RESTful 连接器，再调用客户端 API（libtaos.so）。 +* JDBC-RESTful：Java 应用通过 JDBC-RESTful 的 API ，将 SQL 封装成一个 RESTful 请求，发送给物理节点2的 RESTful 连接器。 -* TDengine 不提供针对单条数据记录的删除和修改的操作，驱动中也没有支持相关方法。 -* 由于不支持删除和修改，所以也不支持事务操作。 -* 目前不支持表间的 union 操作。 -* 目前不支持嵌套查询(nested query)，`对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet还没关闭的情况下执行了新的查询，TSDBJDBCDriver 则会自动关闭上一个 ResultSet`。 +TDengine 的 JDBC 驱动实现尽可能与关系型数据库驱动保持一致，但时序空间数据库与关系对象型数据库服务的对象和技术特征存在差异，导致 `taos-jdbcdriver` 与传统的 JDBC driver 也存在一定差异。在使用时需要注意以下几点： + +* TDengine 目前不支持针对单条数据记录的删除操作。 +* 目前不支持事务操作。 +* 目前不支持嵌套查询（nested query）。 +* 对每个 Connection 的实例，至多只能有一个打开的 ResultSet 实例；如果在 ResultSet 还没关闭的情况下执行了新的查询，taos-jdbcdriver 会自动关闭上一个 ResultSet。 -## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 +## JDBC-JNI和JDBC-RESTful的对比 -| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | -| --- | --- | --- | -| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | -| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | + + + + + + + + + + + + + + + + + + + + + + + + + +
对比项JDBC-JNIJDBC-RESTful
支持的操作系统linux、windows全平台
是否需要安装 client需要不需要
server 升级后是否需要升级 client需要不需要
写入性能JDBC-RESTful 是 JDBC-JNI 的 50%～90%
查询性能JDBC-RESTful 与 JDBC-JNI 没有差别
-## TDengine DataType 和 Java DataType +注意：与 JNI 方式不同，RESTful 接口是无状态的，因此 `USE db_name` 指令没有效果，RESTful 下所有对表名、超级表名的引用都需要指定数据库名前缀。 -TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： - -| TDengine DataType | Java DataType | -| --- | --- | -| TIMESTAMP | java.sql.Timestamp | -| INT | java.lang.Integer | -| BIGINT | java.lang.Long | -| FLOAT | java.lang.Float | -| DOUBLE | java.lang.Double | -| SMALLINT, TINYINT |java.lang.Short | -| BOOL | java.lang.Boolean | -| BINARY, NCHAR | java.lang.String | - -## 如何获取 TAOS-JDBCDriver +## 如何获取 taos-jdbcdriver ### maven 仓库 目前 taos-jdbcdriver 已经发布到 [Sonatype Repository][1] 仓库，且各大仓库都已同步。 + * [sonatype][8] * [mvnrepository][9] * [maven.aliyun][10] @@ -56,56 +64,86 @@ TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对 maven 项目中使用如下 pom.xml 配置即可： ```xml - - - com.taosdata.jdbc - taos-jdbcdriver - 1.0.3 - - + + com.taosdata.jdbc + taos-jdbcdriver + 2.0.18 + ``` ### 源码编译打包 -下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package` 即可生成相应 jar 包。 +下载 [TDengine][3] 源码之后，进入 taos-jdbcdriver 源码目录 `src/connector/jdbc` 执行 `mvn clean package -Dmaven.test.skip=true` 即可生成相应 jar 包。 -## 使用说明 + +## JDBC的使用说明 ### 获取连接 -如下所示配置即可获取 TDengine Connection： +#### 指定URL获取连接 + +通过指定URL获取连接，如下所示： + ```java -Class.forName("com.taosdata.jdbc.TSDBDriver"); -String jdbcUrl = "jdbc:TAOS://127.0.0.1:6030/log?user=root&password=taosdata"; +Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); +String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; Connection conn = DriverManager.getConnection(jdbcUrl); ``` -> 端口 6030 为默认连接端口，JDBC URL 中的 log 为系统本身的监控数据库。 + +以上示例，使用 **JDBC-RESTful** 的 driver，建立了到 hostname 为 taosdemo.com，端口为 6041，数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。 + +使用 JDBC-RESTful 接口，不需要依赖本地函数库。与 JDBC-JNI 相比，仅需要： + +1. driverClass 指定为“com.taosdata.jdbc.rs.RestfulDriver”； +2. jdbcUrl 以“jdbc:TAOS-RS://”开头； +3. 使用 6041 作为连接端口。 + +如果希望获得更好的写入和查询性能，Java 应用可以使用 **JDBC-JNI** 的driver，如下所示： + +```java +Class.forName("com.taosdata.jdbc.TSDBDriver"); +String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; +Connection conn = DriverManager.getConnection(jdbcUrl); +``` + +以上示例，使用了 JDBC-JNI 的 driver，建立了到 hostname 为 taosdemo.com，端口为 6030（TDengine 的默认端口），数据库名为 test 的连接。这个 URL 中指定用户名（user）为 root，密码（password）为 taosdata。 + +**注意**：使用 JDBC-JNI 的 driver，taos-jdbcdriver 驱动包时需要依赖系统对应的本地函数库。 + +* libtaos.so + 在 linux 系统中成功安装 TDengine 后，依赖的本地函数库 libtaos.so 文件会被自动拷贝至 /usr/lib/libtaos.so，该目录包含在 Linux 自动扫描路径上，无需单独指定。 + +* taos.dll + 在 windows 系统中安装完客户端之后，驱动包依赖的 taos.dll 文件会自动拷贝到系统默认搜索路径 C:/Windows/System32 下，同样无需要单独指定。 + +> 在 windows 环境开发时需要安装 TDengine 对应的 [windows 客户端][14]，Linux 服务器安装完 TDengine 之后默认已安装 client，也可以单独安装 [Linux 客户端][15] 连接远程 TDengine Server。 + +JDBC-JNI 的使用请参见[视频教程](https://www.taosdata.com/blog/2020/11/11/1955.html)。 TDengine 的 JDBC URL 规范格式为： -`jdbc:TSDB://{host_ip}:{port}/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]` - -其中，`{}` 中的内容必须，`[]` 中为可选。配置参数说明如下： +`jdbc:[TAOS|TAOS-RS]://[host_name]:[port]/[database_name]?[user={user}|&password={password}|&charset={charset}|&cfgdir={config_dir}|&locale={locale}|&timezone={timezone}]` +url中的配置参数如下： * user：登录 TDengine 用户名，默认值 root。 * password：用户登录密码，默认值 taosdata。 -* charset：客户端使用的字符集，默认值为系统字符集。 * cfgdir：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。 +* charset：客户端使用的字符集，默认值为系统字符集。 * locale：客户端语言环境，默认值系统当前 locale。 * timezone：客户端使用的时区，默认值为系统当前时区。 -以上参数可以在 3 处配置，`优先级由高到低`分别如下： -1. JDBC URL 参数 - 如上所述，可以在 JDBC URL 的参数中指定。 -2. java.sql.DriverManager.getConnection(String jdbcUrl, Properties connProps) + + +#### 指定URL和Properties获取连接 + +除了通过指定的 URL 获取连接，还可以使用 Properties 指定建立连接时的参数，如下所示： ```java public Connection getConn() throws Exception{ Class.forName("com.taosdata.jdbc.TSDBDriver"); - String jdbcUrl = "jdbc:TAOS://127.0.0.1:0/log?user=root&password=taosdata"; + // Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + String jdbcUrl = "jdbc:TAOS://taosdemo.com:6030/test?user=root&password=taosdata"; + // String jdbcUrl = "jdbc:TAOS-RS://taosdemo.com:6041/test?user=root&password=taosdata"; Properties connProps = new Properties(); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_USER, "root"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_PASSWORD, "taosdata"); - connProps.setProperty(TSDBDriver.PROPERTY_KEY_CONFIG_DIR, "/etc/taos"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); @@ -114,22 +152,68 @@ public Connection getConn() throws Exception{ } ``` -3. 客户端配置文件 taos.cfg +以上示例，建立一个到 hostname 为 taosdemo.com，端口为 6030，数据库名为 test 的连接。注释为使用 JDBC-RESTful 时的方法。这个连接在 url 中指定了用户名(user)为 root，密码（password）为 taosdata，并在 connProps 中指定了使用的字符集、语言环境、时区等信息。 - linux 系统默认配置文件为 /var/lib/taos/taos.cfg，windows 系统默认配置文件路径为 C:\TDengine\cfg\taos.cfg。 -```properties -# client default username -# defaultUser root +properties 中的配置参数如下： +* TSDBDriver.PROPERTY_KEY_USER：登录 TDengine 用户名，默认值 root。 +* TSDBDriver.PROPERTY_KEY_PASSWORD：用户登录密码，默认值 taosdata。 +* TSDBDriver.PROPERTY_KEY_CONFIG_DIR：客户端配置文件目录路径，Linux OS 上默认值 /etc/taos ，Windows OS 上默认值 C:/TDengine/cfg。 +* TSDBDriver.PROPERTY_KEY_CHARSET：客户端使用的字符集，默认值为系统字符集。 +* TSDBDriver.PROPERTY_KEY_LOCALE：客户端语言环境，默认值系统当前 locale。 +* TSDBDriver.PROPERTY_KEY_TIME_ZONE：客户端使用的时区，默认值为系统当前时区。 -# client default password -# defaultPass taosdata + + +#### 使用客户端配置文件建立连接 + +当使用 JDBC-JNI 连接 TDengine 集群时，可以使用客户端配置文件，在客户端配置文件中指定集群的 firstEp、secondEp参数。 +如下所示： + +1. 在 Java 应用中不指定 hostname 和 port + +```java +public Connection getConn() throws Exception{ + Class.forName("com.taosdata.jdbc.TSDBDriver"); + String jdbcUrl = "jdbc:TAOS://:/test?user=root&password=taosdata"; + Properties connProps = new Properties(); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_CHARSET, "UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_LOCALE, "en_US.UTF-8"); + connProps.setProperty(TSDBDriver.PROPERTY_KEY_TIME_ZONE, "UTC-8"); + Connection conn = DriverManager.getConnection(jdbcUrl, connProps); + return conn; +} +``` + +2. 在配置文件中指定 firstEp 和 secondEp + +``` +# first fully qualified domain name (FQDN) for TDengine system +firstEp cluster_node1:6030 + +# second fully qualified domain name (FQDN) for TDengine system, for cluster only +secondEp cluster_node2:6030 # default system charset -# charset UTF-8 +# charset UTF-8 # system locale # locale en_US.UTF-8 ``` + +以上示例，jdbc 会使用客户端的配置文件，建立到 hostname 为 cluster_node1、端口为 6030、数据库名为 test 的连接。当集群中 firstEp 节点失效时，JDBC 会尝试使用 secondEp 连接集群。 +TDengine 中，只要保证 firstEp 和 secondEp 中一个节点有效，就可以正常建立到集群的连接。 + +> 注意：这里的配置文件指的是调用 JDBC Connector 的应用程序所在机器上的配置文件，Linux OS 上默认值 /etc/taos/taos.cfg ，Windows OS 上默认值 C://TDengine/cfg/taos.cfg。 + +#### 配置参数的优先级 + +通过以上 3 种方式获取连接，如果配置参数在 url、Properties、客户端配置文件中有重复，则参数的`优先级由高到低`分别如下： +1. JDBC URL 参数，如上所述，可以在 JDBC URL 的参数中指定。 +2. Properties connProps +3. 客户端配置文件 taos.cfg + +例如：在 url 中指定了 password 为 taosdata，在 Properties 中指定了 password 为 taosdemo，那么，JDBC 会使用 url 中的 password 建立连接。 + > 更多详细配置请参考[客户端配置][13] ### 创建数据库和表 @@ -146,6 +230,7 @@ stmt.executeUpdate("use db"); // create table stmt.executeUpdate("create table if not exists tb (ts timestamp, temperature int, humidity float)"); ``` + > 注意：如果不使用 `use db` 指定数据库，则后续对表的操作都需要增加数据库名称作为前缀，如 db.tb。 ### 插入数据 @@ -156,6 +241,7 @@ int affectedRows = stmt.executeUpdate("insert into tb values(now, 23, 10.3) (now System.out.println("insert " + affectedRows + " rows."); ``` + > now 为系统内部函数，默认为服务器当前时间。 > `now + 1s` 代表服务器当前时间往后加 1 秒，数字后面代表时间单位：a(毫秒), s(秒), m(分), h(小时), d(天)，w(周), n(月), y(年)。 @@ -177,8 +263,150 @@ while(resultSet.next()){ System.out.printf("%s, %d, %s\n", ts, temperature, humidity); } ``` + > 查询和操作关系型数据库一致，使用下标获取返回字段内容时从 1 开始，建议使用字段名称获取。 +### 处理异常 + +在报错后，通过SQLException可以获取到错误的信息和错误码： + +```java +try (Statement statement = connection.createStatement()) { + // executeQuery + ResultSet resultSet = statement.executeQuery(sql); + // print result + printResult(resultSet); +} catch (SQLException e) { + System.out.println("ERROR Message: " + e.getMessage()); + System.out.println("ERROR Code: " + e.getErrorCode()); + e.printStackTrace(); +} +``` + +JDBC连接器可能报错的错误码包括3种：JDBC driver本身的报错（错误码在0x2301到0x2350之间），JNI方法的报错（错误码在0x2351到0x2400之间），TDengine其他功能模块的报错。 +具体的错误码请参考： +* https://github.com/taosdata/TDengine/blob/develop/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBErrorNumbers.java +* https://github.com/taosdata/TDengine/blob/develop/src/inc/taoserror.h + +### 通过参数绑定写入数据 + +从 2.1.2.0 版本开始，TDengine 的 **JDBC-JNI** 实现大幅改进了参数绑定方式对数据写入（INSERT）场景的支持。采用这种方式写入数据时，能避免 SQL 语法解析的资源消耗，从而在很多情况下显著提升写入性能。（注意：**JDBC-RESTful** 实现并不提供参数绑定这种使用方式。） + +```java +Statement stmt = conn.createStatement(); +Random r = new Random(); + +// INSERT 语句中，VALUES 部分允许指定具体的数据列；如果采取自动建表，则 TAGS 部分需要设定全部 TAGS 列的参数值： +TSDBPreparedStatement s = (TSDBPreparedStatement) conn.prepareStatement("insert into ? using weather_test tags (?, ?) (ts, c1, c2) values(?, ?, ?)"); + +// 设定数据表名： +s.setTableName("w1"); +// 设定 TAGS 取值： +s.setTagInt(0, r.nextInt(10)); +s.setTagString(1, "Beijing"); + +int numOfRows = 10; + +// VALUES 部分以逐列的方式进行设置： +ArrayList ts = new ArrayList<>(); +for (int i = 0; i < numOfRows; i++){ + ts.add(System.currentTimeMillis() + i); +} +s.setTimestamp(0, ts); + +ArrayList s1 = new ArrayList<>(); +for (int i = 0; i < numOfRows; i++){ + s1.add(r.nextInt(100)); +} +s.setInt(1, s1); + +ArrayList s2 = new ArrayList<>(); +for (int i = 0; i < numOfRows; i++){ + s2.add("test" + r.nextInt(100)); +} +s.setString(2, s2, 10); + +// AddBatch 之后，缓存并未清空。为避免混乱，并不推荐在 ExecuteBatch 之前再次绑定新一批的数据： +s.columnDataAddBatch(); +// 执行绑定数据后的语句： +s.columnDataExecuteBatch(); +// 执行语句后清空缓存。在清空之后，可以复用当前的对象，绑定新的一批数据（可以是新表名、新 TAGS 值、新 VALUES 值）： +s.columnDataClearBatch(); +// 执行完毕，释放资源： +s.columnDataCloseBatch(); +``` + +用于设定 TAGS 取值的方法总共有： +```java +public void setTagNull(int index, int type) +public void setTagBoolean(int index, boolean value) +public void setTagInt(int index, int value) +public void setTagByte(int index, byte value) +public void setTagShort(int index, short value) +public void setTagLong(int index, long value) +public void setTagTimestamp(int index, long value) +public void setTagFloat(int index, float value) +public void setTagDouble(int index, double value) +public void setTagString(int index, String value) +public void setTagNString(int index, String value) +``` + +用于设定 VALUES 数据列的取值的方法总共有： +```java +public void setInt(int columnIndex, ArrayList list) throws SQLException +public void setFloat(int columnIndex, ArrayList list) throws SQLException +public void setTimestamp(int columnIndex, ArrayList list) throws SQLException +public void setLong(int columnIndex, ArrayList list) throws SQLException +public void setDouble(int columnIndex, ArrayList list) throws SQLException +public void setBoolean(int columnIndex, ArrayList list) throws SQLException +public void setByte(int columnIndex, ArrayList list) throws SQLException +public void setShort(int columnIndex, ArrayList list) throws SQLException +public void setString(int columnIndex, ArrayList list, int size) throws SQLException +public void setNString(int columnIndex, ArrayList list, int size) throws SQLException +``` +其中 setString 和 setNString 都要求用户在 size 参数里声明表定义中对应列的列宽。 + +### 订阅 + +#### 创建 + +```java +TSDBSubscribe sub = ((TSDBConnection)conn).subscribe("topic", "select * from meters", false); +``` + +`subscribe` 方法的三个参数含义如下： + +* topic：订阅的主题（即名称），此参数是订阅的唯一标识 +* sql：订阅的查询语句，此语句只能是 `select` 语句，只应查询原始数据，只能按时间正序查询数据 +* restart：如果订阅已经存在，是重新开始，还是继续之前的订阅 + +如上面的例子将使用 SQL 语句 `select * from meters` 创建一个名为 `topic` 的订阅，如果这个订阅已经存在，将继续之前的查询进度，而不是从头开始消费所有的数据。 + +#### 消费数据 + +```java +int total = 0; +while(true) { + TSDBResultSet rs = sub.consume(); + int count = 0; + while(rs.next()) { + count++; + } + total += count; + System.out.printf("%d rows consumed, total %d\n", count, total); + Thread.sleep(1000); +} +``` + +`consume` 方法返回一个结果集，其中包含从上次 `consume` 到目前为止的所有新数据。请务必按需选择合理的调用 `consume` 的频率（如例子中的 `Thread.sleep(1000)`），否则会给服务端造成不必要的压力。 + +#### 关闭订阅 + +```java +sub.close(true); +``` + +`close` 方法关闭一个订阅。如果其参数为 `true` 表示保留订阅进度信息，后续可以创建同名订阅继续消费数据；如为 `false` 则不保留订阅进度。 ### 关闭资源 @@ -187,12 +415,17 @@ resultSet.close(); stmt.close(); conn.close(); ``` + > `注意务必要将 connection 进行关闭`，否则会出现连接泄露。 + + + ## 与连接池使用 **HikariCP** * 引入相应 HikariCP maven 依赖： + ```xml com.zaxxer @@ -202,31 +435,34 @@ conn.close(); ``` * 使用示例如下： + ```java public static void main(String[] args) throws SQLException { HikariConfig config = new HikariConfig(); + // jdbc properties config.setJdbcUrl("jdbc:TAOS://127.0.0.1:6030/log"); config.setUsername("root"); config.setPassword("taosdata"); - - config.setMinimumIdle(3); //minimum number of idle connection + // connection pool configurations + config.setMinimumIdle(10); //minimum number of idle connection config.setMaximumPoolSize(10); //maximum number of connection in the pool - config.setConnectionTimeout(10000); //maximum wait milliseconds for get connection from pool - config.setIdleTimeout(60000); // max idle time for recycle idle connection - config.setConnectionTestQuery("describe log.dn"); //validation query - config.setValidationTimeout(3000); //validation query timeout + config.setConnectionTimeout(30000); //maximum wait milliseconds for get connection from pool + config.setMaxLifetime(0); // maximum life time for each connection + config.setIdleTimeout(0); // max idle time for recycle idle connection + config.setConnectionTestQuery("select server_status()"); //validation query HikariDataSource ds = new HikariDataSource(config); //create datasource - + Connection connection = ds.getConnection(); // get connection Statement statement = connection.createStatement(); // get statement - - //query or insert + + //query or insert // ... - + connection.close(); // put back to conneciton pool } ``` + > 通过 HikariDataSource.getConnection() 获取连接后，使用完成后需要调用 close() 方法，实际上它并不会关闭连接，只是放回连接池中。 > 更多 HikariCP 使用问题请查看[官方说明][5] @@ -243,40 +479,32 @@ conn.close(); ``` * 使用示例如下： + ```java public static void main(String[] args) throws Exception { - Properties properties = new Properties(); - properties.put("driverClassName","com.taosdata.jdbc.TSDBDriver"); - properties.put("url","jdbc:TAOS://127.0.0.1:6030/log"); - properties.put("username","root"); - properties.put("password","taosdata"); - properties.put("maxActive","10"); //maximum number of connection in the pool - properties.put("initialSize","3");//initial number of connection - properties.put("maxWait","10000");//maximum wait milliseconds for get connection from pool - properties.put("minIdle","3");//minimum number of connection in the pool - - properties.put("timeBetweenEvictionRunsMillis","3000");// the interval milliseconds to test connection - - properties.put("minEvictableIdleTimeMillis","60000");//the minimum milliseconds to keep idle - properties.put("maxEvictableIdleTimeMillis","90000");//the maximum milliseconds to keep idle - - properties.put("validationQuery","describe log.dn"); //validation query - properties.put("testWhileIdle","true"); // test connection while idle - properties.put("testOnBorrow","false"); // don't need while testWhileIdle is true - properties.put("testOnReturn","false"); // don't need while testWhileIdle is true - - //create druid datasource - DataSource ds = DruidDataSourceFactory.createDataSource(properties); - Connection connection = ds.getConnection(); // get connection + DruidDataSource dataSource = new DruidDataSource(); + // jdbc properties + dataSource.setDriverClassName("com.taosdata.jdbc.TSDBDriver"); + dataSource.setUrl(url); + dataSource.setUsername("root"); + dataSource.setPassword("taosdata"); + // pool configurations + dataSource.setInitialSize(10); + dataSource.setMinIdle(10); + dataSource.setMaxActive(10); + dataSource.setMaxWait(30000); + dataSource.setValidationQuery("select server_status()"); + + Connection connection = dataSource.getConnection(); // get connection Statement statement = connection.createStatement(); // get statement - //query or insert // ... connection.close(); // put back to conneciton pool } ``` + > 更多 druid 使用问题请查看[官方说明][6] **注意事项** @@ -291,29 +519,64 @@ server_status()| Query OK, 1 row(s) in set (0.000141s) ``` + + ## 与框架使用 * Spring JdbcTemplate 中使用 taos-jdbcdriver，可参考 [SpringJdbcTemplate][11] * Springboot + Mybatis 中使用，可参考 [springbootdemo][12] + + +## TAOS-JDBCDriver 版本以及支持的 TDengine 版本和 JDK 版本 + +| taos-jdbcdriver 版本 | TDengine 版本 | JDK 版本 | +| -------------------- | ----------------- | -------- | +| 2.0.31 | 2.1.3.0 及以上 | 1.8.x | +| 2.0.22 - 2.0.30 | 2.0.18.0 - 2.1.2.x | 1.8.x | +| 2.0.12 - 2.0.21 | 2.0.8.0 - 2.0.17.x | 1.8.x | +| 2.0.4 - 2.0.11 | 2.0.0.0 - 2.0.7.x | 1.8.x | +| 1.0.3 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.2 | 1.6.1.x 及以上 | 1.8.x | +| 1.0.1 | 1.6.1.x 及以上 | 1.8.x | + + + +## TDengine DataType 和 Java DataType + +TDengine 目前支持时间戳、数字、字符、布尔类型，与 Java 对应类型转换如下： + +| TDengine DataType | Java DataType | +| ----------------- | ------------------ | +| TIMESTAMP | java.sql.Timestamp | +| INT | java.lang.Integer | +| BIGINT | java.lang.Long | +| FLOAT | java.lang.Float | +| DOUBLE | java.lang.Double | +| SMALLINT | java.lang.Short | +| TINYINT | java.lang.Byte | +| BOOL | java.lang.Boolean | +| BINARY | byte array | +| NCHAR | java.lang.String | + + + ## 常见问题 * java.lang.UnsatisfiedLinkError: no taos in java.library.path - + **原因**：程序没有找到依赖的本地函数库 taos。 - - **解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 ` ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 - + + **解决方法**：windows 下可以将 C:\TDengine\driver\taos.dll 拷贝到 C:\Windows\System32\ 目录下，linux 下将建立如下软链 `ln -s /usr/local/taos/driver/libtaos.so.x.x.x.x /usr/lib/libtaos.so` 即可。 + * java.lang.UnsatisfiedLinkError: taos.dll Can't load AMD 64 bit on a IA 32-bit platform - + **原因**：目前 TDengine 只支持 64 位 JDK。 - + **解决方法**：重新安装 64 位 JDK。 * 其它问题请参考 [Issues][7] - - [1]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver [2]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver [3]: https://github.com/taosdata/TDengine @@ -324,6 +587,9 @@ Query OK, 1 row(s) in set (0.000141s) [8]: https://search.maven.org/artifact/com.taosdata.jdbc/taos-jdbcdriver [9]: https://mvnrepository.com/artifact/com.taosdata.jdbc/taos-jdbcdriver [10]: https://maven.aliyun.com/mvn/search -[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate +[11]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/SpringJdbcTemplate [12]: https://github.com/taosdata/TDengine/tree/develop/tests/examples/JDBC/springbootdemo -[13]: https://www.taosdata.com/cn/documentation/administrator/#%E5%AE%A2%E6%88%B7%E7%AB%AF%E9%85%8D%E7%BD%AE \ No newline at end of file +[13]: https://www.taosdata.com/cn/documentation/administrator/#client +[14]: https://www.taosdata.com/cn/all-downloads/#TDengine-Windows-Client +[15]: https://www.taosdata.com/cn/getting-started/#%E5%AE%A2%E6%88%B7%E7%AB%AF + diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java index 8b6c074d1b..a801f5a674 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/AbstractStatement.java @@ -2,6 +2,7 @@ package com.taosdata.jdbc; import java.sql.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public abstract class AbstractStatement extends WrapperImpl implements Statement { @@ -196,13 +197,44 @@ public abstract class AbstractStatement extends WrapperImpl implements Statement if (batchedArgs == null || batchedArgs.isEmpty()) throw TSDBError.createSQLException(TSDBErrorNumbers.ERROR_BATCH_IS_EMPTY); + String clientInfo = getConnection().getClientInfo(TSDBDriver.PROPERTY_KEY_BATCH_ERROR_IGNORE); + boolean batchErrorIgnore = clientInfo == null ? TSDBConstants.DEFAULT_BATCH_ERROR_IGNORE : Boolean.parseBoolean(clientInfo); + + if (batchErrorIgnore) { + return executeBatchIgnoreException(); + } + return executeBatchThrowException(); + } + + private int[] executeBatchIgnoreException() { + return batchedArgs.stream().mapToInt(sql -> { + try { + boolean isSelect = execute(sql); + if (isSelect) { + return SUCCESS_NO_INFO; + } else { + return getUpdateCount(); + } + } catch (SQLException e) { + return EXECUTE_FAILED; + } + }).toArray(); + } + + private int[] executeBatchThrowException() throws BatchUpdateException { int[] res = new int[batchedArgs.size()]; for (int i = 0; i < batchedArgs.size(); i++) { - boolean isSelect = execute(batchedArgs.get(i)); - if (isSelect) { - res[i] = SUCCESS_NO_INFO; - } else { - res[i] = getUpdateCount(); + try { + boolean isSelect = execute(batchedArgs.get(i)); + if (isSelect) { + res[i] = SUCCESS_NO_INFO; + } else { + res[i] = getUpdateCount(); + } + } catch (SQLException e) { + String reason = e.getMessage(); + int[] updateCounts = Arrays.copyOfRange(res, 0, i); + throw new BatchUpdateException(reason, updateCounts, e); } } return res; diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java index 5b5128e720..f3f04eff12 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBConstants.java @@ -74,6 +74,8 @@ public abstract class TSDBConstants { public static final String DEFAULT_PRECISION = "ms"; + public static final boolean DEFAULT_BATCH_ERROR_IGNORE = false; + public static int typeName2JdbcType(String type) { switch (type.toUpperCase()) { case "TIMESTAMP": diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java index 98a7d1929b..f5f16758c1 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBDriver.java @@ -100,6 +100,11 @@ public class TSDBDriver extends AbstractDriver { */ public static final String PROPERTY_KEY_TIMESTAMP_FORMAT = "timestampFormat"; + /** + * continue process commands in executeBatch + */ + public static final String PROPERTY_KEY_BATCH_ERROR_IGNORE = "batchErrorIgnore"; + private TSDBDatabaseMetaData dbMetaData = null; static { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java new file mode 100644 index 0000000000..2934b54b5b --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/BatchErrorIgnoreTest.java @@ -0,0 +1,144 @@ +package com.taosdata.jdbc.cases; + +import org.junit.*; + +import java.sql.*; +import java.util.stream.IntStream; + +public class BatchErrorIgnoreTest { + + private static final String host = "127.0.0.1"; + + @Test + public void batchErrorThrowException() throws SQLException { + // given + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + + // when + try (Statement stmt = conn.createStatement()) { + IntStream.range(1, 6).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("insert into t11 values(now, 11)"); + IntStream.range(6, 11).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + "),(now + 1s, " + (10 * i) + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("select count(*) from test.weather"); + + stmt.executeBatch(); + } catch (BatchUpdateException e) { + int[] updateCounts = e.getUpdateCounts(); + Assert.assertEquals(5, updateCounts.length); + Assert.assertEquals(1, updateCounts[0]); + Assert.assertEquals(1, updateCounts[1]); + Assert.assertEquals(1, updateCounts[2]); + Assert.assertEquals(1, updateCounts[3]); + Assert.assertEquals(1, updateCounts[4]); + } + + } + + @Test + public void batchErrorIgnore() throws SQLException { + // given + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata&batchErrorIgnore=true"); + + // when + int[] results = null; + try (Statement stmt = conn.createStatement()) { + IntStream.range(1, 6).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("insert into t11 values(now, 11)"); + IntStream.range(6, 11).mapToObj(i -> "insert into test.t" + i + " values(now, " + i + "),(now + 1s, " + (10 * i) + ")").forEach(sql -> { + try { + stmt.addBatch(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.addBatch("select count(*) from test.weather"); + + results = stmt.executeBatch(); + } catch (SQLException e) { + e.printStackTrace(); + } + + // then + assert results != null; + Assert.assertEquals(12, results.length); + Assert.assertEquals(1, results[0]); + Assert.assertEquals(1, results[1]); + Assert.assertEquals(1, results[2]); + Assert.assertEquals(1, results[3]); + Assert.assertEquals(1, results[4]); + Assert.assertEquals(Statement.EXECUTE_FAILED, results[5]); + Assert.assertEquals(2, results[6]); + Assert.assertEquals(2, results[7]); + Assert.assertEquals(2, results[8]); + Assert.assertEquals(2, results[9]); + Assert.assertEquals(2, results[10]); + Assert.assertEquals(Statement.SUCCESS_NO_INFO, results[11]); + } + + @Before + public void before() { + try { + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("use test"); + stmt.execute("drop table if exists weather"); + stmt.execute("create table weather (ts timestamp, f1 float) tags(t1 int)"); + IntStream.range(1, 11).mapToObj(i -> "create table t" + i + " using weather tags(" + i + ")").forEach(sql -> { + try { + stmt.execute(sql); + } catch (SQLException e) { + e.printStackTrace(); + } + }); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @BeforeClass + public static void beforeClass() { + try { + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists test"); + stmt.execute("create database if not exists test"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @AfterClass + public static void afterClass() { + try { + Connection conn = DriverManager.getConnection("jdbc:TAOS://" + host + ":6030/?user=root&password=taosdata"); + Statement stmt = conn.createStatement(); + stmt.execute("drop database if exists test"); + stmt.close(); + conn.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } +}