From f3fd5d1a66cc63bf03fa0e1f6119fca9ad18975e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 14:49:11 +0800 Subject: [PATCH 1/6] [td-5376]: improve last_row query performance. --- src/tsdb/src/tsdbRead.c | 22 +++-- src/util/CMakeLists.txt | 5 +- src/util/inc/tarray.h | 9 ++ src/util/src/tarray.c | 41 ++++++++++ src/util/src/tskiplist.c | 140 -------------------------------- src/util/tests/cacheTest.cpp | 8 +- src/util/tests/skiplistTest.cpp | 27 +++--- src/util/tests/trefTest.c | 16 ++-- 8 files changed, 90 insertions(+), 178 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 1eafb5e233..924b5e24a8 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3002,7 +3002,7 @@ int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *g } // update the tsdb query time range - if (pQueryHandle->cachelastrow) { + if (pQueryHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { pQueryHandle->window = TSWINDOW_INITIALIZER; pQueryHandle->checkFiles = false; pQueryHandle->activeIndex = -1; // start from -1 @@ -3034,6 +3034,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; int32_t totalNumOfTable = 0; + SArray* emptyGroup = taosArrayInit(16, sizeof(int32_t)); // NOTE: starts from the buffer in case of descending timestamp order check data blocks size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); @@ -3076,18 +3077,18 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { } } - taosArrayClear(pGroup); - // more than one table in each group, only one table left for each group if (keyInfo.pTable != NULL) { totalNumOfTable++; - taosArrayPush(pGroup, &keyInfo); - } else { + if (taosArrayGetSize(pGroup) == 1) { + // do nothing + } else { + taosArrayClear(pGroup); + taosArrayPush(pGroup, &keyInfo); + } + } else { // mark all the empty groups, and remove it later taosArrayDestroy(pGroup); - - taosArrayRemove(groupList->pGroupList, j); - numOfGroups -= 1; - j -= 1; + taosArrayPush(emptyGroup, &j); } } @@ -3097,6 +3098,9 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0); } + taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), taosArrayGetSize(emptyGroup)); + taosArrayDestroy(emptyGroup); + groupList->numOfTables = totalNumOfTable; return window; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 85b15c0a4f..cb08944cbc 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -4,13 +4,14 @@ PROJECT(TDengine) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) + AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tutil ${SRC}) +ADD_LIBRARY(tutil ${SRC} tests/arrayTest.cpp) TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - # ADD_SUBDIRECTORY(tests) + ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 63cadf39a3..97aeaa61d4 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -52,6 +52,15 @@ void* taosArrayInit(size_t size, size_t elemSize); */ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); +/** + * + * @param pArray + * @param pData position array list + * @param numOfElems the number of removed position + */ +void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems); + + /** * add all element from the source array list into the destination * @param pArray diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index fe529edaac..d25d547be1 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -83,6 +83,47 @@ void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles) { return dst; } +void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems) { + assert(pArray != NULL && pData != NULL); + if (numOfElems <= 0) { + return; + } + + size_t size = taosArrayGetSize(pArray); + if (numOfElems >= size) { + taosArrayClear(pArray); + return; + } + + int32_t i = pData[0] + 1, j = 0; + while(i < size) { + if (j == numOfElems - 1) { + break; + } + + char* p = TARRAY_GET_ELEM(pArray, i); + if (i > pData[j] && i < pData[j + 1]) { + char* dst = TARRAY_GET_ELEM(pArray, i - (j + 1)); + memmove(dst, p, pArray->elemSize); + } else if (i == pData[j + 1]) { + j += 1; + } + + i += 1; + } + + assert(i == pData[numOfElems - 1] + 1); + + int32_t dstIndex = pData[numOfElems - 1] - numOfElems + 1; + int32_t srcIndex = pData[numOfElems - 1] + 1; + + char* dst = TARRAY_GET_ELEM(pArray, dstIndex); + char* src = TARRAY_GET_ELEM(pArray, srcIndex); + memmove(dst, src, pArray->elemSize * (pArray->size - numOfElems)); + + pArray->size -= numOfElems; +} + void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { return taosArrayAddBatch(pArray, pInput->pData, (int32_t) taosArrayGetSize(pInput)); } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index a88fee51d7..082b454bb5 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -681,143 +681,3 @@ static SSkipListNode *tSkipListPutImpl(SSkipList *pSkipList, void *pData, SSkipL return pNode; } - -// static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey, -// int32_t cond, SSkipListNode ***pRes) { -// pthread_rwlock_rdlock(&pSkipList->lock); -// SSkipListNode *p = pStartNode; -// int32_t numOfRes = 0; -// -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pEndKey->nType); -// while (p != NULL) { -// int32_t ret = filterComparFn(&p->key, pEndKey); -// if (ret > 0) { -// break; -// } -// -// if (ret < 0) { -// numOfRes++; -// p = p->pForward[0]; -// } else if (ret == 0) { -// if (cond == TSDB_RELATION_LESS_EQUAL) { -// numOfRes++; -// p = p->pForward[0]; -// } else { -// break; -// } -// } -// } -// -// (*pRes) = (SSkipListNode **)malloc(POINTER_BYTES * numOfRes); -// for (int32_t i = 0; i < numOfRes; ++i) { -// (*pRes)[i] = pStartNode; -// pStartNode = pStartNode->pForward[0]; -// } -// pthread_rwlock_unlock(&pSkipList->lock); -// -// return numOfRes; -//} -// -///* -// * maybe return the copy of SSkipListNode would be better -// */ -// int32_t tSkipListGets(SSkipList *pSkipList, SSkipListKey *pKey, SSkipListNode ***pRes) { -// (*pRes) = NULL; -// -// SSkipListNode *pNode = tSkipListGet(pSkipList, pKey); -// if (pNode == NULL) { -// return 0; -// } -// -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// // backward check if previous nodes are with the same value. -// SSkipListNode *pPrev = pNode->pBackward[0]; -// while ((pPrev != &pSkipList->pHead) && filterComparFn(&pPrev->key, pKey) == 0) { -// pPrev = pPrev->pBackward[0]; -// } -// -// return tSkipListEndParQuery(pSkipList, pPrev->pForward[0], &pNode->key, TSDB_RELATION_LESS_EQUAL, pRes); -//} -// -// static SSkipListNode *tSkipListParQuery(SSkipList *pSkipList, SSkipListKey *pKey, int32_t cond) { -// int32_t sLevel = pSkipList->level - 1; -// int32_t ret = -1; -// -// SSkipListNode *x = &pSkipList->pHead; -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// -// if (cond == TSDB_RELATION_GREATER_EQUAL || cond == TSDB_RELATION_GREATER) { -// for (int32_t i = sLevel; i >= 0; --i) { -// while (x->pForward[i] != NULL && (ret = filterComparFn(&x->pForward[i]->key, pKey)) < 0) { -// x = x->pForward[i]; -// } -// } -// -// // backward check if previous nodes are with the same value. -// if (cond == TSDB_RELATION_GREATER_EQUAL && ret == 0) { -// SSkipListNode *pNode = x->pForward[0]; -// while ((pNode->pBackward[0] != &pSkipList->pHead) && (filterComparFn(&pNode->pBackward[0]->key, pKey) == 0)) { -// pNode = pNode->pBackward[0]; -// } -// pthread_rwlock_unlock(&pSkipList->lock); -// return pNode; -// } -// -// if (ret > 0 || cond == TSDB_RELATION_GREATER_EQUAL) { -// pthread_rwlock_unlock(&pSkipList->lock); -// return x->pForward[0]; -// } else { // cond == TSDB_RELATION_GREATER && ret == 0 -// SSkipListNode *pn = x->pForward[0]; -// while (pn != NULL && filterComparFn(&pn->key, pKey) == 0) { -// pn = pn->pForward[0]; -// } -// pthread_rwlock_unlock(&pSkipList->lock); -// return pn; -// } -// } -// -// pthread_rwlock_unlock(&pSkipList->lock); -// return NULL; -//} -// -// -// static bool removeSupport(SSkipList *pSkipList, SSkipListNode **forward, SSkipListKey *pKey) { -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// if (filterComparFn(&forward[0]->pForward[0]->key, pKey) == 0) { -// SSkipListNode *p = forward[0]->pForward[0]; -// doRemove(pSkipList, p, forward); -// } else { // failed to find the node of specified value,abort -// return false; -// } -// -// // compress the minimum level of skip list -// while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { -// pSkipList->level -= 1; -// } -// -// return true; -//} -// -// bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) { -// SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; -// __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); -// -// pthread_rwlock_rdlock(&pSkipList->lock); -// -// SSkipListNode *x = &pSkipList->pHead; -// for (int32_t i = pSkipList->level - 1; i >= 0; --i) { -// while (x->pForward[i] != NULL && (filterComparFn(&x->pForward[i]->key, pKey) < 0)) { -// x = x->pForward[i]; -// } -// forward[i] = x; -// } -// -// bool ret = removeSupport(pSkipList, forward, pKey); -// pthread_rwlock_unlock(&pSkipList->lock); -// -// return ret; -//} diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 0a4791f6a9..970f1c23a9 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -5,10 +5,6 @@ #include "taos.h" #include "tcache.h" -namespace { -int32_t tsMaxMgmtConnections = 10000; -int32_t tsMaxMeterConnections = 200; -} // test cache TEST(testCase, client_cache_test) { const int32_t REFRESH_TIME_IN_SEC = 2; @@ -43,7 +39,7 @@ TEST(testCase, client_cache_test) { sleep(3); char* d = (char*) taosCacheAcquireByKey(tscMetaCache, key3, strlen(key3)); -// assert(d == NULL); + assert(d == NULL); char key5[] = "test5"; char data5[] = "data5kkkkk"; @@ -102,7 +98,7 @@ TEST(testCase, cache_resize_test) { char key[256] = {0}; char data[1024] = "abcdefghijk"; - int32_t len = strlen(data); +// int32_t len = strlen(data); uint64_t startTime = taosGetTimestampUs(); int32_t num = 10000; diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 2203ae8e4f..dfbe0f6716 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include "os.h" @@ -8,12 +9,14 @@ #include "tskiplist.h" #include "tutil.h" +#if 0 namespace { char* getkey(const void* data) { return (char*)(data); } void doubleSkipListTest() { - SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), 0, false, true, getkey); + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_DOUBLE, sizeof(double), + getKeyComparFunc(TSDB_DATA_TYPE_DOUBLE), false, getkey); double doubleVal[1000] = {0}; int32_t size = 20000; @@ -25,18 +28,15 @@ void doubleSkipListTest() { doubleVal[i] = i * 0.997; } - int32_t level = 0; - int32_t size = 0; +// int32_t level = 0; + size = 0; - tSkipListNewNodeInfo(pSkipList, &level, &size); - auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); - d->level = level; + // tSkipListNewNodeInfo(pSkipList, &level, &size); + // auto d = (SSkipListNode*)calloc(1, size + sizeof(double) * 2); + // d->level = level; - double* key = (double*)SL_GET_NODE_KEY(pSkipList, d); - key[0] = i * 0.997; - key[1] = i * 0.997; - - tSkipListPut(pSkipList, d); + double key = 0.997; + tSkipListPut(pSkipList, &key); } printf("the first level of skip list is:\n"); @@ -70,7 +70,8 @@ void doubleSkipListTest() { } void randKeyTest() { - SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), 0, false, true, getkey); + SSkipList* pSkipList = tSkipListCreate(10, TSDB_DATA_TYPE_INT, sizeof(int32_t), getKeyComparFunc(TSDB_DATA_TYPE_INT), + false, getkey); int32_t size = 200000; srand(time(NULL)); @@ -375,3 +376,5 @@ TEST(testCase, skiplist_test) { free(pKeys);*/ } + +#endif \ No newline at end of file diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c index 454860410b..e01da070af 100644 --- a/src/util/tests/trefTest.c +++ b/src/util/tests/trefTest.c @@ -14,18 +14,18 @@ typedef struct { int refNum; int steps; int rsetId; - int64_t rid; + int64_t*rid; void **p; } SRefSpace; void iterateRefs(int rsetId) { int count = 0; - void *p = taosIterateRef(rsetId, NULL); + void *p = taosIterateRef(rsetId, 0); while (p) { // process P count++; - p = taosIterateRef(rsetId, p); + p = taosIterateRef(rsetId, (int64_t) p); } printf(" %d ", count); @@ -34,7 +34,6 @@ void iterateRefs(int rsetId) { void *addRef(void *param) { SRefSpace *pSpace = (SRefSpace *)param; int id; - int64_t rid; for (int i=0; i < pSpace->steps; ++i) { printf("a"); @@ -51,8 +50,7 @@ void *addRef(void *param) { void *removeRef(void *param) { SRefSpace *pSpace = (SRefSpace *)param; - int id; - int64_t rid; + int id, code; for (int i=0; i < pSpace->steps; ++i) { printf("d"); @@ -71,16 +69,15 @@ void *removeRef(void *param) { void *acquireRelease(void *param) { SRefSpace *pSpace = (SRefSpace *)param; int id; - int64_t rid; for (int i=0; i < pSpace->steps; ++i) { printf("a"); id = random() % pSpace->refNum; - void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); + void *p = taosAcquireRef(pSpace->rsetId, (int64_t) pSpace->p[id]); if (p) { usleep(id % 5 + 1); - taosReleaseRef(pSpace->rsetId, pSpace->p[id]); + taosReleaseRef(pSpace->rsetId, (int64_t) pSpace->p[id]); } } @@ -103,6 +100,7 @@ void *openRefSpace(void *param) { } pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); + pSpace->rid = calloc(pSpace->refNum, sizeof(int64_t)); pthread_attr_t thattr; pthread_attr_init(&thattr); From 83dc3b31cfd73309225d1b2ada263f6a394f231e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 15:13:57 +0800 Subject: [PATCH 2/6] [td-225]add test files. --- src/tsdb/src/tsdbRead.c | 2 +- src/util/tests/arrayTest.cpp | 52 ++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 src/util/tests/arrayTest.cpp diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 924b5e24a8..6c16c72c2a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3095,7 +3095,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { // window does not being updated, so set the original if (window.skey == INT64_MAX && window.ekey == INT64_MIN) { window = TSWINDOW_INITIALIZER; - assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == 0); + assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == numOfGroups); } taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), taosArrayGetSize(emptyGroup)); diff --git a/src/util/tests/arrayTest.cpp b/src/util/tests/arrayTest.cpp new file mode 100644 index 0000000000..08983b6c06 --- /dev/null +++ b/src/util/tests/arrayTest.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include "tarray.h" + +namespace { + +static void remove_batch_test() { + SArray *pa = (SArray*) taosArrayInit(4, sizeof(int32_t)); + + for(int32_t i = 0; i < 20; ++i) { + int32_t a = i; + taosArrayPush(pa, &a); + } + + SArray* delList = (SArray*)taosArrayInit(4, sizeof(int32_t)); + taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); + EXPECT_EQ(taosArrayGetSize(pa), 20); + + int32_t a = 5; + taosArrayPush(delList, &a); + + taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); + EXPECT_EQ(taosArrayGetSize(pa), 19); + EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 6); + + taosArrayInsert(pa, 5, &a); + EXPECT_EQ(taosArrayGetSize(pa), 20); + EXPECT_EQ(*(int*)taosArrayGet(pa, 5), 5); + + taosArrayClear(delList); + + a = 6; + taosArrayPush(delList, &a); + + a = 9; + taosArrayPush(delList, &a); + + a = 14; + taosArrayPush(delList, &a); + taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList)); + EXPECT_EQ(taosArrayGetSize(pa), 17); +} +} // namespace + +TEST(arrayTest, array_list_test) { + + remove_batch_test(); + +} From b9a321e8c55249e74eabf1bfc211a303f1d86c79 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 15:18:27 +0800 Subject: [PATCH 3/6] [td-225]disable utiltest --- src/util/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index cb08944cbc..bcbadc26d5 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -11,7 +11,7 @@ TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - ADD_SUBDIRECTORY(tests) +# ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) From b4154d8ad6a1535b57987fbd5b5facdecc117560 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 15:45:48 +0800 Subject: [PATCH 4/6] [td-255]update cmake files. --- src/util/CMakeLists.txt | 2 +- src/util/tests/arrayTest.cpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index bcbadc26d5..27d9e11053 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -6,7 +6,7 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sync/inc) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/rmonotonic/inc) AUX_SOURCE_DIRECTORY(src SRC) -ADD_LIBRARY(tutil ${SRC} tests/arrayTest.cpp) +ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) diff --git a/src/util/tests/arrayTest.cpp b/src/util/tests/arrayTest.cpp index 08983b6c06..94b08ca6d7 100644 --- a/src/util/tests/arrayTest.cpp +++ b/src/util/tests/arrayTest.cpp @@ -46,7 +46,5 @@ static void remove_batch_test() { } // namespace TEST(arrayTest, array_list_test) { - remove_batch_test(); - } From 78f58194c44201a4dd362c7f0425133ecd1c20f0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 17:55:42 +0800 Subject: [PATCH 5/6] [td-5404]: fix memory leak in union query. --- src/client/src/tscSQLParser.c | 29 +++++++++++++++-- src/client/src/tscUtil.c | 57 +++++++++++++++------------------- src/util/CMakeLists.txt | 2 +- src/util/inc/tarray.h | 7 +++++ src/util/src/tarray.c | 40 ++++++++++++++++++++++++ tests/script/sh/prepare_udf.sh | 1 + 6 files changed, 101 insertions(+), 35 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 77c37239da..f82e1b62e9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -8005,6 +8005,28 @@ static void freeElem(void* p) { tfree(*(char**)p); } +int32_t tnameComparFn(const void* p1, const void* p2) { + SName* pn1 = (SName*)p1; + SName* pn2 = (SName*)p2; + + int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId)); + if (ret != 0) { + return ret > 0? 1:-1; + } else { + ret = strncmp(pn1->dbname, pn2->dbname, tListLen(pn1->dbname)); + if (ret != 0) { + return ret > 0? 1:-1; + } else { + ret = strncmp(pn1->tname, pn2->tname, tListLen(pn1->tname)); + if (ret != 0) { + return ret > 0? 1:-1; + } else { + return 0; + } + } + } +} + int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { SSqlCmd* pCmd = &pSql->cmd; @@ -8055,6 +8077,9 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { plist = taosArrayInit(4, POINTER_BYTES); pVgroupList = taosArrayInit(4, POINTER_BYTES); + taosArraySort(tableNameList, tnameComparFn); + taosArrayRemoveDuplicate(tableNameList, tnameComparFn, NULL); + size_t numOfTables = taosArrayGetSize(tableNameList); for (int32_t i = 0; i < numOfTables; ++i) { SName* pname = taosArrayGet(tableNameList, i); @@ -8075,8 +8100,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { continue; } } else if (pTableMeta->tableType == TSDB_SUPER_TABLE) { - // the vgroup list of a super table is not kept in local buffer, so here need retrieve it - // from the mnode each time + // the vgroup list of super table is not kept in local buffer, so here need retrieve it from the mnode each time char* t = strdup(name); taosArrayPush(pVgroupList, &t); } @@ -8103,6 +8127,7 @@ int32_t loadAllTableMeta(SSqlObj* pSql, struct SSqlInfo* pInfo) { if (pInfo->funcs) { funcSize = taosArrayGetSize(pInfo->funcs); } + if (funcSize > 0) { for (size_t i = 0; i < funcSize; ++i) { SStrToken* t = taosArrayGet(pInfo->funcs, i); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 6ae0249329..5b291c6e1f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1315,8 +1315,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeMeta) { return; } - SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); - + SQueryInfo* pQueryInfo = pCmd->pQueryInfo; while(pQueryInfo != NULL) { SQueryInfo* p = pQueryInfo->sibling; @@ -1596,7 +1595,7 @@ void freeUdfInfo(SUdfInfo* pUdfInfo) { taosCloseDll(pUdfInfo->handle); } - +// todo refactor void* tscDestroyUdfArrayList(SArray* pUdfList) { if (pUdfList == NULL) { return NULL; @@ -2986,6 +2985,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { int32_t tscAddQueryInfo(SSqlCmd* pCmd) { assert(pCmd != NULL); SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + if (pQueryInfo == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -3968,17 +3968,21 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { //backup the total number of result first int64_t num = pRes->numOfTotal + pRes->numOfClauseTotal; - // DON't free final since it may be recoreded and used later in APP TAOS_FIELD* finalBk = pRes->final; pRes->final = NULL; tscFreeSqlResult(pSql); + pRes->final = finalBk; - pRes->numOfTotal = num; - + + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + taos_free_result(pSql->pSubs[i]); + } + tfree(pSql->pSubs); pSql->subState.numOfSub = 0; + pSql->fp = fp; tscDebug("0x%"PRIx64" try data in the next subclause", pSql->self); @@ -4618,6 +4622,20 @@ int32_t nameComparFn(const void* n1, const void* n2) { } } +static void freeContent(void* p) { + char* ptr = *(char**)p; + tfree(ptr); +} + +static int32_t contCompare(const void* p1, const void* p2) { + int32_t ret = strcmp(p1, p2); + if (ret == 0) { + return 0; + } else { + return ret > 0 ? 1:-1; + } +} + int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t length, SArray* pNameArray) { SSqlCmd *pCmd = &pSql->cmd; @@ -4665,32 +4683,7 @@ int tscTransferTableNameList(SSqlObj *pSql, const char *pNameList, int32_t lengt } taosArraySort(pNameArray, nameComparFn); - - int32_t pos = 0; - for(int32_t i = 1; i < len; ++i) { - char** p1 = taosArrayGet(pNameArray, pos); - char** p2 = taosArrayGet(pNameArray, i); - - if (strcmp(*p1, *p2) == 0) { - // do nothing - } else { - if (pos + 1 != i) { - char* p = taosArrayGetP(pNameArray, pos + 1); - tfree(p); - taosArraySet(pNameArray, pos + 1, p2); - pos += 1; - } else { - pos += 1; - } - } - } - - for(int32_t i = pos + 1; i < pNameArray->size; ++i) { - char* p = taosArrayGetP(pNameArray, i); - tfree(p); - } - - pNameArray->size = pos + 1; + taosArrayRemoveDuplicate(pNameArray, contCompare, freeContent); return TSDB_CODE_SUCCESS; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 27d9e11053..44808b1025 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -11,7 +11,7 @@ TARGET_LINK_LIBRARIES(tutil pthread os lz4 z rmonotonic) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) -# ADD_SUBDIRECTORY(tests) + ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/tarray.h b/src/util/inc/tarray.h index 97aeaa61d4..2da74eac82 100644 --- a/src/util/inc/tarray.h +++ b/src/util/inc/tarray.h @@ -60,6 +60,13 @@ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); */ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems); +/** + * + * @param pArray + * @param comparFn + * @param fp + */ +void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)); /** * add all element from the source array list into the destination diff --git a/src/util/src/tarray.c b/src/util/src/tarray.c index d25d547be1..d0d126c1e4 100644 --- a/src/util/src/tarray.c +++ b/src/util/src/tarray.c @@ -124,6 +124,46 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle pArray->size -= numOfElems; } +void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)) { + assert(pArray); + + size_t size = pArray->size; + if (size <= 1) { + return; + } + + int32_t pos = 0; + for(int32_t i = 1; i < size; ++i) { + char* p1 = taosArrayGet(pArray, pos); + char* p2 = taosArrayGet(pArray, i); + + if (comparFn(p1, p2) == 0) { + // do nothing + } else { + if (pos + 1 != i) { + void* p = taosArrayGet(pArray, pos + 1); + if (fp != NULL) { + fp(p); + } + + taosArraySet(pArray, pos + 1, p2); + pos += 1; + } else { + pos += 1; + } + } + } + + if (fp != NULL) { + for(int32_t i = pos + 1; i < pArray->size; ++i) { + void* p = taosArrayGet(pArray, i); + fp(p); + } + } + + pArray->size = pos + 1; +} + void* taosArrayAddAll(SArray* pArray, const SArray* pInput) { return taosArrayAddBatch(pArray, pInput->pData, (int32_t) taosArrayGetSize(pInput)); } diff --git a/tests/script/sh/prepare_udf.sh b/tests/script/sh/prepare_udf.sh index ecfc53660e..a856b96c98 100644 --- a/tests/script/sh/prepare_udf.sh +++ b/tests/script/sh/prepare_udf.sh @@ -5,6 +5,7 @@ echo -n "" > /tmp/empty dd if=/dev/zero bs=3584 of=/tmp/big count=1 rm -rf /tmp/sum_double.so /tmp/add_one.so +touch /tmp/normal gcc -g -O0 -fPIC -shared sh/sum_double.c -o /tmp/sum_double.so gcc -g -O0 -fPIC -shared sh/add_one.c -o /tmp/add_one.so From 5bea3eabe55af4b9630e6b0d4d4d8587f1251fa1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 19 Jul 2021 22:58:54 +0800 Subject: [PATCH 6/6] [td-225]fix compiler error. --- src/tsdb/src/tsdbRead.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 6c16c72c2a..a68a170554 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -3098,7 +3098,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { assert(totalNumOfTable == 0 && taosArrayGetSize(groupList->pGroupList) == numOfGroups); } - taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), taosArrayGetSize(emptyGroup)); + taosArrayRemoveBatch(groupList->pGroupList, TARRAY_GET_START(emptyGroup), (int32_t) taosArrayGetSize(emptyGroup)); taosArrayDestroy(emptyGroup); groupList->numOfTables = totalNumOfTable;