From 01e5eaea12a070aa6b6aa72d6687790e508da1be Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 28 Mar 2024 08:19:58 +0800 Subject: [PATCH 001/103] enh: remove obsolete code --- source/os/src/osThread.c | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 0dd5374cf0..3e2d4110eb 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -17,15 +17,6 @@ #include #include "os.h" -#ifdef WINDOWS -#define THREAD_PTR_CHECK(p) \ - do { \ - if (!(p) || !(*(p))) return 0; \ - } while (0); -#else -#define THREAD_PTR_CHECK(p) -#endif - int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void *(*start)(void *), void *arg) { return pthread_create(tid, attr, start, arg); } @@ -126,7 +117,6 @@ int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { } return 0; #else - THREAD_PTR_CHECK(mutex) return pthread_cond_wait(cond, mutex); #endif } @@ -140,7 +130,6 @@ int32_t taosThreadCondTimedWait(TdThreadCond *cond, TdThreadMutex *mutex, const } return EINVAL; #else - THREAD_PTR_CHECK(mutex) return pthread_cond_timedwait(cond, mutex, abstime); #endif } @@ -201,7 +190,6 @@ int32_t taosThreadKeyDelete(TdThreadKey key) { return pthread_key_delete(key); } int32_t taosThreadKill(TdThread thread, int32_t sig) { return pthread_kill(thread, sig); } // int32_t taosThreadMutexConsistent(TdThreadMutex* mutex) { -// THREAD_PTR_CHECK(mutex) // return pthread_mutex_consistent(mutex); // } @@ -210,7 +198,6 @@ int32_t taosThreadMutexDestroy(TdThreadMutex *mutex) { DeleteCriticalSection(mutex); return 0; #else - THREAD_PTR_CHECK(mutex) return pthread_mutex_destroy(mutex); #endif } @@ -234,7 +221,6 @@ int32_t taosThreadMutexLock(TdThreadMutex *mutex) { EnterCriticalSection(mutex); return 0; #else - THREAD_PTR_CHECK(mutex) return pthread_mutex_lock(mutex); #endif } @@ -248,7 +234,6 @@ int32_t taosThreadMutexTryLock(TdThreadMutex *mutex) { if (TryEnterCriticalSection(mutex)) return 0; return EBUSY; #else - THREAD_PTR_CHECK(mutex) return pthread_mutex_trylock(mutex); #endif } @@ -258,7 +243,6 @@ int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { LeaveCriticalSection(mutex); return 0; #else - THREAD_PTR_CHECK(mutex) return pthread_mutex_unlock(mutex); #endif } @@ -451,7 +435,6 @@ int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sc int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { return pthread_setspecific(key, value); } int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { - THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_destroy((pthread_mutex_t *)lock); #else @@ -470,7 +453,6 @@ int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int32_t pshared) { } int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { - THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_lock((pthread_mutex_t *)lock); #else @@ -479,7 +461,6 @@ int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { } int32_t taosThreadSpinTrylock(TdThreadSpinlock *lock) { - THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_trylock((pthread_mutex_t *)lock); #else @@ -488,7 +469,6 @@ int32_t taosThreadSpinTrylock(TdThreadSpinlock *lock) { } int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { - THREAD_PTR_CHECK(lock) #ifdef TD_USE_SPINLOCK_AS_MUTEX return pthread_mutex_unlock((pthread_mutex_t *)lock); #else From 03f1250b933ef351f59a071856d0ede38c91f2c9 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 18 Apr 2024 02:57:53 +0000 Subject: [PATCH 002/103] limit index bg thread --- source/libs/index/src/index.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index f41b5525f3..d7e5d87e80 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -25,8 +25,10 @@ #include "tref.h" #include "tsched.h" -#define INDEX_NUM_OF_THREADS 5 -#define INDEX_QUEUE_SIZE 200 +#define INDEX_NUM_OF_THREADS 5 +#define INDEX_MAX_NUM_OF_THREADS 10 + +#define INDEX_QUEUE_SIZE 200 #define INDEX_DATA_BOOL_NULL 0x02 #define INDEX_DATA_TINYINT_NULL 0x80 @@ -61,6 +63,7 @@ static void indexDestroy(void* sIdx); void indexInit(int32_t threadNum) { indexThreads = threadNum; if (indexThreads <= 1) indexThreads = INDEX_NUM_OF_THREADS; + if (indexThreads >= INDEX_MAX_NUM_OF_THREADS) indexThreads = INDEX_MAX_NUM_OF_THREADS; } void indexEnvInit() { // refactor later From 5a72630b3de683c268a405eb2d415588bae4b2dd Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 26 Apr 2024 09:40:29 +0800 Subject: [PATCH 003/103] fix: select failed as using table alias name --- source/libs/parser/src/parTranslater.c | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 8f77f0dedf..b513d10a3e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3155,6 +3155,31 @@ static bool fromSingleTable(SNode* table) { return false; } +static bool IsEqualTbNameFuncNode(SSelectStmt* pSelect, SNode* pFunc1, SNode* pFunc2) { + if (isTbnameFuction(pFunc1) && isTbnameFuction(pFunc2)) { + SValueNode* pVal1 = (SValueNode*)nodesListGetNode(((SFunctionNode*)pFunc1)->pParameterList, 0); + SValueNode* pVal2 = (SValueNode*)nodesListGetNode(((SFunctionNode*)pFunc1)->pParameterList, 0); + if (!pVal1 && !pVal2) { + return true; + } else if (pVal1 && pVal2) { + return strcmp(pVal1->literal, pVal2->literal) == 0; + } + + if (pSelect->pFromTable && + (pSelect->pFromTable->type == QUERY_NODE_REAL_TABLE || pSelect->pFromTable->type == QUERY_NODE_TEMP_TABLE)) { + STableNode* pTable = (STableNode*)pSelect->pFromTable; + if (pVal1) { + return strcmp(pTable->tableAlias, pVal1->literal) == 0; + } else if (!pVal1) { + return strcmp(pTable->tableAlias, pVal2->literal) == 0; + } + } else { + return false; + } + } + return false; +} + static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { STranslateContext* pCxt = (STranslateContext*)pContext; SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; @@ -3174,6 +3199,9 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } + if (IsEqualTbNameFuncNode(pSelect, pActualNode, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt, pNode); + } } SNode* pPartKey = NULL; bool partionByTbname = hasTbnameFunction(pSelect->pPartitionByList); @@ -3185,6 +3213,9 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { return rewriteExprToGroupKeyFunc(pCxt, pNode); } + if (IsEqualTbNameFuncNode(pSelect, pPartKey, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt, pNode); + } } if (NULL != pSelect->pWindow && QUERY_NODE_STATE_WINDOW == nodeType(pSelect->pWindow)) { if (nodesEqualNode(((SStateWindowNode*)pSelect->pWindow)->pExpr, *pNode)) { From b314ca8509a5377d98eb36e427b3bff36f324d59 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Sun, 28 Apr 2024 09:45:54 +0800 Subject: [PATCH 004/103] fix: rewrite expr groupby --- source/libs/parser/src/parTranslater.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b513d10a3e..167b4bd43c 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3193,7 +3193,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { FOREACH(pGroupNode, getGroupByList(pCxt)) { SNode* pActualNode = getGroupByNode(pGroupNode); if (nodesEqualNode(pActualNode, *pNode)) { - return DEAL_RES_IGNORE_CHILD; + return rewriteExprToGroupKeyFunc(pCxt, pNode); } if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { From 446fc9f859a1e496c34ebe8f155b8dafca344e87 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 7 May 2024 15:57:49 +0800 Subject: [PATCH 005/103] group by alias tablename --- source/libs/parser/src/parTranslater.c | 2 +- source/libs/planner/src/planOptimizer.c | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 167b4bd43c..b513d10a3e 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3193,7 +3193,7 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { FOREACH(pGroupNode, getGroupByList(pCxt)) { SNode* pActualNode = getGroupByNode(pGroupNode); if (nodesEqualNode(pActualNode, *pNode)) { - return rewriteExprToGroupKeyFunc(pCxt, pNode); + return DEAL_RES_IGNORE_CHILD; } if (isTbnameFuction(pActualNode) && QUERY_NODE_COLUMN == nodeType(*pNode) && ((SColumnNode*)*pNode)->colType == COLUMN_TYPE_TAG) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index da39228a62..f998e342ab 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2835,7 +2835,9 @@ static SNode* partTagsCreateWrapperFunc(const char* pFuncName, SNode* pNode) { } snprintf(pFunc->functionName, sizeof(pFunc->functionName), "%s", pFuncName); - if (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { + if ((QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) || + (QUERY_NODE_COLUMN == nodeType(pNode) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType && + ((SColumnNode*)pNode)->tableAlias[0] != '\0')){ SColumnNode* pCol = (SColumnNode*)pNode; partTagsSetAlias(pFunc->node.aliasName, pCol->tableAlias, pCol->colName); } else { From 0d86361a94300e4a4c0f62ccf48ef4f945f87723 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 7 May 2024 17:21:10 +0800 Subject: [PATCH 006/103] add test case --- .../2-query/agg_group_AlwaysReturnValue.py | 34 +++++++++++++++++-- .../2-query/agg_group_NotReturnValue.py | 6 +++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py index 9ed0e3fc4a..6a330197c7 100755 --- a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py +++ b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py @@ -640,6 +640,15 @@ class TDTestCase(TDTestCase): sql = f"select tbname,AGG(COLUMN) from {dbname}.stable_1 group by tbname order by tbname,count(*) " self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + sql = f"select tbname,AGG(COLUMN) from {dbname}.stable_1 a group by a.tbname order by a.tbname,count(*) " + self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select a.tbname,AGG(COLUMN) from {dbname}.stable_1 a group by a.tbname order by a.tbname,count(*) " + self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select a.tbname,AGG(COLUMN) from {dbname}.stable_1 a group by tbname order by a.tbname,count(*) " + self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + sql1 = f"select * from ({sql})" self.data_check_tbname(sql1,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') @@ -647,7 +656,16 @@ class TDTestCase(TDTestCase): self.data_check_tbname(sql2,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') sql = f"select tbname,AGG(COLUMN) from {dbname}.stable_1 where ts is null group by tbname order by tbname,count(*) " - self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select tbname,AGG(COLUMN) from {dbname}.stable_1 a where ts is null group by a.tbname order by a.tbname,count(*) " + self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select a.tbname,AGG(COLUMN) from {dbname}.stable_1 a where ts is null group by a.tbname order by a.tbname,count(*) " + self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select a.tbname,AGG(COLUMN) from {dbname}.stable_1 a where ts is null group by a.tbname order by tbname,count(*) " + self.data_check_tbname(sql,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') sql1 = f"select * from ({sql})" self.data_check_tbname(sql1,'NULL','NULL',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') @@ -1011,7 +1029,19 @@ class TDTestCase(TDTestCase): #union all sql = f"select tbname tb,AGG(COLUMN) from {dbname}.stable_1 group by tbname order by tbname,AGG(COLUMN),count(*)" sql = f"({sql}) union all ({sql}) order by tb" - self.data_check_tbname(sql,'AGG24','AGG24',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + self.data_check_tbname(sql,'AGG24','AGG24',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select a.tbname tb,AGG(COLUMN) from {dbname}.stable_1 a group by a.tbname order by tbname,AGG(COLUMN),count(*)" + sql = f"({sql}) union all ({sql}) order by tb" + self.data_check_tbname(sql,'AGG24','AGG24',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select tbname tb,AGG(COLUMN) from {dbname}.stable_1 a group by a.tbname order by a.tbname,AGG(COLUMN),count(*)" + sql = f"({sql}) union all ({sql}) order by tb" + self.data_check_tbname(sql,'AGG24','AGG24',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select a.tbname tb,AGG(COLUMN) from {dbname}.stable_1 a group by tbname order by a.tbname,AGG(COLUMN),count(*)" + sql = f"({sql}) union all ({sql}) order by tb" + self.data_check_tbname(sql,'AGG24','AGG24',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') sql1 = f"select * from ({sql})" self.data_check_tbname(sql1,'AGG24','AGG24',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') diff --git a/tests/system-test/2-query/agg_group_NotReturnValue.py b/tests/system-test/2-query/agg_group_NotReturnValue.py index 73a8fe04c3..83f0acd362 100755 --- a/tests/system-test/2-query/agg_group_NotReturnValue.py +++ b/tests/system-test/2-query/agg_group_NotReturnValue.py @@ -382,7 +382,11 @@ class TDTestCase(TDTestCase): #union all sql = f"select tbname tb,AGG(COLUMN) from {dbname}.stable_1 group by tbname order by tbname " sql = f"({sql}) union all ({sql}) order by tb" - self.data_check_tbname(sql,'HAVING>04','HAVING>04',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + self.data_check_tbname(sql,'HAVING>04','HAVING>04',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') + + sql = f"select a.tbname tb,AGG(COLUMN) from {dbname}.stable_1 a group by a.tbname order by tbname " + sql = f"({sql}) union all ({sql}) order by tb" + self.data_check_tbname(sql,'HAVING>04','HAVING>04',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') sql1 = f"select * from ({sql})" self.data_check_tbname(sql1,'HAVING>04','HAVING>04',f'{base_fun}',f'{replace_fun}',f'{base_column}',f'{replace_column}') From 5a75c38897525277dca001a34ea7ef4e683454e9 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 8 May 2024 23:28:04 +0800 Subject: [PATCH 007/103] test --- source/libs/parser/src/parTranslater.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index b513d10a3e..9d0c8034be 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3168,11 +3168,7 @@ static bool IsEqualTbNameFuncNode(SSelectStmt* pSelect, SNode* pFunc1, SNode* pF if (pSelect->pFromTable && (pSelect->pFromTable->type == QUERY_NODE_REAL_TABLE || pSelect->pFromTable->type == QUERY_NODE_TEMP_TABLE)) { STableNode* pTable = (STableNode*)pSelect->pFromTable; - if (pVal1) { - return strcmp(pTable->tableAlias, pVal1->literal) == 0; - } else if (!pVal1) { - return strcmp(pTable->tableAlias, pVal2->literal) == 0; - } + return true; } else { return false; } From d538bf304804135dd90f284d633fe0a38c217988 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 13 May 2024 00:25:40 +0800 Subject: [PATCH 008/103] join test case --- .../2-query/agg_group_AlwaysReturnValue.py | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py index 6a330197c7..26ae390567 100755 --- a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py +++ b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py @@ -1602,14 +1602,54 @@ class TDTestCase(TDTestCase): tdSql.execute('alter stable stable_1 drop column q_binary5;') tdSql.execute('alter stable stable_1 drop column q_nchar4;') tdSql.execute('alter stable stable_1 drop column q_binary4;') - + + def testTBNameUseJoin(self): + tdSql.execute('CREATE STABLE `meter1` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT)') + tdSql.execute('CREATE STABLE `meter2` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT)') + + tdSql.execute('CREATE TABLE `d1` USING `meter1` (`t1`) TAGS (1)') + tdSql.execute('CREATE TABLE `d2` USING `meter1` (`t1`) TAGS (2)') + tdSql.execute('CREATE TABLE `d21` USING `meter2` (`t1`) TAGS (21)') + tdSql.execute('CREATE TABLE `d22` USING `meter2` (`t1`) TAGS (22)') + + time.sleep(1) + tdSql.query('select tbname,count(*) from d2') + tdSql.checkData(0, 1, 0) + + tdSql.query('select b.tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') + tdSql.checkData(0, 0, 'd2') + tdSql.checkData(0, 1, 0) + tdSql.query('select meter1.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') + tdSql.checkData(0, 0, 'd1') + tdSql.checkData(1, 0, 'd2') + tdSql.checkData(0, 1, 0) + tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter2.tbname order by meter2.tbname') + tdSql.checkData(0, 0, 'd21') + tdSql.checkData(1, 0, 'd22') + tdSql.checkData(0, 1, 0) + tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by meter2.tbname order by meter2.tbname') + tdSql.checkData(0, 0, 'd21') + tdSql.checkData(1, 0, 'd22') + tdSql.checkData(0, 1, 0) + tdSql.query('select m2.tbname, count(*) from meter1 m1, meter2 m2 where m1.ts = m2.ts partition by m2.tbname order by m2.tbname') + tdSql.checkData(0, 0, 'd21') + tdSql.checkData(1, 0, 'd22') + tdSql.checkData(0, 1, 0) + + tdSql.error('select tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') + tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') + tdSql.error('select tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter2.tbname order by meter2.tbname') + tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by meter2.tbname order by meter.tbname') + tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by tbname order by meter2.tbname') + tdSql.error('select m2.tbname, count(*) from meter1 m1, meter2 m2 where meter1.ts = meter2.ts partition by m2.tbname order by meter2.tbname') + def run(self): tdSql.prepare() startTime = time.time() # self.create_tables() - # self.insert_data() + # self.insert_data() self.dropandcreateDB_random("nested", 1) self.modify_tables() @@ -1620,6 +1660,8 @@ class TDTestCase(TDTestCase): self.tbname_agg_all() + self.testTBNameUseJoin() + endTime = time.time() print("total time %ds" % (endTime - startTime)) From e3f5cc9af1e333970a95200813457d31fcabdc6d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Mon, 13 May 2024 13:32:41 +0800 Subject: [PATCH 009/103] skip join case --- .../2-query/agg_group_AlwaysReturnValue.py | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py index 26ae390567..f7c82fc859 100755 --- a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py +++ b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py @@ -1602,20 +1602,20 @@ class TDTestCase(TDTestCase): tdSql.execute('alter stable stable_1 drop column q_binary5;') tdSql.execute('alter stable stable_1 drop column q_nchar4;') tdSql.execute('alter stable stable_1 drop column q_binary4;') - + def testTBNameUseJoin(self): tdSql.execute('CREATE STABLE `meter1` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT)') tdSql.execute('CREATE STABLE `meter2` (`ts` TIMESTAMP, `v1` INT) TAGS (`t1` INT)') - + tdSql.execute('CREATE TABLE `d1` USING `meter1` (`t1`) TAGS (1)') tdSql.execute('CREATE TABLE `d2` USING `meter1` (`t1`) TAGS (2)') tdSql.execute('CREATE TABLE `d21` USING `meter2` (`t1`) TAGS (21)') tdSql.execute('CREATE TABLE `d22` USING `meter2` (`t1`) TAGS (22)') - + time.sleep(1) tdSql.query('select tbname,count(*) from d2') tdSql.checkData(0, 1, 0) - + tdSql.query('select b.tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') tdSql.checkData(0, 0, 'd2') tdSql.checkData(0, 1, 0) @@ -1635,23 +1635,46 @@ class TDTestCase(TDTestCase): tdSql.checkData(0, 0, 'd21') tdSql.checkData(1, 0, 'd22') tdSql.checkData(0, 1, 0) - + + tdSql.execute('insert into `d1` VALUES (now, 1) `d21` VALUES (now, 21)') + tdSql.execute('insert into `d1` VALUES (now, 2) `d21` VALUES (now, 22)') + tdSql.execute('insert into `d1` VALUES (now, 3) `d21` VALUES (now, 32)') + + # tdSql.query('select b.tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') + # tdSql.checkData(0, 0, 'd2') + + tdSql.query('select meter1.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') + tdSql.checkData(0, 0, 'd1') + tdSql.checkData(1, 0, 'd2') + tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter2.tbname order by meter2.tbname') + tdSql.checkData(0, 0, 'd21') + tdSql.checkData(1, 0, 'd22') + tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by meter2.tbname order by meter2.tbname') + tdSql.checkData(0, 0, 'd21') + tdSql.checkData(1, 0, 'd22') + tdSql.query('select m2.tbname, count(*) from meter1 m1, meter2 m2 where m1.ts = m2.ts partition by m2.tbname order by m2.tbname') + tdSql.checkData(0, 0, 'd21') + tdSql.checkData(1, 0, 'd22') + tdSql.error('select tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') - tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') + # tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') tdSql.error('select tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter2.tbname order by meter2.tbname') tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by meter2.tbname order by meter.tbname') tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by tbname order by meter2.tbname') tdSql.error('select m2.tbname, count(*) from meter1 m1, meter2 m2 where meter1.ts = meter2.ts partition by m2.tbname order by meter2.tbname') - + def run(self): tdSql.prepare() startTime = time.time() # self.create_tables() - # self.insert_data() - + # self.insert_data() + + self.testTBNameUseJoin() self.dropandcreateDB_random("nested", 1) + #self.testTBNameUseJoin() + self.modify_tables() for i in range(1): @@ -1660,7 +1683,7 @@ class TDTestCase(TDTestCase): self.tbname_agg_all() - self.testTBNameUseJoin() + endTime = time.time() print("total time %ds" % (endTime - startTime)) From a6217eec0323abdffb33de188e532cf8f3d7605a Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Fri, 31 May 2024 18:58:35 +0800 Subject: [PATCH 010/103] partition interval and limimt, dataload error --- source/libs/executor/src/groupoperator.c | 148 ++++++++++++++--------- source/libs/executor/src/scanoperator.c | 1 + 2 files changed, 94 insertions(+), 55 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9a31e993b2..c6a7804b9c 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -53,6 +53,8 @@ typedef struct SPartitionOperatorInfo { int32_t rowCapacity; // maximum number of rows for each buffer page int32_t* columnOffset; // start position for each column data SArray* sortedGroupArray; // SDataGroupInfo sorted by group id + SArray* blockForNotLoaded; // SSDataBlock that data is not loaded + int32_t offsetForNotLoaded;// read offset for SSDataBlock that data is not loaded int32_t groupIndex; // group index int32_t pageIndex; // page index of current group SExprSupp scalarSup; @@ -584,71 +586,86 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { pGroupInfo->groupId = calcGroupId(pInfo->keyBuf, len); } - // number of rows - int32_t* rows = (int32_t*)pPage; + if (pBlock->info.dataLoad) { + // number of rows + int32_t* rows = (int32_t*)pPage; - size_t numOfCols = pOperator->exprSupp.numOfExprs; - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; - int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + size_t numOfCols = pOperator->exprSupp.numOfExprs; + for (int32_t i = 0; i < numOfCols; ++i) { + SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - int32_t bytes = pColInfoData->info.bytes; - int32_t startOffset = pInfo->columnOffset[i]; + int32_t bytes = pColInfoData->info.bytes; + int32_t startOffset = pInfo->columnOffset[i]; - int32_t* columnLen = NULL; - int32_t contentLen = 0; + int32_t* columnLen = NULL; + int32_t contentLen = 0; - if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - int32_t* offset = (int32_t*)((char*)pPage + startOffset); - columnLen = (int32_t*)((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); - char* data = (char*)((char*)columnLen + sizeof(int32_t)); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + int32_t* offset = (int32_t*)((char*)pPage + startOffset); + columnLen = (int32_t*)((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); + char* data = (char*)((char*)columnLen + sizeof(int32_t)); - if (colDataIsNull_s(pColInfoData, j)) { - offset[(*rows)] = -1; - contentLen = 0; - } else if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { - offset[*rows] = (*columnLen); - char* src = colDataGetData(pColInfoData, j); - int32_t dataLen = getJsonValueLen(src); + if (colDataIsNull_s(pColInfoData, j)) { + offset[(*rows)] = -1; + contentLen = 0; + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { + offset[*rows] = (*columnLen); + char* src = colDataGetData(pColInfoData, j); + int32_t dataLen = getJsonValueLen(src); - memcpy(data + (*columnLen), src, dataLen); - int32_t v = (data + (*columnLen) + dataLen - (char*)pPage); - ASSERT(v > 0); + memcpy(data + (*columnLen), src, dataLen); + int32_t v = (data + (*columnLen) + dataLen - (char*)pPage); + ASSERT(v > 0); - contentLen = dataLen; + contentLen = dataLen; + } else { + offset[*rows] = (*columnLen); + char* src = colDataGetData(pColInfoData, j); + memcpy(data + (*columnLen), src, varDataTLen(src)); + int32_t v = (data + (*columnLen) + varDataTLen(src) - (char*)pPage); + ASSERT(v > 0); + + contentLen = varDataTLen(src); + } } else { - offset[*rows] = (*columnLen); - char* src = colDataGetData(pColInfoData, j); - memcpy(data + (*columnLen), src, varDataTLen(src)); - int32_t v = (data + (*columnLen) + varDataTLen(src) - (char*)pPage); - ASSERT(v > 0); + char* bitmap = (char*)pPage + startOffset; + columnLen = (int32_t*)((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); + char* data = (char*)columnLen + sizeof(int32_t); - contentLen = varDataTLen(src); + bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); + if (isNull) { + colDataSetNull_f(bitmap, (*rows)); + } else { + memcpy(data + (*columnLen), colDataGetData(pColInfoData, j), bytes); + ASSERT((data + (*columnLen) + bytes - (char*)pPage) <= getBufPageSize(pInfo->pBuf)); + } + contentLen = bytes; } - } else { - char* bitmap = (char*)pPage + startOffset; - columnLen = (int32_t*)((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); - char* data = (char*)columnLen + sizeof(int32_t); - bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); - if (isNull) { - colDataSetNull_f(bitmap, (*rows)); - } else { - memcpy(data + (*columnLen), colDataGetData(pColInfoData, j), bytes); - ASSERT((data + (*columnLen) + bytes - (char*)pPage) <= getBufPageSize(pInfo->pBuf)); - } - contentLen = bytes; + (*columnLen) += contentLen; } - (*columnLen) += contentLen; + (*rows) += 1; + + setBufPageDirty(pPage, true); + releaseBufPage(pInfo->pBuf, pPage); + } else { + SSDataBlock* dataNotLoadBlock = createOneDataBlock(pBlock, true); + if (dataNotLoadBlock == NULL) { + T_LONG_JMP(pTaskInfo->env, terrno); + } + if (pInfo->blockForNotLoaded == NULL) { + pInfo->blockForNotLoaded = taosArrayInit(1, sizeof(SSDataBlock)); + pInfo->offsetForNotLoaded = 0; + } + dataNotLoadBlock->info.id.groupId = pGroupInfo->groupId; + dataNotLoadBlock->info.dataLoad = 0; + taosArrayInsert(pInfo->blockForNotLoaded, pInfo->blockForNotLoaded->size, &dataNotLoadBlock); + break; } - - (*rows) += 1; - - setBufPageDirty(pPage, true); - releaseBufPage(pInfo->pBuf, pPage); } } @@ -734,6 +751,14 @@ static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { } taosArrayClear(pInfo->sortedGroupArray); clearDiskbasedBuf(pInfo->pBuf); + if (pInfo->blockForNotLoaded) { + for (int32_t i = 0; i < pInfo->blockForNotLoaded->size; i++) { + SSDataBlock** pBlock = taosArrayGet(pInfo->blockForNotLoaded, i); + blockDataDestroy(*pBlock); + } + taosArrayClear(pInfo->blockForNotLoaded); + pInfo->offsetForNotLoaded = 0; + } } static int compareDataGroupInfo(const void* group1, const void* group2) { @@ -747,6 +772,21 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1; } +static SSDataBlock* buildPartitionResultForNotLoadBlock(SOperatorInfo* pOperator) { + SPartitionOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + if (pInfo->blockForNotLoaded && pInfo->offsetForNotLoaded < pInfo->blockForNotLoaded->size) { + SSDataBlock** pBlock = taosArrayGet(pInfo->blockForNotLoaded, pInfo->offsetForNotLoaded); + pInfo->offsetForNotLoaded++; + return *pBlock; + } else { + setOperatorCompleted(pOperator); + clearPartitionOperator(pInfo); + return NULL; + } +} + static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -757,12 +797,10 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { // try next group data - ++pInfo->groupIndex; - if (pInfo->groupIndex >= taosArrayGetSize(pInfo->sortedGroupArray)) { - setOperatorCompleted(pOperator); - clearPartitionOperator(pInfo); - return NULL; + if (pInfo->groupIndex + 1 >= taosArrayGetSize(pInfo->sortedGroupArray)) { + return buildPartitionResultForNotLoadBlock(pOperator); } + ++pInfo->groupIndex; pGroupInfo = taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex); pInfo->pageIndex = 0; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 110aabf9b1..fb86e73a64 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -769,6 +769,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pTableScanInfo->pResBlock; bool hasNext = false; int32_t code = TSDB_CODE_SUCCESS; + pBlock->info.dataLoad = false; int64_t st = taosGetTimestampUs(); From c42e627a41eda9e2d79052ed13010c58408409cb Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 4 Jun 2024 11:39:47 +0800 Subject: [PATCH 011/103] test --- include/common/tcommon.h | 2 +- include/common/tdatablock.h | 1 + include/libs/scalar/filter.h | 2 +- source/common/src/tdatablock.c | 58 ++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbRead2.c | 8 +-- source/libs/executor/src/executil.c | 4 +- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/groupoperator.c | 6 +- source/libs/executor/src/scanoperator.c | 2 +- source/libs/executor/src/timewindowoperator.c | 2 +- source/libs/executor/src/tsort.c | 8 +-- source/libs/scalar/src/filter.c | 16 ++--- 12 files changed, 84 insertions(+), 27 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index d28477ae40..6d8c1b90f8 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -239,7 +239,7 @@ typedef struct SDataBlockInfo { } SDataBlockInfo; typedef struct SSDataBlock { - SColumnDataAgg** pBlockAgg; + SColumnDataAgg* pBlockAgg; SArray* pDataBlock; // SArray SDataBlockInfo info; } SSDataBlock; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 197fa125f5..58fb3a6f4c 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -256,6 +256,7 @@ SSDataBlock* createDataBlock(); void* blockDataDestroy(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +SSDataBlock* createBlockDataNotLoaded(SSDataBlock* pDataBlock); SSDataBlock* createSpecialDataBlock(EStreamType type); SSDataBlock* blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx); diff --git a/include/libs/scalar/filter.h b/include/libs/scalar/filter.h index c1ce1e6fd8..750179ee3b 100644 --- a/include/libs/scalar/filter.h +++ b/include/libs/scalar/filter.h @@ -58,7 +58,7 @@ extern int32_t filterGetTimeRange(SNode *pNode, STimeWindow *win, bool *isStrict extern int32_t filterConverNcharColumns(SFilterInfo *pFilterInfo, int32_t rows, bool *gotNchar); extern int32_t filterFreeNcharColumns(SFilterInfo *pFilterInfo); extern void filterFreeInfo(SFilterInfo *info); -extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg, int32_t numOfCols, int32_t numOfRows); +extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pColsAgg, int32_t numOfCols, int32_t numOfRows); /* condition split interface */ int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond, diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index ac4811fb1b..7b4d20238c 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -848,7 +848,7 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 if (pBlock->pBlockAgg == NULL) { isNull = colDataIsNull_s(pColData, j); } else { - isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg[i]); + isNull = colDataIsNull(pColData, pBlock->info.rows, j, &pBlock->pBlockAgg[i]); } if (isNull) { @@ -1733,6 +1733,62 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return pDstBlock; } +SSDataBlock* createBlockDataNotLoaded(SSDataBlock* pDataBlock) { + if (pDataBlock == NULL) { + return NULL; + } + + SSDataBlock* pDstBlock = createDataBlock(); + pDstBlock->info = pDataBlock->info; + + pDstBlock->info.rows = 0; + pDstBlock->info.capacity = 0; + pDstBlock->info.rowSize = 0; + pDstBlock->info.id = pDataBlock->info.id; + pDstBlock->info.blankFill = pDataBlock->info.blankFill; + + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; + blockDataAppendColInfo(pDstBlock, &colInfo); + } + + copyPkVal(&pDstBlock->info, &pDataBlock->info); + + int32_t code = blockDataEnsureCapacity(pDstBlock, pDataBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pDstBlock); + return NULL; + } + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); + } + + pDstBlock->info.rows = pDataBlock->info.rows; + pDstBlock->info.capacity = pDataBlock->info.rows; + + pDstBlock->pBlockAgg = pDataBlock->pBlockAgg; + pDataBlock->pBlockAgg = NULL; + // int numOfSlots = sizeof(pDataBlock->pBlockAgg)/POINTER_BYTES; + // if (pDataBlock->pBlockAgg != NULL) { + // pDstBlock->pBlockAgg = taosMemoryCalloc(numOfSlots, POINTER_BYTES); + // if (pDstBlock->pBlockAgg == NULL) { + // terrno = TSDB_CODE_OUT_OF_MEMORY; + // return NULL; + // } + // for (int j = 0; j < numOfSlots; ++j) { + // pDstBlock->pBlockAgg[j] = &(*pDataBlock->pBlockAgg)[j]; + // } + // } + + return pDstBlock; +} + SSDataBlock* createDataBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pBlock == NULL) { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 99520f7c92..0771e4d5bf 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4903,7 +4903,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ } int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool* hasNullSMA) { - SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; + SColumnDataAgg** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; *allHave = false; @@ -4958,7 +4958,7 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, if (pResBlock->pBlockAgg == NULL) { size_t num = taosArrayGetSize(pResBlock->pDataBlock); - pResBlock->pBlockAgg = taosMemoryCalloc(num, POINTER_BYTES); + pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); } // do fill all null column value SMA info @@ -4970,13 +4970,13 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, while (j < numOfCols && i < size) { SColumnDataAgg* pAgg = &pSup->colAggArray.data[i]; if (pAgg->colId == pSup->colId[j]) { - pResBlock->pBlockAgg[pSup->slotId[j]] = pAgg; + pResBlock->pBlockAgg[pSup->slotId[j]] = *pAgg; i += 1; j += 1; } else if (pAgg->colId < pSup->colId[j]) { i += 1; } else if (pSup->colId[j] < pAgg->colId) { - pResBlock->pBlockAgg[pSup->slotId[j]] = NULL; + pResBlock->pBlockAgg[pSup->slotId[j]].colId = -1; *allHave = false; j += 1; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d06beebd6b..7672dd60aa 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2354,7 +2354,7 @@ int32_t compKeys(const SArray* pSortGroupCols, const char* oldkeyBuf, int32_t ol for (int32_t i = 0; i < pSortGroupCols->size; ++i) { const SColumn* pCol = (SColumn*)TARRAY_GET_ELEM(pSortGroupCols, i); const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId); - if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId]; + if (pBlock->pBlockAgg) pColAgg = &pBlock->pBlockAgg[pCol->slotId]; if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { if (isNull[i] != 1) return 1; @@ -2389,7 +2389,7 @@ int32_t buildKeys(char* keyBuf, const SArray* pSortGroupCols, const SSDataBlock* const SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, pCol->slotId); if (pCol->slotId > pBlock->pDataBlock->size) continue; - if (pBlock->pBlockAgg) pColAgg = pBlock->pBlockAgg[pCol->slotId]; + if (pBlock->pBlockAgg) pColAgg = &pBlock->pBlockAgg[pCol->slotId]; if (colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg)) { isNull[i] = 1; diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 43c04ca8d9..03b9a374c1 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -434,7 +434,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; - pInput->pColumnDataAgg[j] = pBlock->pBlockAgg[slotId]; + pInput->pColumnDataAgg[j] = &pBlock->pBlockAgg[slotId]; if (pInput->pColumnDataAgg[j] == NULL) { pInput->colDataSMAIsSet = false; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index c6a7804b9c..714c25aa3f 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -133,7 +133,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo SColumn* pCol = taosArrayGet(pGroupCols, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pCol->slotId); if (pBlock->pBlockAgg != NULL) { - pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } bool isNull = colDataIsNull(pColInfoData, pBlock->info.rows, rowIndex, pColAgg); @@ -189,7 +189,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData } if (pBlock->pBlockAgg != NULL) { - pColAgg = pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? + pColAgg = &pBlock->pBlockAgg[pCol->slotId]; // TODO is agg data matched? } SGroupKeys* pkey = taosArrayGet(pGroupColVals, i); @@ -653,7 +653,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { setBufPageDirty(pPage, true); releaseBufPage(pInfo->pBuf, pPage); } else { - SSDataBlock* dataNotLoadBlock = createOneDataBlock(pBlock, true); + SSDataBlock* dataNotLoadBlock = createBlockDataNotLoaded(pBlock); if (dataNotLoadBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index fb86e73a64..1bbc92005c 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -220,7 +220,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo* return code; } -static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsAgg, int32_t numOfCols, +static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg* pColsAgg, int32_t numOfCols, int32_t numOfRows) { if (pColsAgg == NULL || pFilterInfo == NULL) { return true; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b72811cdcc..7a70a3c2a0 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -911,7 +911,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI struct SColumnDataAgg* pAgg = NULL; for (int32_t j = 0; j < pBlock->info.rows; ++j) { - pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL; + pAgg = (pBlock->pBlockAgg != NULL) ? &pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL; if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) { continue; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index daac98bbfc..1715065d24 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -651,7 +651,7 @@ int32_t tsortComparBlockCell(SSDataBlock* pLeftBlock, SSDataBlock* pRightBlock, leftNull = colDataIsNull_t(pLeftColInfoData, leftRowIndex, isVarType); } else { leftNull = - colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, leftRowIndex, pLeftBlock->pBlockAgg[pOrder->slotId]); + colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, leftRowIndex, &pLeftBlock->pBlockAgg[pOrder->slotId]); } } @@ -661,7 +661,7 @@ int32_t tsortComparBlockCell(SSDataBlock* pLeftBlock, SSDataBlock* pRightBlock, rightNull = colDataIsNull_t(pRightColInfoData, rightRowIndex, isVarType); } else { rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, rightRowIndex, - pRightBlock->pBlockAgg[pOrder->slotId]); + &pRightBlock->pBlockAgg[pOrder->slotId]); } } @@ -742,7 +742,7 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) { leftNull = colDataIsNull_t(pLeftColInfoData, pLeftSource->src.rowIndex, isVarType); } else { leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->src.rowIndex, - pLeftBlock->pBlockAgg[i]); + &pLeftBlock->pBlockAgg[i]); } } @@ -752,7 +752,7 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) { rightNull = colDataIsNull_t(pRightColInfoData, pRightSource->src.rowIndex, isVarType); } else { rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->src.rowIndex, - pRightBlock->pBlockAgg[i]); + &pRightBlock->pBlockAgg[i]); } } diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 57f2543691..ea80ffd076 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3782,7 +3782,7 @@ int32_t fltSclBuildRangeFromBlockSma(SFltSclColumnRange *colRange, SColumnDataAg return TSDB_CODE_SUCCESS; } -bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t numOfCols, int32_t numOfRows) { +bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg *pDataStatis, int32_t numOfCols, int32_t numOfRows) { if (info->scalarMode) { SArray *colRanges = info->sclCtx.fltSclRange; for (int32_t i = 0; i < taosArrayGetSize(colRanges); ++i) { @@ -3790,13 +3790,13 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t bool foundCol = false; int32_t j = 0; for (; j < numOfCols; ++j) { - if (pDataStatis[j] != NULL && pDataStatis[j]->colId == colRange->colNode->colId) { + if (pDataStatis[j].colId == colRange->colNode->colId) { foundCol = true; break; } } if (foundCol) { - SColumnDataAgg *pAgg = pDataStatis[j]; + SColumnDataAgg *pAgg = &pDataStatis[j]; SArray *points = taosArrayInit(2, sizeof(SFltSclPoint)); fltSclBuildRangeFromBlockSma(colRange, pAgg, numOfRows, points); qDebug("column data agg: nulls %d, rows %d, max %" PRId64 " min %" PRId64, pAgg->numOfNull, numOfRows, @@ -3833,7 +3833,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t int32_t index = -1; SFilterRangeCtx *ctx = info->colRange[k]; for (int32_t i = 0; i < numOfCols; ++i) { - if (pDataStatis[i] != NULL && pDataStatis[i]->colId == ctx->colId) { + if (pDataStatis[i].colId == ctx->colId) { index = i; break; } @@ -3849,13 +3849,13 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t break; } - if (pDataStatis[index]->numOfNull <= 0) { + if (pDataStatis[index].numOfNull <= 0) { if (ctx->isnull && !ctx->notnull && !ctx->isrange) { ret = false; break; } - } else if (pDataStatis[index]->numOfNull > 0) { - if (pDataStatis[index]->numOfNull == numOfRows) { + } else if (pDataStatis[index].numOfNull > 0) { + if (pDataStatis[index].numOfNull == numOfRows) { if ((ctx->notnull || ctx->isrange) && (!ctx->isnull)) { ret = false; break; @@ -3869,7 +3869,7 @@ bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pDataStatis, int32_t } } - SColumnDataAgg *pDataBlockst = pDataStatis[index]; + SColumnDataAgg *pDataBlockst = &pDataStatis[index]; SFilterRangeNode *r = ctx->rs; float minv = 0; From 7a8e87f8cd7f27378393a87e91ed9bed3f6e0ce8 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 4 Jun 2024 19:05:19 +0800 Subject: [PATCH 012/103] fix: slot id --- include/common/tdatablock.h | 3 +- source/common/src/tdatablock.c | 58 +--------------------- source/libs/executor/src/groupoperator.c | 63 +++++++++++++++++++++++- 3 files changed, 65 insertions(+), 59 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 58fb3a6f4c..722f18c52d 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -256,7 +256,6 @@ SSDataBlock* createDataBlock(); void* blockDataDestroy(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); -SSDataBlock* createBlockDataNotLoaded(SSDataBlock* pDataBlock); SSDataBlock* createSpecialDataBlock(EStreamType type); SSDataBlock* blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx); @@ -283,6 +282,8 @@ int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* p void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList); +void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc); + #ifdef __cplusplus } #endif diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7b4d20238c..7b0de5adcf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -21,7 +21,7 @@ #define MALLOC_ALIGN_BYTES 32 -static void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc); + int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { @@ -1733,62 +1733,6 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return pDstBlock; } -SSDataBlock* createBlockDataNotLoaded(SSDataBlock* pDataBlock) { - if (pDataBlock == NULL) { - return NULL; - } - - SSDataBlock* pDstBlock = createDataBlock(); - pDstBlock->info = pDataBlock->info; - - pDstBlock->info.rows = 0; - pDstBlock->info.capacity = 0; - pDstBlock->info.rowSize = 0; - pDstBlock->info.id = pDataBlock->info.id; - pDstBlock->info.blankFill = pDataBlock->info.blankFill; - - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; - blockDataAppendColInfo(pDstBlock, &colInfo); - } - - copyPkVal(&pDstBlock->info, &pDataBlock->info); - - int32_t code = blockDataEnsureCapacity(pDstBlock, pDataBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - blockDataDestroy(pDstBlock); - return NULL; - } - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); - SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); - } - - pDstBlock->info.rows = pDataBlock->info.rows; - pDstBlock->info.capacity = pDataBlock->info.rows; - - pDstBlock->pBlockAgg = pDataBlock->pBlockAgg; - pDataBlock->pBlockAgg = NULL; - // int numOfSlots = sizeof(pDataBlock->pBlockAgg)/POINTER_BYTES; - // if (pDataBlock->pBlockAgg != NULL) { - // pDstBlock->pBlockAgg = taosMemoryCalloc(numOfSlots, POINTER_BYTES); - // if (pDstBlock->pBlockAgg == NULL) { - // terrno = TSDB_CODE_OUT_OF_MEMORY; - // return NULL; - // } - // for (int j = 0; j < numOfSlots; ++j) { - // pDstBlock->pBlockAgg[j] = &(*pDataBlock->pBlockAgg)[j]; - // } - // } - - return pDstBlock; -} - SSDataBlock* createDataBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pBlock == NULL) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 714c25aa3f..c44ba4eecb 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -565,6 +565,67 @@ _error: return NULL; } +SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBlock* pDataBlock) { + if (pDataBlock == NULL) { + return NULL; + } + + SSDataBlock* pDstBlock = createDataBlock(); + pDstBlock->info = pDataBlock->info; + + pDstBlock->info.rows = 0; + pDstBlock->info.capacity = 0; + pDstBlock->info.rowSize = 0; + pDstBlock->info.id = pDataBlock->info.id; + pDstBlock->info.blankFill = pDataBlock->info.blankFill; + + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + + pDstBlock->pBlockAgg = taosMemoryCalloc(numOfCols, sizeof(SColumnDataAgg)); + if (pDstBlock->pBlockAgg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + copyPkVal(&pDstBlock->info, &pDataBlock->info); + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId); + if (pSrc) { + SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info}; + blockDataAppendColInfo(pDstBlock, &colInfo); + } + } + + int32_t code = blockDataEnsureCapacity(pDstBlock, pDataBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pDstBlock); + return NULL; + } + + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + if (slotId < numOfCols) { + pDstBlock->pBlockAgg[slotId] = pDataBlock->pBlockAgg[i]; + pDstBlock->pBlockAgg[slotId].colId = i; + } + + // SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId); + // if (pSrc) { + // SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); + // colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); + // } + } + + pDstBlock->info.rows = pDataBlock->info.rows; + pDstBlock->info.capacity = pDataBlock->info.rows; + + return pDstBlock; +} + static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { SPartitionOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -653,7 +714,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { setBufPageDirty(pPage, true); releaseBufPage(pInfo->pBuf, pPage); } else { - SSDataBlock* dataNotLoadBlock = createBlockDataNotLoaded(pBlock); + SSDataBlock* dataNotLoadBlock = createBlockDataNotLoaded(pOperator, pBlock); if (dataNotLoadBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } From a5db0b4de5ff746194f5ecdda1f84366a61c2406 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 5 Jun 2024 10:41:22 +0800 Subject: [PATCH 013/103] fix:[TD-30365] test case error --- tests/system-test/7-tmq/subscribeDb3.py | 2 +- tests/system-test/7-tmq/tmqDropStbCtb.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb3.py b/tests/system-test/7-tmq/subscribeDb3.py index 37e3a17100..185d1b01cf 100644 --- a/tests/system-test/7-tmq/subscribeDb3.py +++ b/tests/system-test/7-tmq/subscribeDb3.py @@ -219,7 +219,7 @@ class TDTestCase: expectrowcnt = parameterDict["rowsPerTbl"] * parameterDict["ctbNum"] topicList = topicName1 ifcheckdata = 0 - ifManualCommit = 1 + ifManualCommit = 0 keyList = 'group.id:cgrp1,\ enable.auto.commit:false,\ auto.commit.interval.ms:6000,\ diff --git a/tests/system-test/7-tmq/tmqDropStbCtb.py b/tests/system-test/7-tmq/tmqDropStbCtb.py index eacacd913b..a1929237b7 100644 --- a/tests/system-test/7-tmq/tmqDropStbCtb.py +++ b/tests/system-test/7-tmq/tmqDropStbCtb.py @@ -157,7 +157,7 @@ class TDTestCase: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) if self.snapshot == 0: - if not ((totalConsumeRows > expectrowcnt / 2) and (totalConsumeRows < expectrowcnt)): + if not ((totalConsumeRows > expectrowcnt / 2) and (totalConsumeRows <= expectrowcnt)): tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") @@ -249,7 +249,7 @@ class TDTestCase: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) if self.snapshot == 0: - if not ((totalConsumeRows > expectrowcnt / 2) and (totalConsumeRows < expectrowcnt)): + if not ((totalConsumeRows > expectrowcnt / 2) and (totalConsumeRows <= expectrowcnt)): tdLog.exit("tmq consume rows error with snapshot = 0!") tdLog.info("wait subscriptions exit ....") From 48ecba95da8d7380f8962079fa373e5f8403b2ca Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 5 Jun 2024 18:02:57 +0800 Subject: [PATCH 014/103] fix:[TD-30365] ci case error & drop topic error if vnode is splitted --- source/dnode/mnode/impl/src/mndSubscribe.c | 12 ++++---- source/dnode/vnode/src/tq/tq.c | 29 +++++++------------ source/dnode/vnode/src/tq/tqMeta.c | 1 - source/dnode/vnode/src/tq/tqPush.c | 2 +- .../6-cluster/clusterCommonCheck.py | 18 +++++------- .../7-tmq/tmqVnodeSplit-stb-select-false.py | 5 ++-- .../7-tmq/tmqVnodeSplit-stb-select.py | 6 ++-- 7 files changed, 29 insertions(+), 44 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 0068b582cf..9f84e25c9f 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -912,6 +912,11 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran int32_t sz = taosArrayGetSize(pSub->unassignedVgs); for (int32_t i = 0; i < sz; i++) { SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i); + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pVgEp->vgId); + if (pVgObj == NULL) { + mError("sendDeleteSubToVnode %s failed since vg %d doesn't exist", pSub->key, pVgEp->vgId); + continue; + } SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); if(pReq == NULL){ terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -922,17 +927,12 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran pReq->consumerId = -1; memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pVgEp->vgId); - if (pVgObj == NULL) { - taosMemoryFree(pReq); - terrno = TSDB_CODE_MND_VGROUP_NOT_EXIST; - return -1; - } STransAction action = {0}; action.epSet = mndGetVgroupEpset(pMnode, pVgObj);; action.pCont = pReq; action.contLen = sizeof(SMqVDeleteReq); action.msgType = TDMT_VND_TMQ_DELETE_SUB; + action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST; mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 79f53e6dec..18442f182b 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -649,21 +649,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - STqHandle* pHandle = NULL; - while (1) { - pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); - if (pHandle) { - break; - } - taosRLockLatch(&pTq->lock); - ret = tqMetaGetHandle(pTq, req.subKey); - taosRUnLockLatch(&pTq->lock); - - if (ret < 0) { - break; - } - } - + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); if (pHandle == NULL) { if (req.oldConsumerId != -1) { tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, @@ -698,10 +684,17 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); - atomic_store_64(&pHandle->consumerId, req.newConsumerId); - atomic_store_32(&pHandle->epoch, 0); tqUnregisterPushHandle(pTq, pHandle); - ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); + taosHashRemove(pTq->pHandle, pHandle->subKey, strlen(pHandle->subKey)); + + // update handle to avoid req->qmsg changed if spilt vnode is failed + STqHandle handle = {0}; + ret = tqCreateHandle(pTq, &req, &handle); + if (ret < 0) { + tqDestroyTqHandle(&handle); + goto end; + } + ret = tqMetaSaveHandle(pTq, req.subKey, &handle); } taosWUnLockLatch(&pTq->lock); break; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 76322c527f..cb64c9033a 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -351,7 +351,6 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); handle->consumerId = req->newConsumerId; - handle->epoch = -1; handle->execHandle.subType = req->subType; handle->fetchMeta = req->withMeta; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 71e6771370..7375478e61 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -87,7 +87,7 @@ int tqUnregisterPushHandle(STQ* pTq, void *handle) { int32_t ret = taosHashRemove(pTq->pPushMgr, pHandle->subKey, strlen(pHandle->subKey)); tqInfo("vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%" PRIx64, vgId, pHandle, ret, pHandle->consumerId); - if(pHandle->msg != NULL) { + if(ret == 0 && pHandle->msg != NULL) { // tqPushDataRsp(pHandle, vgId); tqPushEmptyDataRsp(pHandle, vgId); diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py index 694227cea7..be99d01a5c 100644 --- a/tests/system-test/6-cluster/clusterCommonCheck.py +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -237,21 +237,19 @@ class ClusterComCheck: last_number=vgroup_numbers-1 while count < count_number: time.sleep(1) + count+=1 + print("check vgroup count :", count) tdSql.query(f"show {db_name}.vgroups;") - if count == 0 : - if tdSql.checkRows(vgroup_numbers) : - tdLog.success(f"{db_name} has {vgroup_numbers} vgroups" ) - else: - tdLog.exit(f"vgroup number of {db_name} is not correct") + if tdSql.getRows() != vgroup_numbers : + continue if self.db_replica == 1 : if tdSql.queryResult[0][4] == 'leader' and tdSql.queryResult[last_number][4] == 'leader': tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';") print("db replica :",tdSql.queryResult[0][0]) if tdSql.queryResult[0][0] == db_replica: - ready_time= (count + 1) - tdLog.success(f"all vgroups with replica {self.db_replica} of {db_name} are leaders in {count + 1} s") + tdLog.success(f"all vgroups with replica {self.db_replica} of {db_name} are leaders in {count} s") return True - count+=1 + elif self.db_replica == 3 : vgroup_status_first=[tdSql.queryResult[0][4],tdSql.queryResult[0][6],tdSql.queryResult[0][8]] @@ -261,10 +259,8 @@ class ClusterComCheck: tdSql.query(f"select `replica` from information_schema.ins_databases where `name`='{db_name}';") print("db replica :",tdSql.queryResult[0][0]) if tdSql.queryResult[0][0] == db_replica: - ready_time= (count + 1) - tdLog.success(f"elections of {db_name}.vgroups with replica {self.db_replica} are ready in {ready_time} s") + tdLog.success(f"elections of {db_name}.vgroups with replica {self.db_replica} are ready in {count} s") return True - count+=1 else: tdLog.debug(tdSql.queryResult) tdLog.notice(f"elections of {db_name} all vgroups with replica {self.db_replica} are failed in {count} s ") diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py index f01bf2558c..a5e61adc8d 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select-false.py @@ -200,12 +200,11 @@ class TDTestCase: # tmqCom.checkFileContent(consumerId, queryString) - time.sleep(2) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - if deleteWal == True: - clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) + tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): diff --git a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py index 5e11de04cb..eb35ebc718 100644 --- a/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py +++ b/tests/system-test/7-tmq/tmqVnodeSplit-stb-select.py @@ -199,13 +199,11 @@ class TDTestCase: tdLog.exit("%d tmq consume rows error!"%consumerId) # tmqCom.checkFileContent(consumerId, queryString) + clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) - time.sleep(2) + time.sleep(3) for i in range(len(topicNameList)): tdSql.query("drop topic %s"%topicNameList[i]) - - if deleteWal == True: - clusterComCheck.check_vgroups_status(vgroup_numbers=2,db_replica=self.replicaVar,db_name="dbt",count_number=240) tdLog.printNoPrefix("======== test case 1 end ...... ") def run(self): From 697b81aa8ade6241520e44001380df7e35f6f541 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 6 Jun 2024 11:15:23 +0800 Subject: [PATCH 015/103] fix:[TD-30365] ci case error & drop topic error if vnode is splitted --- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 12 ++++++------ source/dnode/vnode/src/tq/tqMeta.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 08d32b2b81..68d966add6 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -133,7 +133,7 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaRestoreCheckInfo(STQ* pTq); int32_t tqMetaGetHandle(STQ* pTq, const char* key); -int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle, int64_t snapshotVer); STqOffsetStore* tqOffsetOpen(STQ* pTq); int32_t tqMetaTransform(STQ* pTq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 18442f182b..925f8feb05 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -50,6 +50,9 @@ void tqDestroyTqHandle(void* data) { if (pData->block != NULL) { blockDataDestroy(pData->block); } + if (pData->pRef) { + walCloseRef(pData->pRef->pWal, pData->pRef->refId); + } } static bool tqOffsetEqual(const STqOffset* pLeft, const STqOffset* pRight) { @@ -571,9 +574,6 @@ int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg taosMsleep(10); continue; } - if (pHandle->pRef) { - walCloseRef(pTq->pVnode->pWal, pHandle->pRef->refId); - } tqUnregisterPushHandle(pTq, pHandle); @@ -660,7 +660,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } STqHandle handle = {0}; - ret = tqCreateHandle(pTq, &req, &handle); + ret = tqCreateHandle(pTq, &req, &handle, walGetCommittedVer(pTq->pVnode->pWal)); if (ret < 0) { tqDestroyTqHandle(&handle); goto end; @@ -689,7 +689,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg // update handle to avoid req->qmsg changed if spilt vnode is failed STqHandle handle = {0}; - ret = tqCreateHandle(pTq, &req, &handle); + ret = tqCreateHandle(pTq, &req, &handle, pHandle->snapshotVer); if (ret < 0) { tqDestroyTqHandle(&handle); goto end; @@ -701,7 +701,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } } -end: + end: tDecoderClear(&dc); return ret; } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index cb64c9033a..5162136591 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -346,7 +346,7 @@ end: return code; } -int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle, int64_t snapshotVer){ int32_t vgId = TD_VID(pTq->pVnode); memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); @@ -364,7 +364,7 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ handle->execHandle.execTb.qmsg = taosStrdup(req->qmsg); } - handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); + handle->snapshotVer = snapshotVer; if(buildHandle(pTq, handle) < 0){ return -1; From 0e337d34186717aae935cd6278588710659e9794 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 6 Jun 2024 17:26:20 +0800 Subject: [PATCH 016/103] fix:[TD-30365] ci case error & drop topic error if vnode is splitted --- source/dnode/vnode/src/tq/tq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 925f8feb05..58085e2cc2 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -685,13 +685,13 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); tqUnregisterPushHandle(pTq, pHandle); - taosHashRemove(pTq->pHandle, pHandle->subKey, strlen(pHandle->subKey)); // update handle to avoid req->qmsg changed if spilt vnode is failed STqHandle handle = {0}; ret = tqCreateHandle(pTq, &req, &handle, pHandle->snapshotVer); if (ret < 0) { tqDestroyTqHandle(&handle); + taosWUnLockLatch(&pTq->lock); goto end; } ret = tqMetaSaveHandle(pTq, req.subKey, &handle); From 6d98a56778f114c7618fdc9bf3f9707fe5fcf714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 7 Jun 2024 16:49:29 +0800 Subject: [PATCH 017/103] fix group count --- source/libs/executor/inc/executorInt.h | 2 + source/libs/executor/src/groupoperator.c | 113 +++++++++++++---------- source/libs/function/src/builtinsimpl.c | 2 +- 3 files changed, 66 insertions(+), 51 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 592231f043..fba8193ee5 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -624,6 +624,8 @@ typedef struct SDataGroupInfo { uint64_t groupId; int64_t numOfRows; SArray* pPageList; + SArray* blockForNotLoaded; // SSDataBlock that data is not loaded + int32_t offsetForNotLoaded; // read offset for SSDataBlock that data is not loaded } SDataGroupInfo; typedef struct SWindowRowsSup { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index c44ba4eecb..9473f650ae 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -53,8 +53,6 @@ typedef struct SPartitionOperatorInfo { int32_t rowCapacity; // maximum number of rows for each buffer page int32_t* columnOffset; // start position for each column data SArray* sortedGroupArray; // SDataGroupInfo sorted by group id - SArray* blockForNotLoaded; // SSDataBlock that data is not loaded - int32_t offsetForNotLoaded;// read offset for SSDataBlock that data is not loaded int32_t groupIndex; // group index int32_t pageIndex; // page index of current group SExprSupp scalarSup; @@ -573,21 +571,14 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc SSDataBlock* pDstBlock = createDataBlock(); pDstBlock->info = pDataBlock->info; - pDstBlock->info.rows = 0; + copyPkVal(&pDstBlock->info, &pDataBlock->info); + pDstBlock->info.rows = pDataBlock->info.rows; pDstBlock->info.capacity = 0; pDstBlock->info.rowSize = 0; pDstBlock->info.id = pDataBlock->info.id; pDstBlock->info.blankFill = pDataBlock->info.blankFill; size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - - pDstBlock->pBlockAgg = taosMemoryCalloc(numOfCols, sizeof(SColumnDataAgg)); - if (pDstBlock->pBlockAgg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - copyPkVal(&pDstBlock->info, &pDataBlock->info); - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; int32_t slotId = pExpr->base.pParam[0].pCol->slotId; @@ -605,24 +596,25 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc return NULL; } - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; - int32_t slotId = pExpr->base.pParam[0].pCol->slotId; - if (slotId < numOfCols) { - pDstBlock->pBlockAgg[slotId] = pDataBlock->pBlockAgg[i]; - pDstBlock->pBlockAgg[slotId].colId = i; + if (pDataBlock->pBlockAgg != NULL) { + pDstBlock->pBlockAgg = taosMemoryCalloc(numOfCols, sizeof(SColumnDataAgg)); + if (pDstBlock->pBlockAgg == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + blockDataDestroy(pDstBlock); + return NULL; + } + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + if (slotId < numOfCols) { + pDstBlock->pBlockAgg[slotId] = pDataBlock->pBlockAgg[i]; + SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); + SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, slotId); + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); + } } - - // SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId); - // if (pSrc) { - // SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); - // colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); - // } } - pDstBlock->info.rows = pDataBlock->info.rows; - pDstBlock->info.capacity = pDataBlock->info.rows; - return pDstBlock; } @@ -718,13 +710,14 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (dataNotLoadBlock == NULL) { T_LONG_JMP(pTaskInfo->env, terrno); } - if (pInfo->blockForNotLoaded == NULL) { - pInfo->blockForNotLoaded = taosArrayInit(1, sizeof(SSDataBlock)); - pInfo->offsetForNotLoaded = 0; + if (pGroupInfo->blockForNotLoaded == NULL) { + pGroupInfo->blockForNotLoaded = taosArrayInit(1, sizeof(SSDataBlock)); + pGroupInfo->offsetForNotLoaded = 0; } dataNotLoadBlock->info.id.groupId = pGroupInfo->groupId; dataNotLoadBlock->info.dataLoad = 0; - taosArrayInsert(pInfo->blockForNotLoaded, pInfo->blockForNotLoaded->size, &dataNotLoadBlock); + pInfo->binfo.pRes->info.rows = pBlock->info.rows; + taosArrayInsert(pGroupInfo->blockForNotLoaded, pGroupInfo->blockForNotLoaded->size, &dataNotLoadBlock); break; } } @@ -808,18 +801,18 @@ static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { int32_t size = taosArrayGetSize(pInfo->sortedGroupArray); for (int32_t i = 0; i < size; i++) { SDataGroupInfo* pGp = taosArrayGet(pInfo->sortedGroupArray, i); + if (pGp->blockForNotLoaded) { + for (int32_t i = 0; i < pGp->blockForNotLoaded->size; i++) { + SSDataBlock** pBlock = taosArrayGet(pGp->blockForNotLoaded, i); + blockDataDestroy(*pBlock); + } + taosArrayClear(pGp->blockForNotLoaded); + pGp->offsetForNotLoaded = 0; + } taosArrayDestroy(pGp->pPageList); } taosArrayClear(pInfo->sortedGroupArray); clearDiskbasedBuf(pInfo->pBuf); - if (pInfo->blockForNotLoaded) { - for (int32_t i = 0; i < pInfo->blockForNotLoaded->size; i++) { - SSDataBlock** pBlock = taosArrayGet(pInfo->blockForNotLoaded, i); - blockDataDestroy(*pBlock); - } - taosArrayClear(pInfo->blockForNotLoaded); - pInfo->offsetForNotLoaded = 0; - } } static int compareDataGroupInfo(const void* group1, const void* group2) { @@ -833,19 +826,13 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1; } -static SSDataBlock* buildPartitionResultForNotLoadBlock(SOperatorInfo* pOperator) { - SPartitionOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - if (pInfo->blockForNotLoaded && pInfo->offsetForNotLoaded < pInfo->blockForNotLoaded->size) { - SSDataBlock** pBlock = taosArrayGet(pInfo->blockForNotLoaded, pInfo->offsetForNotLoaded); - pInfo->offsetForNotLoaded++; +static SSDataBlock* buildPartitionResultForNotLoadBlock(SDataGroupInfo* pGroupInfo) { + if (pGroupInfo->blockForNotLoaded && pGroupInfo->offsetForNotLoaded < pGroupInfo->blockForNotLoaded->size) { + SSDataBlock** pBlock = taosArrayGet(pGroupInfo->blockForNotLoaded, pGroupInfo->offsetForNotLoaded); + pGroupInfo->offsetForNotLoaded++; return *pBlock; - } else { - setOperatorCompleted(pOperator); - clearPartitionOperator(pInfo); - return NULL; } + return NULL; } static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { @@ -857,9 +844,15 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { + if(pGroupInfo != NULL) { + SSDataBlock* ret = buildPartitionResultForNotLoadBlock(pGroupInfo); + if(ret != NULL) return ret; + } // try next group data if (pInfo->groupIndex + 1 >= taosArrayGetSize(pInfo->sortedGroupArray)) { - return buildPartitionResultForNotLoadBlock(pOperator); + setOperatorCompleted(pOperator); + clearPartitionOperator(pInfo); + return NULL; } ++pInfo->groupIndex; @@ -873,6 +866,22 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); T_LONG_JMP(pTaskInfo->env, terrno); } + if (*(int32_t*)page == 0) { + SSDataBlock* ret = buildPartitionResultForNotLoadBlock(pGroupInfo); + if (ret != NULL) return ret; + releaseBufPage(pInfo->pBuf, page); + if (pInfo->pageIndex < taosArrayGetSize(pGroupInfo->pPageList)) { + pInfo->pageIndex += 1; + } else if (pInfo->groupIndex + 1 < taosArrayGetSize(pInfo->sortedGroupArray)) { + pInfo->groupIndex++; + pInfo->pageIndex = 0; + } else { + setOperatorCompleted(pOperator); + clearPartitionOperator(pInfo); + return NULL; + } + return buildPartitionResult(pOperator); + } blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); @@ -882,6 +891,8 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId; pInfo->binfo.pRes->info.dataLoad = 1; pInfo->orderedRows = 0; + } else if (pInfo->pOrderInfoArr == NULL) { + qError("Exception, remainRows not zero, but pOrderInfoArr is NULL"); } if (pInfo->pOrderInfoArr) { @@ -944,6 +955,8 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { while (pGroupIter != NULL) { SDataGroupInfo* pGroupInfo = pGroupIter; taosArrayPush(groupArray, pGroupInfo); + static int i = 0; + qInfo("groupArray push %p %p %d times", pGroupInfo, pGroupInfo->blockForNotLoaded, ++i); pGroupIter = taosHashIterate(pInfo->pGroupSet, pGroupIter); } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3fb298e1ea..36de40a38a 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -3628,7 +3628,7 @@ int32_t saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* SColumnInfoData* pColInfo = taosArrayGet(pSrcBlock->pDataBlock, pCtx->saveHandle.pState->tsIndex); ASSERT(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); key.groupId = pSrcBlock->info.id.groupId; - key.ts = *(int64_t*)colDataGetData(pColInfo, rowIndex);; + key.ts = *(int64_t*)colDataGetData(pColInfo, rowIndex); } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); From d0675a660ef2811ea8c246d63e47521d3ae503cb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 Jun 2024 17:53:53 +0800 Subject: [PATCH 018/103] fix(stream): init the node list and task list when starting mnode. --- source/dnode/mnode/impl/inc/mndStream.h | 5 +- source/dnode/mnode/impl/src/mndMain.c | 2 + source/dnode/mnode/impl/src/mndStream.c | 74 ++++++++------------- source/dnode/mnode/impl/src/mndStreamHb.c | 60 ----------------- source/dnode/mnode/impl/src/mndStreamUtil.c | 42 ++++++++++++ 5 files changed, 75 insertions(+), 108 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 6aed50e508..c8a967620c 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -120,8 +120,9 @@ void destroyStreamTaskIter(SStreamTaskIter *pIter); bool streamTaskIterNextTask(SStreamTaskIter *pIter); SStreamTask *streamTaskIterGetCurrent(SStreamTaskIter *pIter); void mndInitExecInfo(); -void removeExpiredNodeInfo(const SArray *pNodeSnapshot); -void removeTasksInBuf(SArray* pTaskIds, SStreamExecInfo* pExecInfo); +void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo); +int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); +void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index cad8c6d745..9252843d2f 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -680,6 +680,8 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } + mndInitStreamExecInfo(pMnode, &execInfo); + mInfo("mnode open successfully"); return pMnode; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5fc6e465ad..1f1eaa999e 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -56,13 +56,14 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int int64_t streamId, int32_t taskId, int32_t transId, int8_t mndTrigger); static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); -static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList); +static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList); static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); -void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); -static int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot); +static void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); +static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo); +static void removeExpiredNodeInfo(const SArray *pNodeSnapshot); static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); static SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); @@ -131,9 +132,11 @@ int32_t mndInitStream(SMnode *pMnode) { if (sdbSetTable(pMnode->pSdb, table) != 0) { return -1; } + if (sdbSetTable(pMnode->pSdb, tableSeq) != 0) { return -1; } + return 0; } @@ -2169,7 +2172,7 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange return 0; } -static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) { +static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) { SSdb *pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; void *pIter = NULL; @@ -2215,48 +2218,6 @@ static SArray *extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList) { return TSDB_CODE_SUCCESS; } -static bool taskNodeExists(SArray *pList, int32_t nodeId) { - size_t num = taosArrayGetSize(pList); - - for (int32_t i = 0; i < num; ++i) { - SNodeEntry *pEntry = taosArrayGet(pList, i); - if (pEntry->nodeId == nodeId) { - return true; - } - } - - return false; -} - -int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) { - SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId)); - - int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList); - for (int32_t i = 0; i < numOfTask; ++i) { - STaskId *pId = taosArrayGet(execInfo.pTaskList, i); - - STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); - if (pEntry->nodeId == SNODE_HANDLE) { - continue; - } - - bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); - if (!existed) { - taosArrayPush(pRemovedTasks, pId); - } - } - - removeTasksInBuf(pRemovedTasks, &execInfo); - - mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks), - (int32_t)taosArrayGetSize(execInfo.pTaskList)); - - removeExpiredNodeInfo(pNodeSnapshot); - - taosArrayDestroy(pRemovedTasks); - return 0; -} - // this function runs by only one thread, so it is not multi-thread safe static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { int32_t code = 0; @@ -2510,3 +2471,24 @@ static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) { } return code; } + +void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) { + addAllStreamTasksIntoBuf(pMnode, pExecInfo); + extractNodeListFromStream(pMnode, pExecInfo->pNodeList); +} + +void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { + SSdb *pSdb = pMnode->pSdb; + SStreamObj *pStream = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) { + break; + } + + saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo); + sdbRelease(pSdb, pStream); + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 42efb6589e..6f398fbc11 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -22,55 +22,6 @@ typedef struct SFailedCheckpointInfo { int32_t transId; } SFailedCheckpointInfo; -static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { - SSdb *pSdb = pMnode->pSdb; - SStreamObj *pStream = NULL; - void *pIter = NULL; - - while (1) { - pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); - if (pIter == NULL) { - break; - } - - saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo); - sdbRelease(pSdb, pStream); - } -} - -static void removeDroppedStreamTasksInBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo) { - if (pMnode == NULL) { - return; - } - - int32_t num = taosArrayGetSize(pExecInfo->pTaskList); - - SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - SArray *pIdList = taosArrayInit(4, sizeof(STaskId)); - - for (int32_t i = 0; i < num; ++i) { - STaskId* pId = taosArrayGet(pExecInfo->pTaskList, i); - - void* p = taosHashGet(pHash, &pId->streamId, sizeof(int64_t)); - if (p != NULL) { - continue; - } - - void* pObj = mndGetStreamObj(pMnode, pId->streamId); - if (pObj != NULL) { - mndReleaseStream(pMnode, pObj); - taosHashPut(pHash, &pId->streamId, sizeof(int64_t), NULL, 0); - } else { - taosArrayPush(pIdList, pId); - } - } - - removeTasksInBuf(pIdList, &execInfo); - - taosArrayDestroy(pIdList); - taosHashCleanup(pHash); -} - static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList); for (int32_t j = 0; j < numOfNodes; ++j) { @@ -290,17 +241,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { taosThreadMutexLock(&execInfo.lock); - // extract stream task list - if (taosHashGetSize(execInfo.pTaskMap) == 0) { - addAllStreamTasksIntoBuf(pMnode, &execInfo); - } else { - // the already dropped tasks may be added by hb from vnode at the time when the pTaskMap happens to be empty. - // let's drop them here. - removeDroppedStreamTasksInBuf(pMnode, &execInfo); - } - - extractStreamNodeList(pMnode); - int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); if (numOfUpdated > 0) { mDebug("%d stream node(s) need updated from hbMsg(vgId:%d)", numOfUpdated, req.vgId); diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index d138254afd..7e72f67f45 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -645,4 +645,46 @@ void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { taosThreadMutexUnlock(&pExecNode->lock); destroyStreamTaskIter(pIter); +} + +static bool taskNodeExists(SArray *pList, int32_t nodeId) { + size_t num = taosArrayGetSize(pList); + + for (int32_t i = 0; i < num; ++i) { + SNodeEntry *pEntry = taosArrayGet(pList, i); + if (pEntry->nodeId == nodeId) { + return true; + } + } + + return false; +} + +int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) { + SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId)); + + int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList); + for (int32_t i = 0; i < numOfTask; ++i) { + STaskId *pId = taosArrayGet(execInfo.pTaskList, i); + + STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); + if (pEntry->nodeId == SNODE_HANDLE) { + continue; + } + + bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); + if (!existed) { + taosArrayPush(pRemovedTasks, pId); + } + } + + removeTasksInBuf(pRemovedTasks, &execInfo); + + mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks), + (int32_t)taosArrayGetSize(execInfo.pTaskList)); + + removeExpiredNodeInfo(pNodeSnapshot); + + taosArrayDestroy(pRemovedTasks); + return 0; } \ No newline at end of file From 16ceacac2b3d2fbb6c9b150eb93923617b46e42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Fri, 7 Jun 2024 20:12:18 +0800 Subject: [PATCH 019/103] fix: no tag value --- source/libs/executor/src/groupoperator.c | 26 ++++++++++-------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9473f650ae..63be7eaa67 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -570,23 +570,16 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc SSDataBlock* pDstBlock = createDataBlock(); pDstBlock->info = pDataBlock->info; - - copyPkVal(&pDstBlock->info, &pDataBlock->info); - pDstBlock->info.rows = pDataBlock->info.rows; pDstBlock->info.capacity = 0; pDstBlock->info.rowSize = 0; - pDstBlock->info.id = pDataBlock->info.id; - pDstBlock->info.blankFill = pDataBlock->info.blankFill; size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; int32_t slotId = pExpr->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId); - if (pSrc) { - SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info}; - blockDataAppendColInfo(pDstBlock, &colInfo); - } + SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info}; + blockDataAppendColInfo(pDstBlock, &colInfo); } int32_t code = blockDataEnsureCapacity(pDstBlock, pDataBlock->info.rows); @@ -606,15 +599,18 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; int32_t slotId = pExpr->base.pParam[0].pCol->slotId; - if (slotId < numOfCols) { - pDstBlock->pBlockAgg[slotId] = pDataBlock->pBlockAgg[i]; - SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, slotId); - colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); - } + pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId]; } } + for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { + SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId); + SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); + } + return pDstBlock; } From cbf8b363fc3317f4c638531a220785b6da3857e2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Jun 2024 00:46:01 +0800 Subject: [PATCH 020/103] fix(stream): update checkpoint info by using trans. --- include/common/tmsg.h | 4 - include/common/tmsgdef.h | 5 +- include/dnode/vnode/tqCommon.h | 1 + include/libs/stream/streammsg.h | 14 ++ include/libs/stream/tstream.h | 3 +- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 + source/dnode/mgmt/mgmt_snode/src/smHandle.c | 9 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 + source/dnode/mnode/impl/inc/mndStream.h | 34 +++- source/dnode/mnode/impl/src/mndMain.c | 2 - source/dnode/mnode/impl/src/mndStream.c | 201 +++++++++++++++++++- source/dnode/mnode/impl/src/mndStreamHb.c | 19 +- source/dnode/mnode/impl/src/mndStreamUtil.c | 112 +++++++++++ source/dnode/snode/src/snode.c | 2 + source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/tq/tq.c | 4 + source/dnode/vnode/src/tqCommon/tqCommon.c | 2 + source/dnode/vnode/src/vnd/vnodeSvr.c | 2 + source/libs/stream/src/streamCheckpoint.c | 22 ++- source/libs/stream/src/streamTask.c | 65 ++++--- source/libs/stream/src/streammsg.c | 28 +++ 21 files changed, 465 insertions(+), 68 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fa294c3fc5..98ca414466 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3627,10 +3627,6 @@ typedef struct { int32_t taskId; } SVPauseStreamTaskReq, SVResetStreamTaskReq; -typedef struct { - int8_t reserved; -} SVPauseStreamTaskRsp; - typedef struct { char name[TSDB_STREAM_FNAME_LEN]; int8_t igNotExists; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index d1ac2c79c3..db5ae21692 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -207,9 +207,9 @@ TD_DEF_MSG_TYPE(TDMT_MND_RESTORE_DNODE, "restore-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_PAUSE_STREAM, "pause-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) // unused,reserved TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_CANDIDITATE, "stream-checkpoint-remain", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHKPT_REPORT, "stream-chkpt-report", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TRIM_DB_TIMER, "trim-db-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT_NOTIFY, "grant-notify", NULL, NULL) @@ -370,6 +370,7 @@ TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL) TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 0076d79312..0fb690e756 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -28,6 +28,7 @@ int32_t tqStreamTaskProcessCheckRsp(SStreamMeta* pMeta, SRpcMsg* pMsg, bool isLe int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); +int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg); int32_t tqStreamTaskProcessDeployReq(SStreamMeta* pMeta, SMsgCb* cb, int64_t sversion, char* msg, int32_t msgLen, bool isLeader, bool restored); diff --git a/include/libs/stream/streammsg.h b/include/libs/stream/streammsg.h index 91bfc6afc8..723c9fd099 100644 --- a/include/libs/stream/streammsg.h +++ b/include/libs/stream/streammsg.h @@ -190,6 +190,20 @@ typedef struct SCheckpointTriggerRsp { int32_t rspCode; } SCheckpointTriggerRsp; +typedef struct SCheckpointReport { + int64_t streamId; + int32_t taskId; + int32_t nodeId; + int64_t checkpointId; + int64_t checkpointVer; + int64_t checkpointTs; + int32_t transId; + int8_t dropHTask; +} SCheckpointReport; + +int32_t tEncodeStreamTaskChkptReport(SEncoder* pEncoder, const SCheckpointReport* pReq); +int32_t tDecodeStreamTaskChkptReport(SDecoder* pDecoder, SCheckpointReport* pReq); + typedef struct { SMsgHead head; int64_t streamId; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index d07a302920..1996c2ed63 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -762,8 +762,7 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SStreamTask* pTask); int32_t streamTaskBuildCheckpointSourceRsp(SStreamCheckpointSourceReq* pReq, SRpcHandleInfo* pRpcInfo, SRpcMsg* pMsg, int32_t setCode); -int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId, - SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask); +int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask); int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpointInfoReq* pReq); SActiveCheckpointInfo* streamTaskCreateActiveChkptInfo(); diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index e4afba6722..3d5da860ad 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -232,6 +232,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -240,6 +241,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 9b07b6a3d8..856d7b2051 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -75,26 +75,27 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_CHECKPOINT_READY_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_TRIGGER_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_CHECK_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index bfc9e92293..41b9352f18 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -967,6 +967,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToWriteQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index c8a967620c..6ca61265bb 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -26,13 +26,14 @@ extern "C" { #define MND_STREAM_RESERVE_SIZE 64 #define MND_STREAM_VER_NUMBER 5 -#define MND_STREAM_CREATE_NAME "stream-create" -#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint" -#define MND_STREAM_PAUSE_NAME "stream-pause" -#define MND_STREAM_RESUME_NAME "stream-resume" -#define MND_STREAM_DROP_NAME "stream-drop" -#define MND_STREAM_TASK_RESET_NAME "stream-task-reset" -#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update" +#define MND_STREAM_CREATE_NAME "stream-create" +#define MND_STREAM_CHECKPOINT_NAME "stream-checkpoint" +#define MND_STREAM_PAUSE_NAME "stream-pause" +#define MND_STREAM_RESUME_NAME "stream-resume" +#define MND_STREAM_DROP_NAME "stream-drop" +#define MND_STREAM_TASK_RESET_NAME "stream-task-reset" +#define MND_STREAM_TASK_UPDATE_NAME "stream-task-update" +#define MND_STREAM_CHKPT_UPDATE_NAME "stream-chkpt-update" typedef struct SStreamTransInfo { int64_t startTime; @@ -51,6 +52,7 @@ typedef struct SStreamTransMgmt { } SStreamTransMgmt; typedef struct SStreamExecInfo { + bool initTaskList; SArray *pNodeList; int64_t ts; // snapshot ts SStreamTransMgmt transMgmt; @@ -58,6 +60,7 @@ typedef struct SStreamExecInfo { SArray *pTaskList; TdThreadMutex lock; SHashObj *pTransferStateStreams; + SHashObj *pChkptStreams; } SStreamExecInfo; extern SStreamExecInfo execInfo; @@ -78,7 +81,18 @@ typedef struct SOrphanTask { typedef struct { SMsgHead head; -} SMStreamHbRspMsg, SMStreamReqCheckpointRspMsg; +} SMStreamHbRspMsg, SMStreamReqCheckpointRsp, SMStreamUpdateChkptRsp; + +typedef struct STaskChkptInfo { + int32_t nodeId; + int32_t taskId; + int64_t streamId; + int64_t checkpointId; + int64_t version; + int64_t ts; + int32_t transId; + int8_t dropHTask; +}STaskChkptInfo; int32_t mndInitStream(SMnode *pMnode); void mndCleanupStream(SMnode *pMnode); @@ -114,6 +128,10 @@ int32_t mndStreamSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamObj *p int32_t mndStreamSetDropActionFromList(SMnode *pMnode, STrans *pTrans, SArray *pList); int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); +int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList); +void scanCheckpointReportInfo(SMnode *pMnode); +void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); void destroyStreamTaskIter(SStreamTaskIter *pIter); diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 9252843d2f..cad8c6d745 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -680,8 +680,6 @@ SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { return NULL; } - mndInitStreamExecInfo(pMnode, &execInfo); - mInfo("mnode open successfully"); return pMnode; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 1f1eaa999e..7ee34d502a 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -58,10 +58,10 @@ static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static int32_t extractNodeListFromStream(SMnode *pMnode, SArray* pNodeList); static int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq); +static int32_t mndProcessCheckpointReport(SRpcMsg *pReq); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); -static void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); static void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo *pExecInfo); static void removeExpiredNodeInfo(const SArray *pNodeSnapshot); static int32_t doKillCheckpointTrans(SMnode *pMnode, const char *pDbName, size_t len); @@ -104,6 +104,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_STOP_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_UPDATE_CHKPT_RSP, mndTransProcessRsp); // for msgs inside mnode // TODO change the name @@ -114,6 +115,7 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_REPORT, mndProcessCheckpointReport); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp); @@ -1091,7 +1093,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) { } static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { - bool ready = true; + bool ready = true; if (taskNodeIsUpdated(pMnode)) { return -1; } @@ -1102,6 +1104,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { ASSERT(taosArrayGetSize(execInfo.pTaskList) == 0); } + SArray* pInvalidList = taosArrayInit(4, sizeof(STaskId)); + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); @@ -1109,11 +1113,32 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { continue; } + // for stopped stream task entry, we do additional check for it + if (pEntry->status == TASK_STATUS__STOP) { + bool invalid = false; + for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) { + STaskId* pId = taosArrayGet(pInvalidList, j); + if (pEntry->id.streamId == pId->streamId) { + taosArrayPush(pInvalidList, &pEntry->id); + invalid = true; + break; + } + } + + if (!invalid) { + SStreamObj *pObj = mndGetStreamObj(pMnode, pEntry->id.streamId); + if (pObj == NULL) { + taosArrayPush(pInvalidList, &pEntry->id); + } else { + mndReleaseStream(pMnode, pObj); + } + } + } + if (pEntry->status != TASK_STATUS__READY) { mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status)); ready = false; - break; } if (pEntry->hTaskId != 0) { @@ -1126,6 +1151,8 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { } } + removeTasksInBuf(pInvalidList, &execInfo); + taosThreadMutexUnlock(&execInfo.lock); return ready ? 0 : -1; } @@ -1801,6 +1828,10 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfRows = 0; SStreamObj *pStream = NULL; + taosThreadMutexLock(&execInfo.lock); + mndInitStreamExecInfo(pMnode, &execInfo); + taosThreadMutexUnlock(&execInfo.lock); + while (numOfRows < rowsCapacity) { pShow->pIter = sdbFetch(pSdb, SDB_STREAM, pShow->pIter, (void **)&pStream); if (pShow->pIter == NULL) { @@ -2437,13 +2468,136 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) { taosThreadMutexUnlock(&execInfo.lock); { - SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRspMsg)}; + SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamReqCheckpointRsp)}; rsp.pCont = rpcMallocCont(rsp.contLen); SMsgHead *pHead = rsp.pCont; pHead->vgId = htonl(req.nodeId); tmsgSendRsp(&rsp); + pReq->info.handle = NULL; // disable auto rsp + } + return 0; +} + +static void doAddTaskInfo(SArray* pList, SCheckpointReport* pReport) { + bool existed = false; + for(int32_t i = 0; i < taosArrayGetSize(pList); ++i) { + STaskChkptInfo* p = taosArrayGet(pList ,i); + if (p->taskId == pReport->taskId) { + existed = true; + break; + } + } + + if (!existed) { + STaskChkptInfo info = { + .streamId = pReport->streamId, + .taskId = pReport->taskId, + .transId = pReport->transId, + .dropHTask = pReport->dropHTask, + .version = pReport->checkpointVer, + .ts = pReport->checkpointTs, + .checkpointId = pReport->checkpointId, + .nodeId = pReport->nodeId, + }; + taosArrayPush(pList, &info); + } +} + +int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SCheckpointReport req = {0}; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, pReq->pCont, pReq->contLen); + + if (tDecodeStreamTaskChkptReport(&decoder, &req)) { + tDecoderClear(&decoder); + terrno = TSDB_CODE_INVALID_MSG; + mError("invalid task checkpoint-report msg received"); + return -1; + } + tDecoderClear(&decoder); + + mDebug("receive stream task checkpoint-report msg, vgId:%d, s-task:0x%x, checkpointId:%" PRId64 + " checkpointVer:%" PRId64 " transId:%d", + req.nodeId, req.taskId, req.checkpointId, req.checkpointVer, req.transId); + + // register to the stream task done map, if all tasks has sent this kinds of message, start the checkpoint trans. + taosThreadMutexLock(&execInfo.lock); + + SStreamObj *pStream = mndGetStreamObj(pMnode, req.streamId); + if (pStream == NULL) { + mWarn("failed to find the stream:0x%" PRIx64 ", not handle checkpoint-report, try to acquire in buf", + req.streamId); + + // not in meta-store yet, try to acquire the task in exec buffer + // the checkpoint req arrives too soon before the completion of the create stream trans. + STaskId id = {.streamId = req.streamId, .taskId = req.taskId}; + void *p = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); + if (p == NULL) { + mError("failed to find the stream:0x%" PRIx64 " in buf, not handle the checkpoint-report", req.streamId); + terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + taosThreadMutexUnlock(&execInfo.lock); + return -1; + } else { + mDebug("s-task:0x%" PRIx64 "-0x%x in buf not in mnode/meta, create stream trans may not complete yet", + req.streamId, req.taskId); + } + } + + int32_t numOfTasks = (pStream == NULL) ? 0 : mndGetNumOfStreamTasks(pStream); + + SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId)); + if (pReqTaskList == NULL) { + SArray *pList = taosArrayInit(4, sizeof(STaskChkptInfo)); + doAddTaskInfo(pList, &req); + taosHashPut(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId), &pList, POINTER_BYTES); + + pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId)); + } else { + doAddTaskInfo(*pReqTaskList, &req); + } + + int32_t total = taosArrayGetSize(*pReqTaskList); + if (total == numOfTasks) { // all tasks has send the reqs + mInfo("stream:0x%" PRIx64 + " %s all %d tasks send checkpoint-report, start to update checkpoint meta-info for checkpointId:%" PRId64, + req.streamId, pStream->name, total, req.checkpointId); + + if (pStream != NULL) { + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); + if (conflict) { + mDebug("stream:%x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId); + } else { + int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, *pReqTaskList); + if (code == TSDB_CODE_SUCCESS) { // remove this entry + taosHashRemove(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId)); + + int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); + mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", req.streamId, numOfStreams); + } else { + mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet", + req.streamId); + } + } + } + } + + if (pStream != NULL) { + mndReleaseStream(pMnode, pStream); + } + + taosThreadMutexUnlock(&execInfo.lock); + + { + SRpcMsg rsp = {.code = 0, .info = pReq->info, .contLen = sizeof(SMStreamUpdateChkptRsp)}; + rsp.pCont = rpcMallocCont(rsp.contLen); + SMsgHead *pHead = rsp.pCont; + pHead->vgId = htonl(req.nodeId); + + tmsgSendRsp(&rsp); pReq->info.handle = NULL; // disable auto rsp } @@ -2473,8 +2627,13 @@ static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) { } void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) { + if (pExecInfo->initTaskList) { + return; + } + addAllStreamTasksIntoBuf(pMnode, pExecInfo); extractNodeListFromStream(pMnode, pExecInfo->pNodeList); + pExecInfo->initTaskList = true; } void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { @@ -2491,4 +2650,38 @@ void addAllStreamTasksIntoBuf(SMnode *pMnode, SStreamExecInfo* pExecInfo) { saveTaskAndNodeInfoIntoBuf(pStream, pExecInfo); sdbRelease(pSdb, pStream); } +} + +int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray* pChkptInfoList) { + STrans *pTrans = doCreateTrans(pMnode, pStream, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_CHKPT_UPDATE_NAME, "update checkpoint-info"); + if (pTrans == NULL) { + return terrno; + } + + /*int32_t code = */mndStreamRegisterTrans(pTrans, MND_STREAM_CHKPT_UPDATE_NAME, pStream->uid); + int32_t code = mndStreamSetUpdateChkptAction(pMnode, pTrans, pStream); + if (code != 0) { + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return code; + } + + code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_READY); + if (code != TSDB_CODE_SUCCESS) { + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return -1; + } + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare update checkpoint-info meta trans since %s", pTrans->id, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return -1; + } + + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + + return TSDB_CODE_ACTION_IN_PROGRESS; } \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 6f398fbc11..7c07d003b7 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -241,6 +241,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { taosThreadMutexLock(&execInfo.lock); + mndInitStreamExecInfo(pMnode, &execInfo); + int32_t numOfUpdated = taosArrayGetSize(req.pUpdateNodes); if (numOfUpdated > 0) { mDebug("%d stream node(s) need updated from hbMsg(vgId:%d)", numOfUpdated, req.vgId); @@ -266,18 +268,6 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { snodeChanged = true; } } else { - // task is idle for more than 50 sec. -// if (fabs(pTaskEntry->inputQUsed - p->inputQUsed) <= DBL_EPSILON) { -// if (!pTaskEntry->inputQChanging) { -// pTaskEntry->inputQUnchangeCounter++; -// } else { -// pTaskEntry->inputQChanging = false; -// } -// } else { -// pTaskEntry->inputQChanging = true; -// pTaskEntry->inputQUnchangeCounter = 0; -// } - streamTaskStatusCopy(pTaskEntry, p); STaskCkptInfo *pChkInfo = &p->checkpointInfo; @@ -288,6 +278,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { SFailedCheckpointInfo info = { .transId = pChkInfo->activeTransId, .checkpointId = pChkInfo->activeId, .streamUid = p->id.streamId}; addIntoCheckpointList(pFailedChkpt, &info); + + // remove failed trans from pChkptStreams + taosHashRemove(execInfo.pChkptStreams, &p->id.streamId, sizeof(p->id.streamId)); } } @@ -333,6 +326,8 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { mndDropOrphanTasks(pMnode, pOrphanTasks); } + scanCheckpointReportInfo(pMnode); + taosThreadMutexUnlock(&execInfo.lock); tCleanupStreamHbMsg(&req); diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 7e72f67f45..695c33f646 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -574,6 +574,7 @@ void mndInitExecInfo() { execInfo.pTaskMap = taosHashInit(64, fn, true, HASH_NO_LOCK); execInfo.transMgmt.pDBTrans = taosHashInit(32, fn, true, HASH_NO_LOCK); execInfo.pTransferStateStreams = taosHashInit(32, fn, true, HASH_NO_LOCK); + execInfo.pChkptStreams = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry)); taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); @@ -687,4 +688,115 @@ int32_t removeExpiredNodeEntryAndTaskInBuf(SArray *pNodeSnapshot) { taosArrayDestroy(pRemovedTasks); return 0; +} + +static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTask) { + SVUpdateCheckpointInfoReq *pReq = taosMemoryCalloc(1, sizeof(SVUpdateCheckpointInfoReq)); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVUpdateCheckpointInfoReq), + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return terrno; + } + + pReq->head.vgId = htonl(pTask->info.nodeId); + pReq->taskId = pTask->id.taskId; + pReq->streamId = pTask->id.streamId; + + SArray **pReqTaskList = (SArray **)taosHashGet(execInfo.pChkptStreams, &pTask->id.streamId, sizeof(pTask->id.streamId)); + ASSERT(pReqTaskList); + + int32_t size = taosArrayGetSize(*pReqTaskList); + for(int32_t i = 0; i < size; ++i) { + STaskChkptInfo* pInfo = taosArrayGet(*pReqTaskList, i); + if (pInfo->taskId == pTask->id.taskId) { + pReq->checkpointId = pInfo->checkpointId; + pReq->checkpointVer = pInfo->version; + pReq->checkpointTs = pInfo->ts; + pReq->dropRelHTask = pInfo->dropHTask; + pReq->transId = pInfo->transId; + } + } + + SEpSet epset = {0}; + bool hasEpset = false; + int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId); + if (code != TSDB_CODE_SUCCESS || !hasEpset) { + taosMemoryFree(pReq); + return code; + } + + code = setTransAction(pTrans, pReq, sizeof(SVUpdateCheckpointInfoReq), TDMT_STREAM_TASK_UPDATE_CHKPT, &epset, 0); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pReq); + } + + return code; +} + +int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { + taosWLockLatch(&pStream->lock); + + SStreamTaskIter *pIter = createStreamTaskIter(pStream); + while (streamTaskIterNextTask(pIter)) { + SStreamTask *pTask = streamTaskIterGetCurrent(pIter); + + int32_t code = doSetUpdateChkptAction(pMnode, pTrans, pTask); + if (code != TSDB_CODE_SUCCESS) { + destroyStreamTaskIter(pIter); + taosWUnLockLatch(&pStream->lock); + return -1; + } + } + + destroyStreamTaskIter(pIter); + taosWUnLockLatch(&pStream->lock); + return 0; +} + +void scanCheckpointReportInfo(SMnode* pMnode) { + void* pIter = NULL; + mDebug("start to scan checkpoint report info"); + + while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) { + SArray *pList = *(SArray **)pIter; + + STaskChkptInfo* pInfo = taosArrayGet(pList, 0); + SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId); + if (pStream == NULL) { + mError("failed to acquire stream:0x%"PRIx64" remove it from checkpoint-report list", pInfo->streamId); + taosHashRemove(execInfo.pChkptStreams, &pInfo->streamId, sizeof(pInfo->streamId)); + continue; + } + + int32_t total = mndGetNumOfStreamTasks(pStream); + int32_t existed = (int32_t) taosArrayGetSize(pList); + + if (total == existed) { + mDebug("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, start to update checkpoint-info", + pStream->uid, pStream->name, total); + + bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); + if (!conflict) { + int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); + if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry + taosHashRemove(execInfo.pChkptStreams, &pInfo->streamId, sizeof(pInfo->streamId)); + + int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); + mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", pInfo->streamId, numOfStreams); + } else { + mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet", + pInfo->streamId); + } + } else { + mDebug("stream:%x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); + } + } else { + mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name, + existed, total, total - existed); + } + + sdbRelease(pMnode->pSdb, pStream); + } + } \ No newline at end of file diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index c61988574c..b0d61ebc06 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -136,6 +136,8 @@ int32_t sndProcessStreamMsg(SSnode *pSnode, SRpcMsg *pMsg) { return tqStreamProcessReqCheckpointRsp(pSnode->pMeta, pMsg); case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP: return tqStreamProcessCheckpointReadyRsp(pSnode->pMeta, pMsg); + case TDMT_MND_STREAM_CHKPT_REPORT_RSP: + return tqStreamProcessChkptReportRsp(pSnode->pMeta, pMsg); case TDMT_STREAM_RETRIEVE_TRIGGER: return tqStreamTaskProcessRetrieveTriggerReq(pSnode->pMeta, pMsg); case TDMT_STREAM_RETRIEVE_TRIGGER_RSP: diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 8222af4d60..7f5ab8b6e6 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -261,6 +261,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamHbRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 712cfbaa55..a2ca3662d7 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1278,3 +1278,7 @@ int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg) { int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg) { return tqStreamProcessCheckpointReadyRsp(pTq->pStreamMeta, pMsg); } + +int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg) { + return tqStreamProcessChkptReportRsp(pTq->pStreamMeta, pMsg); +} diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c55745e5c5..7c8ad159bc 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -1078,6 +1078,8 @@ int32_t tqStreamProcessStreamHbRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return d int32_t tqStreamProcessReqCheckpointRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { return doProcessDummyRspMsg(pMeta, pMsg); } +int32_t tqStreamProcessChkptReportRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) {return doProcessDummyRspMsg(pMeta, pMsg);} + int32_t tqStreamProcessCheckpointReadyRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SMStreamCheckpointReadyRspMsg* pRsp = pMsg->pCont; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 002f04b8a7..b1f353af81 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -854,6 +854,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg); case TDMT_VND_GET_STREAM_PROGRESS: return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg); + case TDMT_MND_STREAM_CHKPT_REPORT_RSP: + return tqProcessTaskChkptReportRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in stream queue", pMsg->msgType); return TSDB_CODE_APP_ERROR; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index ab3b5d6fa0..56ac794d8e 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -441,8 +441,10 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin if (pReq->checkpointId <= pInfo->checkpointId) { stDebug("s-task:%s vgId:%d latest checkpointId:%" PRId64 " checkpointVer:%" PRId64 - " no need to update the checkpoint info, updated checkpointId:%" PRId64 " checkpointVer:%" PRId64 " ignored", - id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer); + " no need to update the checkpoint info, updated checkpointId:%" PRId64 " checkpointVer:%" PRId64 + " transId:%d ignored", + id, vgId, pInfo->checkpointId, pInfo->checkpointVer, pReq->checkpointId, pReq->checkpointVer, + pReq->transId); taosThreadMutexUnlock(&pTask->lock); { // destroy the related fill-history tasks @@ -703,9 +705,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { // update the latest checkpoint info if all works are done successfully, for rsma, the pMsgCb is null. if (code == TSDB_CODE_SUCCESS && (pTask->pMsgCb != NULL)) { - STaskId* pHTaskId = &pTask->hTaskInfo.id; - code = streamBuildAndSendCheckpointUpdateMsg(pTask->pMsgCb, pMeta->vgId, &pTask->id, pHTaskId, &pTask->chkInfo, - dropRelHTask); + code = streamSendChkptReportMsg(pTask, &pTask->chkInfo, dropRelHTask); if (code == TSDB_CODE_SUCCESS) { code = streamTaskRemoteBackupCheckpoint(pTask, ckId, (char*)id); if (code != TSDB_CODE_SUCCESS) { @@ -770,6 +770,18 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) { streamMetaReleaseTask(pTask->pMeta, pTask); return; } + + // checkpoint-trigger recv flag is set, quit + if (pActiveInfo->allUpstreamTriggerRecv) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s vgId:%d all checkpoint-trigger recv, quit from monitor checkpoint-trigger, ref:%d", + pTask->id.idStr, vgId, ref); + + taosThreadMutexUnlock(&pTask->lock); + streamMetaReleaseTask(pTask->pMeta, pTask); + return; + } + taosThreadMutexUnlock(&pTask->lock); taosThreadMutexLock(&pActiveInfo->lock); diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index dc9d2166e6..f8a4deb6b1 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -731,35 +731,50 @@ int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskI return code; } -int32_t streamBuildAndSendCheckpointUpdateMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId, STaskId* pHTaskId, - SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) { - SVUpdateCheckpointInfoReq* pReq = rpcMallocCont(sizeof(SVUpdateCheckpointInfoReq)); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; +int32_t streamSendChkptReportMsg(SStreamTask* pTask, SCheckpointInfo* pCheckpointInfo, int8_t dropRelHTask) { + int32_t code; + int32_t tlen = 0; + int32_t vgId = pTask->pMeta->vgId; + const char* id = pTask->id.idStr; + SActiveCheckpointInfo* pActive = pCheckpointInfo->pActiveInfo; + + SCheckpointReport req = {.streamId = pTask->id.streamId, + .taskId = pTask->id.taskId, + .nodeId = vgId, + .dropHTask = dropRelHTask, + .transId = pActive->transId, + .checkpointId = pActive->activeId, + .checkpointVer = pCheckpointInfo->processedVer, + .checkpointTs = pCheckpointInfo->startTs}; + + tEncodeSize(tEncodeStreamTaskChkptReport, &req, tlen, code); + if (code < 0) { + stError("s-task:%s vgId:%d encode stream task checkpoint-report failed, code:%s", id, vgId, tstrerror(code)); return -1; } - pReq->head.vgId = vgId; - pReq->taskId = pTaskId->taskId; - pReq->streamId = pTaskId->streamId; - pReq->dropRelHTask = dropRelHTask; - pReq->hStreamId = pHTaskId->streamId; - pReq->hTaskId = pHTaskId->taskId; - pReq->transId = pCheckpointInfo->pActiveInfo->transId; - - pReq->checkpointId = pCheckpointInfo->pActiveInfo->activeId; - pReq->checkpointVer = pCheckpointInfo->processedVer; - pReq->checkpointTs = pCheckpointInfo->startTs; - - SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_UPDATE_CHKPT, .pCont = pReq, .contLen = sizeof(SVUpdateCheckpointInfoReq)}; - int32_t code = tmsgPutToQueue(pMsgCb, WRITE_QUEUE, &msg); - - if (code != TSDB_CODE_SUCCESS) { - stError("vgId:%d task:0x%x failed to send update checkpoint info msg, code:%s", vgId, pTaskId->taskId, tstrerror(code)); - } else { - stDebug("vgId:%d task:0x%x build and send update checkpoint info msg msg", vgId, pTaskId->taskId); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + stError("s-task:%s vgId:%d encode stream task checkpoint-report msg failed, code:%s", id, vgId, + tstrerror(TSDB_CODE_OUT_OF_MEMORY)); + return -1; } - return code; + + SEncoder encoder; + tEncoderInit(&encoder, buf, tlen); + if ((code = tEncodeStreamTaskChkptReport(&encoder, &req)) < 0) { + rpcFreeCont(buf); + stError("s-task:%s vgId:%d encode stream task checkpoint-report msg failed, code:%s", id, vgId, tstrerror(code)); + return -1; + } + tEncoderClear(&encoder); + + SRpcMsg msg = {0}; + initRpcMsg(&msg, TDMT_MND_STREAM_CHKPT_REPORT, buf, tlen); + stDebug("s-task:%s vgId:%d build and send task checkpoint-report to mnode", id, vgId); + + tmsgSendReq(&pTask->info.mnodeEpset, &msg); + return 0; } STaskId streamTaskGetTaskId(const SStreamTask* pTask) { diff --git a/source/libs/stream/src/streammsg.c b/source/libs/stream/src/streammsg.c index 705406f044..a6ab6a60c2 100644 --- a/source/libs/stream/src/streammsg.c +++ b/source/libs/stream/src/streammsg.c @@ -594,6 +594,34 @@ int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { } if (tDecodeCStrTo(pDecoder, pTask->reserve) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t tEncodeStreamTaskChkptReport(SEncoder* pEncoder, const SCheckpointReport* pReq) { + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->nodeId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->checkpointId) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->checkpointVer) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->checkpointTs) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->transId) < 0) return -1; + if (tEncodeI8(pEncoder, pReq->dropHTask) < 0) return -1; + tEndEncode(pEncoder); + return 0; +} + +int32_t tDecodeStreamTaskChkptReport(SDecoder* pDecoder, SCheckpointReport* pReq) { + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->nodeId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->checkpointId) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->checkpointVer) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->checkpointTs) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->transId) < 0) return -1; + if (tDecodeI8(pDecoder, &pReq->dropHTask) < 0) return -1; tEndDecode(pDecoder); return 0; } \ No newline at end of file From 6effeabc8e2a55d6cff6243afc1985044b514b1c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 10:53:32 +0800 Subject: [PATCH 021/103] fix(stream): handle checkpoint-info-update in write queue. --- source/dnode/mgmt/mgmt_mnode/src/mmHandle.c | 2 +- source/dnode/mnode/impl/src/mndStreamTrans.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 3d5da860ad..e137bcbdec 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -241,7 +241,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_KILL_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index ff31aa0f7d..8137081631 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -159,7 +159,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnCo return NULL; } - mInfo("s-task:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id); + mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, pTrans->id); mndTransSetDbName(pTrans, pStream->sourceDb, pStream->targetSTbName); if (mndTransCheckConflict(pMnode, pTrans) != 0) { From 68aac5dee10c5eded4ee062d60e6ed5cce1fa266 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 11:12:24 +0800 Subject: [PATCH 022/103] other: fix syntax error. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndStreamUtil.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7ee34d502a..c1747f9379 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2569,7 +2569,7 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { if (pStream != NULL) { bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); if (conflict) { - mDebug("stream:%x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId); + mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId); } else { int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, *pReqTaskList); if (code == TSDB_CODE_SUCCESS) { // remove this entry diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 695c33f646..f7b33c5e75 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -789,7 +789,7 @@ void scanCheckpointReportInfo(SMnode* pMnode) { pInfo->streamId); } } else { - mDebug("stream:%x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); + mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); } } else { mDebug("stream:0x%" PRIx64 " %s %d/%d tasks send checkpoint-report, %d not send", pInfo->streamId, pStream->name, From 00eb6218257811d8eab3546100bc7c5a6e71da8d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 14:09:50 +0800 Subject: [PATCH 023/103] fix(stream): do scan checkpoint-report in write queue. --- include/common/tmsgdef.h | 2 +- source/dnode/mgmt/mgmt_snode/src/smHandle.c | 2 +- source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 2 +- source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndStream.c | 8 ++++-- source/dnode/mnode/impl/src/mndStreamHb.c | 13 ++++++++- source/dnode/mnode/impl/src/mndStreamUtil.c | 32 +++++++++++++++------ source/libs/stream/src/streamCheckpoint.c | 20 +------------ source/libs/stream/src/streamExec.c | 17 +++++------ 9 files changed, 52 insertions(+), 46 deletions(-) diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index a07830af19..1a25bac0c8 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -207,7 +207,7 @@ TD_DEF_MSG_TYPE(TDMT_MND_RESTORE_DNODE, "restore-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_PAUSE_STREAM, "pause-stream", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_RESUME_STREAM, "resume-stream", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHECKPOINT_TIMER, "stream-checkpoint-tmr", NULL, NULL) // unused,reserved + TD_DEF_MSG_TYPE(TDMT_MND_STREAM_UPDATE_CHKPT_EVT, "stream-update-chkpt-evt", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_BEGIN_CHECKPOINT, "stream-begin-checkpoint", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_CHKPT_REPORT, "stream-chkpt-report", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_STREAM_NODECHANGE_CHECK, "stream-nodechange-check", NULL, NULL) diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 856d7b2051..6a9f2f1275 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -95,7 +95,7 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 41b9352f18..7bc41559c3 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -967,7 +967,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToWriteQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_CHKPT_REPORT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_UPDATE_CHKPT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 6ca61265bb..9289909b19 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -130,7 +130,7 @@ int32_t mndStreamSetResetTaskAction(SMnode *pMnode, STrans *pTrans, SStreamO int32_t mndCreateStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream); int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream, SArray *pChkptInfoList); -void scanCheckpointReportInfo(SMnode *pMnode); +int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq); void removeTasksInBuf(SArray *pTaskIds, SStreamExecInfo *pExecInfo); SStreamTaskIter *createStreamTaskIter(SStreamObj *pStream); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index c1747f9379..ab72996058 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -115,10 +115,11 @@ int32_t mndInitStream(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamCheckpoint); - mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_REPORT, mndProcessCheckpointReport); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_REQ_CHKPT, mndProcessStreamReqCheckpoint); - mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHKPT_REPORT, mndProcessCheckpointReport); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_UPDATE_CHKPT_EVT, mndScanCheckpointReportInfo); mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_REPORT_CHECKPOINT, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_MND_STREAM_HEARTBEAT, mndProcessStreamHb); mndSetMsgHandle(pMnode, TDMT_MND_STREAM_NODECHANGE_CHECK, mndProcessNodeCheckReq); mndSetMsgHandle(pMnode, TDMT_MND_PAUSE_STREAM, mndProcessPauseStreamReq); @@ -1181,7 +1182,8 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) { int32_t numOfCheckpointTrans = 0; if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) { - return code; + terrno = TSDB_CODE_STREAM_TASK_IVLD_STATUS; + return -1; } SArray* pList = taosArrayInit(4, sizeof(SCheckpointInterval)); diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 7c07d003b7..2cb4111e97 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -22,6 +22,8 @@ typedef struct SFailedCheckpointInfo { int32_t transId; } SFailedCheckpointInfo; +static void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode); + static void updateStageInfo(STaskStatusEntry *pTaskEntry, int64_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeList); for (int32_t j = 0; j < numOfNodes; ++j) { @@ -326,7 +328,7 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { mndDropOrphanTasks(pMnode, pOrphanTasks); } - scanCheckpointReportInfo(pMnode); + mndStreamStartUpdateCheckpointInfo(pMnode); taosThreadMutexUnlock(&execInfo.lock); tCleanupStreamHbMsg(&req); @@ -346,3 +348,12 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { return TSDB_CODE_SUCCESS; } + +void mndStreamStartUpdateCheckpointInfo(SMnode *pMnode) { // here reuse the doCheckpointmsg + SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); + if (pMsg != NULL) { + int32_t size = sizeof(SMStreamDoCheckpointMsg); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_UPDATE_CHKPT_EVT, .pCont = pMsg, .contLen = size}; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index f7b33c5e75..7a45ce1f2a 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -754,8 +754,11 @@ int32_t mndStreamSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamObj return 0; } -void scanCheckpointReportInfo(SMnode* pMnode) { - void* pIter = NULL; +int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + void *pIter = NULL; + SArray *pDropped = taosArrayInit(4, sizeof(int64_t)); + mDebug("start to scan checkpoint report info"); while ((pIter = taosHashIterate(execInfo.pChkptStreams, pIter)) != NULL) { @@ -764,8 +767,8 @@ void scanCheckpointReportInfo(SMnode* pMnode) { STaskChkptInfo* pInfo = taosArrayGet(pList, 0); SStreamObj* pStream = mndGetStreamObj(pMnode, pInfo->streamId); if (pStream == NULL) { - mError("failed to acquire stream:0x%"PRIx64" remove it from checkpoint-report list", pInfo->streamId); - taosHashRemove(execInfo.pChkptStreams, &pInfo->streamId, sizeof(pInfo->streamId)); + mDebug("failed to acquire stream:0x%" PRIx64 " remove it from checkpoint-report list", pInfo->streamId); + taosArrayPush(pDropped, &pInfo->streamId); continue; } @@ -780,14 +783,13 @@ void scanCheckpointReportInfo(SMnode* pMnode) { if (!conflict) { int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry - taosHashRemove(execInfo.pChkptStreams, &pInfo->streamId, sizeof(pInfo->streamId)); - - int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); - mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", pInfo->streamId, numOfStreams); + taosArrayPush(pDropped, &pInfo->streamId); + mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", pInfo->streamId); } else { - mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet", + mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet", pInfo->streamId); } + break; } else { mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", pInfo->streamId); } @@ -799,4 +801,16 @@ void scanCheckpointReportInfo(SMnode* pMnode) { sdbRelease(pMnode->pSdb, pStream); } + if (taosArrayGetSize(pDropped) > 0) { + for (int32_t i = 0; i < taosArrayGetSize(pDropped); ++i) { + int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i); + taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId)); + } + + int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); + mDebug("drop %d stream(s) in checkpoint-report list, remain:%d", numOfStreams); + } + + taosArrayDestroy(pDropped); + return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 56ac794d8e..1583734a99 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -40,26 +40,8 @@ static void checkpointTriggerMonitorFn(void* param, void* tmrId); static SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, int32_t transId); -bool streamTaskIsAllUpstreamSendTrigger(SStreamTask* pTask) { - SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; - int32_t numOfUpstreams = taosArrayGetSize(pTask->upstreamInfo.pList); - bool allSend = true; - - taosThreadMutexLock(&pActiveInfo->lock); - int32_t numOfRecv = taosArrayGetSize(pActiveInfo->pReadyMsgList); - - if (numOfRecv < numOfUpstreams) { - stDebug("s-task:%s received checkpoint-trigger block, idx:%d, %d upstream tasks not send yet, total:%d", - pTask->id.idStr, pTask->info.selfChildId, (numOfUpstreams - numOfRecv), numOfUpstreams); - allSend = false; - } - - taosThreadMutexUnlock(&pActiveInfo->lock); - return allSend; -} - SStreamDataBlock* createChkptTriggerBlock(SStreamTask* pTask, int32_t checkpointType, int64_t checkpointId, - int32_t transId) { + int32_t transId) { SStreamDataBlock* pChkpoint = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SSDataBlock)); if (pChkpoint == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 1828409f89..98168abae1 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -615,7 +615,7 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB * appropriate batch of blocks should be handled in 5 to 10 sec. */ static int32_t doStreamExecTask(SStreamTask* pTask) { - const char* id = pTask->id.idStr; + const char* id = pTask->id.idStr; // merge multiple input data if possible in the input queue. stDebug("s-task:%s start to extract data block from inputQ", id); @@ -699,20 +699,15 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { if (type != STREAM_INPUT__CHECKPOINT) { doStreamTaskExecImpl(pTask, pInput); - } - - streamFreeQitem(pInput); - - // todo other thread may change the status + streamFreeQitem(pInput); + } else { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. - if (type == STREAM_INPUT__CHECKPOINT) { - // todo add lock + taosThreadMutexLock(&pTask->lock); SStreamTaskState* pState = streamTaskGetStatus(pTask); if (pState->state == TASK_STATUS__CK) { stDebug("s-task:%s checkpoint block received, set status:%s", id, pState->name); streamTaskBuildCheckpoint(pTask); - } else { - // todo refactor + } else { // todo refactor int32_t code = 0; if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { code = streamTaskSendCheckpointSourceRsp(pTask); @@ -727,6 +722,8 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } } + taosThreadMutexUnlock(&pTask->lock); + streamFreeQitem(pInput); return 0; } } From 0245d82e54990cd78a9128fe5b99745a941c67f1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 14:26:29 +0800 Subject: [PATCH 024/103] fix(stream): make sure that the unit test case can work. --- source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/mnode/impl/src/mndStreamHb.c | 4 +++- source/dnode/mnode/impl/src/mndStreamUtil.c | 7 ++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index ab72996058..2fb0505d86 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -2629,7 +2629,7 @@ static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) { } void mndInitStreamExecInfo(SMnode *pMnode, SStreamExecInfo *pExecInfo) { - if (pExecInfo->initTaskList) { + if (pExecInfo->initTaskList || pMnode == NULL) { return; } diff --git a/source/dnode/mnode/impl/src/mndStreamHb.c b/source/dnode/mnode/impl/src/mndStreamHb.c index 2cb4111e97..a79fe0cf0a 100644 --- a/source/dnode/mnode/impl/src/mndStreamHb.c +++ b/source/dnode/mnode/impl/src/mndStreamHb.c @@ -328,7 +328,9 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { mndDropOrphanTasks(pMnode, pOrphanTasks); } - mndStreamStartUpdateCheckpointInfo(pMnode); + if (pMnode != NULL) { // make sure that the unit test case can work + mndStreamStartUpdateCheckpointInfo(pMnode); + } taosThreadMutexUnlock(&execInfo.lock); tCleanupStreamHbMsg(&req); diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 7a45ce1f2a..915235ab47 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -801,14 +801,15 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { sdbRelease(pMnode->pSdb, pStream); } - if (taosArrayGetSize(pDropped) > 0) { - for (int32_t i = 0; i < taosArrayGetSize(pDropped); ++i) { + int32_t size = taosArrayGetSize(pDropped); + if (size > 0) { + for (int32_t i = 0; i < size; ++i) { int64_t streamId = *(int64_t *)taosArrayGet(pDropped, i); taosHashRemove(execInfo.pChkptStreams, &streamId, sizeof(streamId)); } int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); - mDebug("drop %d stream(s) in checkpoint-report list, remain:%d", numOfStreams); + mDebug("drop %d stream(s) in checkpoint-report list, remain:%d", size, numOfStreams); } taosArrayDestroy(pDropped); From d6e252513e4350fc5571b740e59cf2e327f3b14f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 14:33:59 +0800 Subject: [PATCH 025/103] fix(stream):fix syntax error. --- source/dnode/mnode/impl/src/mndStreamUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 915235ab47..031b97ce54 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -784,7 +784,7 @@ int32_t mndScanCheckpointReportInfo(SRpcMsg *pReq) { int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, pList); if (code == TSDB_CODE_SUCCESS || code == TSDB_CODE_ACTION_IN_PROGRESS) { // remove this entry taosArrayPush(pDropped, &pInfo->streamId); - mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", pInfo->streamId); + mDebug("stream:0x%" PRIx64 " removed", pInfo->streamId); } else { mDebug("stream:0x%" PRIx64 " not launch chkpt-meta update trans, due to checkpoint not finished yet", pInfo->streamId); From 7cc3eeee3d62d790e2adc226460ea6fcc2eabde3 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jun 2024 15:03:17 +0800 Subject: [PATCH 026/103] fix: first dynamic data filter issue --- source/libs/function/src/builtinsimpl.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 3fb298e1ea..699865a9bf 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -2366,7 +2366,10 @@ EFuncDataRequired firstDynDataReq(void* pRes, SDataBlockInfo* pBlockInfo) { } if (pResult->ts < pBlockInfo->window.skey) { return FUNC_DATA_REQUIRED_NOT_LOAD; - } else if (pResult->ts == pBlockInfo->window.skey && pResult->pkData) { + } else if (pResult->ts == pBlockInfo->window.skey) { + if (NULL == pResult->pkData) { + return FUNC_DATA_REQUIRED_NOT_LOAD; + } if (comparePkDataWithSValue(pResult->pkType, pResult->pkData, pBlockInfo->pks + 0, TSDB_ORDER_ASC) < 0) { return FUNC_DATA_REQUIRED_NOT_LOAD; } From 3b537367c924fd82d10887c6ca682d7ee3636c8e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 15:21:48 +0800 Subject: [PATCH 027/103] fix(stream): fix memory leak. --- source/dnode/mnode/impl/src/mndStream.c | 1 + source/dnode/mnode/impl/src/mndStreamUtil.c | 1 + 2 files changed, 2 insertions(+) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 2fb0505d86..09189bf593 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -149,6 +149,7 @@ void mndCleanupStream(SMnode *pMnode) { taosHashCleanup(execInfo.pTaskMap); taosHashCleanup(execInfo.transMgmt.pDBTrans); taosHashCleanup(execInfo.pTransferStateStreams); + taosHashCleanup(execInfo.pChkptStreams); taosThreadMutexDestroy(&execInfo.lock); mDebug("mnd stream exec info cleanup"); } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 031b97ce54..6745acac34 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -578,6 +578,7 @@ void mndInitExecInfo() { execInfo.pNodeList = taosArrayInit(4, sizeof(SNodeEntry)); taosHashSetFreeFp(execInfo.pTransferStateStreams, freeTaskList); + taosHashSetFreeFp(execInfo.pChkptStreams, freeTaskList); } void removeExpiredNodeInfo(const SArray *pNodeSnapshot) { From 8a6d58d339fb42d392bd311a53fc22c868b7e435 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Tue, 11 Jun 2024 07:16:23 +0000 Subject: [PATCH 028/103] support update enableWhiteList dynamicly --- source/common/src/tglobal.c | 6 +- source/dnode/mnode/impl/src/mndDnode.c | 219 +++++++++++++------------ 2 files changed, 113 insertions(+), 112 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index f034244c69..c68dc85c29 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -776,7 +776,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { // min free disk space used to check if the disk is full [50MB, 1GB] if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; // clang-format on @@ -1299,8 +1299,8 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi return 0; } -int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, - const char *envFile, char *apolloUrl, SArray *pArgs) { +int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, + SArray *pArgs) { if (tsCfg == NULL) osDefaultInit(); SConfig *pCfg = cfgInit(); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4de0086f96..d02aec98ca 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -14,9 +14,8 @@ */ #define _DEFAULT_SOURCE -#include -#include "tjson.h" #include "mndDnode.h" +#include #include "audit.h" #include "mndCluster.h" #include "mndDb.h" @@ -28,9 +27,10 @@ #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" +#include "taos_monitor.h" +#include "tjson.h" #include "tmisce.h" #include "tunit.h" -#include "taos_monitor.h" #define TSDB_DNODE_VER_NUMBER 2 #define TSDB_DNODE_RESERVE_SIZE 40 @@ -191,8 +191,8 @@ static SSdbRaw *mndDnodeActionEncode(SDnodeObj *pDnode) { SDB_SET_BINARY(pRaw, dataPos, pDnode->fqdn, TSDB_FQDN_LEN, _OVER) SDB_SET_BINARY(pRaw, dataPos, pDnode->machineId, TSDB_MACHINE_ID_LEN, _OVER) SDB_SET_RESERVE(pRaw, dataPos, TSDB_DNODE_RESERVE_SIZE, _OVER) - SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible - SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible + SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible + SDB_SET_INT16(pRaw, dataPos, 0, _OVER) // forward/backward compatible SDB_SET_DATALEN(pRaw, dataPos, _OVER); terrno = 0; @@ -536,49 +536,49 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { int32_t code = -1; char strClusterId[TSDB_CLUSTER_ID_LEN] = {0}; - sprintf(strClusterId, "%"PRId64, pMnode->clusterId); + sprintf(strClusterId, "%" PRId64, pMnode->clusterId); if (tDeserializeSStatisReq(pReq->pCont, pReq->contLen, &statisReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return code; } - if(tsMonitorLogProtocol){ + if (tsMonitorLogProtocol) { mInfo("process statis req,\n %s", statisReq.pCont); } - SJson* pJson = tjsonParse(statisReq.pCont); + SJson *pJson = tjsonParse(statisReq.pCont); int32_t ts_size = tjsonGetArraySize(pJson); - for(int32_t i = 0; i < ts_size; i++){ - SJson* item = tjsonGetArrayItem(pJson, i); + for (int32_t i = 0; i < ts_size; i++) { + SJson *item = tjsonGetArrayItem(pJson, i); - SJson* tables = tjsonGetObjectItem(item, "tables"); + SJson *tables = tjsonGetObjectItem(item, "tables"); int32_t tableSize = tjsonGetArraySize(tables); - for(int32_t i = 0; i < tableSize; i++){ - SJson* table = tjsonGetArrayItem(tables, i); + for (int32_t i = 0; i < tableSize; i++) { + SJson *table = tjsonGetArrayItem(tables, i); char tableName[MONITOR_TABLENAME_LEN] = {0}; tjsonGetStringValue(table, "name", tableName); - SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + SJson *metricGroups = tjsonGetObjectItem(table, "metric_groups"); int32_t size = tjsonGetArraySize(metricGroups); - for(int32_t i = 0; i < size; i++){ - SJson* item = tjsonGetArrayItem(metricGroups, i); + for (int32_t i = 0; i < size; i++) { + SJson *item = tjsonGetArrayItem(metricGroups, i); - SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + SJson *arrayTag = tjsonGetObjectItem(item, "tags"); int32_t tagSize = tjsonGetArraySize(arrayTag); - for(int32_t j = 0; j < tagSize; j++){ - SJson* item = tjsonGetArrayItem(arrayTag, j); + for (int32_t j = 0; j < tagSize; j++) { + SJson *item = tjsonGetArrayItem(arrayTag, j); char tagName[MONITOR_TAG_NAME_LEN] = {0}; tjsonGetStringValue(item, "name", tagName); - if(strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + if (strncmp(tagName, "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { tjsonDeleteItemFromObject(item, "value"); tjsonAddStringToObject(item, "value", strClusterId); } @@ -590,12 +590,12 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { char *pCont = tjsonToString(pJson); monSendContent(pCont); - if(pJson != NULL){ + if (pJson != NULL) { tjsonDelete(pJson); pJson = NULL; } - if(pCont != NULL){ + if (pCont != NULL) { taosMemoryFree(pCont); pCont = NULL; } @@ -603,132 +603,132 @@ static int32_t mndProcessStatisReq(SRpcMsg *pReq) { tFreeSStatisReq(&statisReq); return 0; -/* - SJson* pJson = tjsonParse(statisReq.pCont); + /* + SJson* pJson = tjsonParse(statisReq.pCont); - int32_t ts_size = tjsonGetArraySize(pJson); + int32_t ts_size = tjsonGetArraySize(pJson); - for(int32_t i = 0; i < ts_size; i++){ - SJson* item = tjsonGetArrayItem(pJson, i); + for(int32_t i = 0; i < ts_size; i++){ + SJson* item = tjsonGetArrayItem(pJson, i); - SJson* tables = tjsonGetObjectItem(item, "tables"); + SJson* tables = tjsonGetObjectItem(item, "tables"); - int32_t tableSize = tjsonGetArraySize(tables); - for(int32_t i = 0; i < tableSize; i++){ - SJson* table = tjsonGetArrayItem(tables, i); + int32_t tableSize = tjsonGetArraySize(tables); + for(int32_t i = 0; i < tableSize; i++){ + SJson* table = tjsonGetArrayItem(tables, i); - char tableName[MONITOR_TABLENAME_LEN] = {0}; - tjsonGetStringValue(table, "name", tableName); + char tableName[MONITOR_TABLENAME_LEN] = {0}; + tjsonGetStringValue(table, "name", tableName); - SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); + SJson* metricGroups = tjsonGetObjectItem(table, "metric_groups"); - int32_t size = tjsonGetArraySize(metricGroups); - for(int32_t i = 0; i < size; i++){ - SJson* item = tjsonGetArrayItem(metricGroups, i); + int32_t size = tjsonGetArraySize(metricGroups); + for(int32_t i = 0; i < size; i++){ + SJson* item = tjsonGetArrayItem(metricGroups, i); - SJson* arrayTag = tjsonGetObjectItem(item, "tags"); + SJson* arrayTag = tjsonGetObjectItem(item, "tags"); - int32_t tagSize = tjsonGetArraySize(arrayTag); + int32_t tagSize = tjsonGetArraySize(arrayTag); - char** labels = taosMemoryMalloc(sizeof(char*) * tagSize); - char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize); + char** labels = taosMemoryMalloc(sizeof(char*) * tagSize); + char** sample_labels = taosMemoryMalloc(sizeof(char*) * tagSize); - for(int32_t j = 0; j < tagSize; j++){ - SJson* item = tjsonGetArrayItem(arrayTag, j); + for(int32_t j = 0; j < tagSize; j++){ + SJson* item = tjsonGetArrayItem(arrayTag, j); - *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); - tjsonGetStringValue(item, "name", *(labels + j)); + *(labels + j) = taosMemoryMalloc(MONITOR_TAG_NAME_LEN); + tjsonGetStringValue(item, "name", *(labels + j)); - *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); - tjsonGetStringValue(item, "value", *(sample_labels + j)); - if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { - strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN); + *(sample_labels + j) = taosMemoryMalloc(MONITOR_TAG_VALUE_LEN); + tjsonGetStringValue(item, "value", *(sample_labels + j)); + if(strncmp(*(labels + j), "cluster_id", MONITOR_TAG_NAME_LEN) == 0) { + strncpy(*(sample_labels + j), strClusterId, MONITOR_TAG_VALUE_LEN); + } } - } - SJson* metrics = tjsonGetObjectItem(item, "metrics"); + SJson* metrics = tjsonGetObjectItem(item, "metrics"); - int32_t metricLen = tjsonGetArraySize(metrics); - for(int32_t j = 0; j < metricLen; j++){ - SJson *item = tjsonGetArrayItem(metrics, j); + int32_t metricLen = tjsonGetArraySize(metrics); + for(int32_t j = 0; j < metricLen; j++){ + SJson *item = tjsonGetArrayItem(metrics, j); - char name[MONITOR_METRIC_NAME_LEN] = {0}; - tjsonGetStringValue(item, "name", name); + char name[MONITOR_METRIC_NAME_LEN] = {0}; + tjsonGetStringValue(item, "name", name); - double value = 0; - tjsonGetDoubleValue(item, "value", &value); + double value = 0; + tjsonGetDoubleValue(item, "value", &value); - double type = 0; - tjsonGetDoubleValue(item, "type", &type); + double type = 0; + tjsonGetDoubleValue(item, "type", &type); - int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; - char* metricName = taosMemoryMalloc(metricNameLen); - memset(metricName, 0, metricNameLen); - sprintf(metricName, "%s:%s", tableName, name); + int32_t metricNameLen = strlen(name) + strlen(tableName) + 2; + char* metricName = taosMemoryMalloc(metricNameLen); + memset(metricName, 0, metricNameLen); + sprintf(metricName, "%s:%s", tableName, name); - taos_metric_t* metric = taos_collector_registry_get_metric(metricName); - if(metric == NULL){ - if(type == 0){ - metric = taos_counter_new(metricName, "", tagSize, (const char**)labels); - } - if(type == 1){ - metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels); - } - mTrace("fail to get metric from registry, new one metric:%p", metric); - - if(taos_collector_registry_register_metric(metric) == 1){ + taos_metric_t* metric = taos_collector_registry_get_metric(metricName); + if(metric == NULL){ if(type == 0){ - taos_counter_destroy(metric); + metric = taos_counter_new(metricName, "", tagSize, (const char**)labels); } if(type == 1){ - taos_gauge_destroy(metric); + metric = taos_gauge_new(metricName, "", tagSize, (const char**)labels); } + mTrace("fail to get metric from registry, new one metric:%p", metric); - metric = taos_collector_registry_get_metric(metricName); + if(taos_collector_registry_register_metric(metric) == 1){ + if(type == 0){ + taos_counter_destroy(metric); + } + if(type == 1){ + taos_gauge_destroy(metric); + } - mTrace("fail to register metric, get metric from registry:%p", metric); + metric = taos_collector_registry_get_metric(metricName); + + mTrace("fail to register metric, get metric from registry:%p", metric); + } + else{ + mTrace("succeed to register metric:%p", metric); + } } else{ - mTrace("succeed to register metric:%p", metric); + mTrace("get metric from registry:%p", metric); } - } - else{ - mTrace("get metric from registry:%p", metric); + + if(type == 0){ + taos_counter_add(metric, value, (const char**)sample_labels); + } + if(type == 1){ + taos_gauge_set(metric, value, (const char**)sample_labels); + } + + taosMemoryFreeClear(metricName); } - if(type == 0){ - taos_counter_add(metric, value, (const char**)sample_labels); - } - if(type == 1){ - taos_gauge_set(metric, value, (const char**)sample_labels); + for(int32_t j = 0; j < tagSize; j++){ + taosMemoryFreeClear(*(labels + j)); + taosMemoryFreeClear(*(sample_labels + j)); } - taosMemoryFreeClear(metricName); + taosMemoryFreeClear(sample_labels); + taosMemoryFreeClear(labels); } - - for(int32_t j = 0; j < tagSize; j++){ - taosMemoryFreeClear(*(labels + j)); - taosMemoryFreeClear(*(sample_labels + j)); - } - - taosMemoryFreeClear(sample_labels); - taosMemoryFreeClear(labels); } + } - } + code = 0; - code = 0; + _OVER: + if(pJson != NULL){ + tjsonDelete(pJson); + pJson = NULL; + } -_OVER: - if(pJson != NULL){ - tjsonDelete(pJson); - pJson = NULL; - } - - tFreeSStatisReq(&statisReq); - return code; - */ + tFreeSStatisReq(&statisReq); + return code; + */ } static int32_t mndUpdateDnodeObj(SMnode *pMnode, SDnodeObj *pDnode) { @@ -816,8 +816,9 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { bool reboot = (pDnode->rebootTime != statusReq.rebootTime); bool supportVnodesChanged = pDnode->numOfSupportVnodes != statusReq.numOfSupportVnodes; bool encryptKeyChanged = pDnode->encryptionKeyChksum != statusReq.clusterCfg.encryptionKeyChksum; + bool enableWhiteListChanged = statusReq.clusterCfg.enableWhiteList != (tsEnableWhiteList ? 1 : 0); bool needCheck = !online || dnodeChanged || reboot || supportVnodesChanged || - pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged; + pMnode->ipWhiteVer != statusReq.ipWhiteVer || encryptKeyChanged || enableWhiteListChanged; const STraceId *trace = &pReq->info.traceId; mGTrace("dnode:%d, status received, accessTimes:%d check:%d online:%d reboot:%d changed:%d statusSeq:%d", pDnode->id, From 6b1d4b842e78fd0c4473b568e320a3af93d9798a Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jun 2024 09:54:53 +0800 Subject: [PATCH 029/103] fix: case when memory leak --- source/libs/nodes/src/nodesCloneFuncs.c | 1 + source/libs/nodes/src/nodesUtilFuncs.c | 2 ++ tests/parallel_test/cases.task | 1 + tests/script/tsim/query/nestedJoinView.sim | 19 +++++++++++++++++++ 4 files changed, 23 insertions(+) create mode 100644 tests/script/tsim/query/nestedJoinView.sim diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 84d3f734fe..f02fefd977 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -843,6 +843,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) { CLONE_NODE_FIELD_EX(pLimit, SLimitNode*); COPY_CHAR_ARRAY_FIELD(stmtName); COPY_SCALAR_FIELD(precision); + COPY_SCALAR_FIELD(isSubquery); COPY_SCALAR_FIELD(isEmptyResult); COPY_SCALAR_FIELD(timeLineResMode); COPY_SCALAR_FIELD(timeLineFromOrderBy); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index beedffc4f2..c081841b3b 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -963,12 +963,14 @@ void nodesDestroyNode(SNode* pNode) { break; case QUERY_NODE_WHEN_THEN: { SWhenThenNode* pWhenThen = (SWhenThenNode*)pNode; + destroyExprNode((SExprNode*)pNode); nodesDestroyNode(pWhenThen->pWhen); nodesDestroyNode(pWhenThen->pThen); break; } case QUERY_NODE_CASE_WHEN: { SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)pNode; + destroyExprNode((SExprNode*)pNode); nodesDestroyNode(pCaseWhen->pCase); nodesDestroyNode(pCaseWhen->pElse); nodesDestroyList(pCaseWhen->pWhenThenList); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 1eaf17ccb1..307260cf6a 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1485,6 +1485,7 @@ ,,y,script,./test.sh -f tsim/view/view.sim ,,y,script,./test.sh -f tsim/query/cache_last.sim ,,y,script,./test.sh -f tsim/query/const.sim +,,y,script,./test.sh -f tsim/query/nestedJoinView.sim #develop test diff --git a/tests/script/tsim/query/nestedJoinView.sim b/tests/script/tsim/query/nestedJoinView.sim new file mode 100644 index 0000000000..efdec9fcbc --- /dev/null +++ b/tests/script/tsim/query/nestedJoinView.sim @@ -0,0 +1,19 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; + +sql CREATE TABLE `resource_info` ( job_id_ts TIMESTAMP , role VARCHAR(20) primary key, start_time TIMESTAMP, ip VARCHAR(15), cpu FLOAT, memory FLOAT, io_write FLOAT, io_read FLOAT, net_write FLOAT, net_read FLOAT) TAGS ( end_time TIMESTAMP); + +sql CREATE STABLE `test_results` ( `job_id_ts` TIMESTAMP , `end_time` VARCHAR(40) PRIMARY KEY, `job_id` BIGINT, `time_cost` FLOAT, `write_speed` FLOAT, `qps` FLOAT, `min_delay` FLOAT, `p90_delay` FLOAT, `p95_delay` FLOAT, `p99_delay` FLOAT, `max_delay` FLOAT, `avg_delay` FLOAT, `hostname` VARCHAR(15), `tdengine_commit_id` VARCHAR(50), `tdinternal_commit_id` VARCHAR(50), `load_type` VARCHAR(50), `cpu` FLOAT, `memory` FLOAT, `io_write` FLOAT, `io_read` FLOAT) TAGS ( `branch` VARCHAR(50), `scenario` VARCHAR(50), `test_case` VARCHAR(1000), `env_id` INT, `type` VARCHAR(50)); + +sql CREATE TABLE `job_info` ( `start_time` TIMESTAMP , `finish_time` TIMESTAMP , `job_id` INT, `job_status` VARCHAR(20), `test_type` VARCHAR(50), `environment` INT, `version` VARCHAR(20), `tdengine_commit_id` VARCHAR(50), `tdinternal_commit_id` VARCHAR(50), `type` VARCHAR(50), `scenario` VARCHAR(50), `note` VARCHAR(500), `version_number` VARCHAR(20)); + +sql create view abc as select * from ( select a.job_id, a.start_time as job_start_time, a.finish_time as job_end_time, a.job_status, a.test_type, a.environment, case when a.version_number <> null then a.version else CONCAT(a.version,'_',a.version_number) end as version_info, a.tdengine_commit_id, a.tdinternal_commit_id, a.type, a.scenario, a.note, a.version_number, b.end_time as tc_end_time, b.time_cost, b.write_speed, b.qps, b.min_delay, b.p90_delay, b.p95_delay, b.p99_delay, b.`max_delay`, b.avg_delay, b.hostname, b.load_type, b.scenario, b.test_case, b.type from job_info a, test_results b where a.start_time=b.job_id_ts and a.job_status='finished') s1 inner join resource_info s2 on s1.job_start_time=s2.job_id_ts and s1.job_id=2 and s1.tc_end_time=s2.end_time; + +sql select * from abc; + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 7e0c4eeb7dc3589e8a08863c5c48107c4e60beca Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 11 Jun 2024 15:36:41 +0800 Subject: [PATCH 030/103] fix: join test case --- .../2-query/agg_group_AlwaysReturnValue.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py index f7c82fc859..0cac9cb03b 100755 --- a/tests/system-test/2-query/agg_group_AlwaysReturnValue.py +++ b/tests/system-test/2-query/agg_group_AlwaysReturnValue.py @@ -1615,47 +1615,47 @@ class TDTestCase(TDTestCase): time.sleep(1) tdSql.query('select tbname,count(*) from d2') tdSql.checkData(0, 1, 0) - - tdSql.query('select b.tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') - tdSql.checkData(0, 0, 'd2') - tdSql.checkData(0, 1, 0) - tdSql.query('select meter1.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') - tdSql.checkData(0, 0, 'd1') - tdSql.checkData(1, 0, 'd2') - tdSql.checkData(0, 1, 0) - tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter2.tbname order by meter2.tbname') - tdSql.checkData(0, 0, 'd21') - tdSql.checkData(1, 0, 'd22') - tdSql.checkData(0, 1, 0) - tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by meter2.tbname order by meter2.tbname') - tdSql.checkData(0, 0, 'd21') - tdSql.checkData(1, 0, 'd22') - tdSql.checkData(0, 1, 0) - tdSql.query('select m2.tbname, count(*) from meter1 m1, meter2 m2 where m1.ts = m2.ts partition by m2.tbname order by m2.tbname') - tdSql.checkData(0, 0, 'd21') - tdSql.checkData(1, 0, 'd22') - tdSql.checkData(0, 1, 0) - - tdSql.execute('insert into `d1` VALUES (now, 1) `d21` VALUES (now, 21)') - tdSql.execute('insert into `d1` VALUES (now, 2) `d21` VALUES (now, 22)') - tdSql.execute('insert into `d1` VALUES (now, 3) `d21` VALUES (now, 32)') + + tdSql.execute('insert into `d1` VALUES (now, 1)') + tdSql.execute('insert into `d1` VALUES (now+1s, 2)') + tdSql.execute('insert into `d1` VALUES (now+2s, 3)') + tdSql.execute('insert into `d2` VALUES (now+3s, 11)') + tdSql.execute('insert into `d2` VALUES (now+4s, 22)') + tdSql.execute('insert into `d2` VALUES (now+5s, 33)') + tdSql.execute('insert into `d21` select * from `d1`') # tdSql.query('select b.tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') # tdSql.checkData(0, 0, 'd2') tdSql.query('select meter1.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') tdSql.checkData(0, 0, 'd1') - tdSql.checkData(1, 0, 'd2') + tdSql.checkData(0, 1, 3) + # tdSql.checkData(1, 0, 'd2') tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter2.tbname order by meter2.tbname') tdSql.checkData(0, 0, 'd21') - tdSql.checkData(1, 0, 'd22') + tdSql.checkData(0, 1, 3) + # tdSql.checkData(1, 0, 'd22') tdSql.query('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts partition by meter2.tbname order by meter2.tbname') tdSql.checkData(0, 0, 'd21') - tdSql.checkData(1, 0, 'd22') + tdSql.checkData(0, 1, 3) + # tdSql.checkData(1, 0, 'd22') tdSql.query('select m2.tbname, count(*) from meter1 m1, meter2 m2 where m1.ts = m2.ts partition by m2.tbname order by m2.tbname') tdSql.checkData(0, 0, 'd21') + tdSql.checkData(0, 1, 3) + # tdSql.checkData(1, 0, 'd22') + + tdSql.execute('insert into `d22` select * from `d1`') + + tdSql.query('select meter1.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') + tdSql.checkData(0, 0, 'd1') + tdSql.checkData(0, 1, 6) + + tdSql.query('select m2.tbname, count(*) from meter1 m1, meter2 m2 where m1.ts = m2.ts partition by m2.tbname order by m2.tbname') + tdSql.checkData(0, 0, 'd21') + tdSql.checkData(0, 1, 3) tdSql.checkData(1, 0, 'd22') - + tdSql.checkData(1, 1, 3) + tdSql.error('select tbname, count(*) from d1 a, d2 b where a.ts = b.ts group by b.tbname') # tdSql.error('select meter2.tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter1.tbname order by meter1.tbname') tdSql.error('select tbname, count(*) from meter1, meter2 where meter1.ts = meter2.ts group by meter2.tbname order by meter2.tbname') @@ -1671,9 +1671,9 @@ class TDTestCase(TDTestCase): # self.create_tables() # self.insert_data() - self.testTBNameUseJoin() - self.dropandcreateDB_random("nested", 1) #self.testTBNameUseJoin() + self.dropandcreateDB_random("nested", 1) + self.testTBNameUseJoin() self.modify_tables() From 781b644a8c66c0121b0866cd1bb3b16695138faa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 16:34:47 +0800 Subject: [PATCH 031/103] fix(stream):fix memory leak and update test cases. --- source/dnode/mnode/impl/src/mndStream.c | 1 + tests/system-test/2-query/compa4096_tsma.json | 140 ++++++++++-------- tests/system-test/2-query/tsma.py | 11 +- .../system-test/8-stream/stream_multi_agg.py | 2 +- 4 files changed, 84 insertions(+), 70 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 09189bf593..7b00978a62 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1154,6 +1154,7 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { } removeTasksInBuf(pInvalidList, &execInfo); + taosArrayDestroy(pInvalidList); taosThreadMutexUnlock(&execInfo.lock); return ready ? 0 : -1; diff --git a/tests/system-test/2-query/compa4096_tsma.json b/tests/system-test/2-query/compa4096_tsma.json index 66d98ceebe..993eb8ae58 100644 --- a/tests/system-test/2-query/compa4096_tsma.json +++ b/tests/system-test/2-query/compa4096_tsma.json @@ -1,66 +1,80 @@ { - "filetype": "insert", - "cfgdir": "/etc/taos", - "host": "localhost", - "port": 6030, - "rest_port": 6041, - "user": "root", - "password": "taosdata", - "thread_count": 100, - "create_table_thread_count": 24, - "result_file": "taosBenchmark_result.log", - "confirm_parameter_prompt": "no", - "insert_interval": 0, - "num_of_records_per_req": 1000000, - "max_sql_len": 1024000, - "databases": [ - { - "dbinfo": { - "name": "db4096", - "drop": "yes", - "replica": 1, - "duration": 10, - "precision": "ms", - "keep": 3650, - "comp": 2, - "vgroups": 2, - "buffer": 1000 - }, - "super_tables": [ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "localhost", + "port": 6030, + "rest_port": 6041, + "user": "root", + "password": "taosdata", + "thread_count": 100, + "create_table_thread_count": 24, + "result_file": "taosBenchmark_result.log", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "num_of_records_per_req": 1000000, + "max_sql_len": 1024000, + "databases": [ { - "name": "stb0", - "child_table_exists": "no", - "childtable_count":2, - "childtable_prefix": "ctb0", - "escape_character": "no", - "auto_create_table": "no", - "batch_create_tbl_num": 500, - "data_source": "rand", - "insert_mode": "taosc", - "rollup": null, - "interlace_rows": 0, - "line_protocol": null, - "tcp_transfer": "no", - "insert_rows": 10, - "childtable_limit": 0, - "childtable_offset": 0, - "rows_per_tbl": 0, - "max_sql_len": 1048576, - "disorder_ratio": 0, - "disorder_range": 1000, - "timestamp_step": 1000, - "start_timestamp": "2022-10-22 17:20:36", - "sample_format": "csv", - "sample_file": "./sample.csv", - "tags_file": "", - "columns": [{ "type": "INT","count": 4093}], - "tags": [{"type": "TINYINT", "count": 1},{"type": "NCHAR","count": 1}] + "dbinfo": { + "name": "db4096", + "drop": "yes", + "replica": 1, + "duration": 10, + "precision": "ms", + "keep": 3650, + "comp": 2, + "vgroups": 2, + "buffer": 1000 + }, + "super_tables": [ + { + "name": "stb0", + "child_table_exists": "no", + "childtable_count": 2, + "childtable_prefix": "ctb0", + "escape_character": "no", + "auto_create_table": "no", + "batch_create_tbl_num": 500, + "data_source": "rand", + "insert_mode": "taosc", + "rollup": null, + "interlace_rows": 0, + "line_protocol": null, + "tcp_transfer": "no", + "insert_rows": 10, + "childtable_limit": 0, + "childtable_offset": 0, + "rows_per_tbl": 0, + "max_sql_len": 1048576, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1000, + "start_timestamp": "2022-10-22 17:20:36", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [ + { + "type": "INT", + "count": 4093 + } + ], + "tags": [ + { + "type": "TINYINT", + "count": 1 + }, + { + "type": "NCHAR", + "count": 1 + } + ] + } + ] } - ] - } - ], - "prepare_rand": 10000, - "chinese": "no", - "streams": false, - "test_log": "/root/testlog/" -} + ], + "prepare_rand": 10000, + "chinese": "no", + "streams": false, + "test_log": "/root/testlog/" +} \ No newline at end of file diff --git a/tests/system-test/2-query/tsma.py b/tests/system-test/2-query/tsma.py index f80aab0e82..d7dc1d24f3 100644 --- a/tests/system-test/2-query/tsma.py +++ b/tests/system-test/2-query/tsma.py @@ -1270,15 +1270,14 @@ class TDTestCase: def test_drop_tsma(self): function_name = sys._getframe().f_code.co_name tdLog.debug(f'-----{function_name}------') - self.create_tsma('tsma1', 'test', 'meters', [ - 'avg(c1)', 'avg(c2)'], '5m') + self.create_tsma('tsma1', 'test', 'meters', ['avg(c1)', 'avg(c2)'], '5m') self.create_recursive_tsma('tsma1', 'tsma2', 'test', '15m', 'meters') # drop recursive tsma first tdSql.error('drop tsma test.tsma1', -2147482491) tdSql.execute('drop tsma test.tsma2', queryTimes=1) tdSql.execute('drop tsma test.tsma1', queryTimes=1) - self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-checkpo') + self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-chkpt-u') tdSql.execute('drop database test', queryTimes=1) self.init_data() @@ -1319,7 +1318,7 @@ class TDTestCase: 'create tsma tsma1 on nsdb.meters function(avg(c1), avg(c2), avg(t3)) interval(5m)', -2147471096) tdSql.execute('alter table nsdb.meters drop tag t3', queryTimes=1) - self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-checkpo') + self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-chkpt-u') tdSql.execute('drop database nsdb') # drop norm table @@ -1346,7 +1345,7 @@ class TDTestCase: # test drop stream tdSql.error('drop stream tsma1', -2147471088) ## TSMA must be dropped first - self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-checkpo') + self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-chkpt-u') tdSql.execute('drop database test', queryTimes=1) self.init_data() @@ -1449,7 +1448,7 @@ class TDTestCase: tdSql.error( 'create tsma tsma1 on test.meters function(avg(c1), avg(c2)) interval(2h)', -2147471097) - self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-checkpo') + self.wait_query('show transactions', 0, 10, lambda row: row[3] != 'stream-chkpt-u') tdSql.execute('drop database nsdb') def test_create_tsma_on_norm_table(self): diff --git a/tests/system-test/8-stream/stream_multi_agg.py b/tests/system-test/8-stream/stream_multi_agg.py index acb80f528b..4e8d965f32 100644 --- a/tests/system-test/8-stream/stream_multi_agg.py +++ b/tests/system-test/8-stream/stream_multi_agg.py @@ -68,7 +68,7 @@ class TDTestCase: # create stream tdSql.execute("use db", queryTimes=100) tdSql.execute("create stream stream1 fill_history 1 into sta as select count(*) as cnt from meters interval(10a);",show=True) - time.sleep(5) + time.sleep(10) sql = "select count(*) from sta" # loop wait max 60s to check count is ok From b22679c9418c1733c01eb036cf28bad05edbebe8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 17:13:44 +0800 Subject: [PATCH 032/103] fix(stream): do NOT remove the stream if the stream object is not accessiable, since the deploy trans may not completed yet. --- source/dnode/mnode/impl/src/mndStream.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 7b00978a62..136839451f 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1115,30 +1115,18 @@ static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { continue; } - // for stopped stream task entry, we do additional check for it if (pEntry->status == TASK_STATUS__STOP) { - bool invalid = false; for(int32_t j = 0; j < taosArrayGetSize(pInvalidList); ++j) { STaskId* pId = taosArrayGet(pInvalidList, j); if (pEntry->id.streamId == pId->streamId) { taosArrayPush(pInvalidList, &pEntry->id); - invalid = true; break; } } - - if (!invalid) { - SStreamObj *pObj = mndGetStreamObj(pMnode, pEntry->id.streamId); - if (pObj == NULL) { - taosArrayPush(pInvalidList, &pEntry->id); - } else { - mndReleaseStream(pMnode, pObj); - } - } } if (pEntry->status != TASK_STATUS__READY) { - mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", + mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s, checkpoint not issued", pEntry->id.streamId, (int32_t)pEntry->id.taskId, pEntry->nodeId, streamTaskGetStatusStr(pEntry->status)); ready = false; } From 70db803aecef56c68094f185f48af4993ee65eab Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 11 Jun 2024 17:24:29 +0800 Subject: [PATCH 033/103] fix:[TD-30365] ci case error & drop topic error if vnode is splitted --- source/dnode/mnode/impl/inc/mndDef.h | 4 +-- source/dnode/mnode/impl/src/mndConsumer.c | 15 ++++------ source/dnode/mnode/impl/src/mndSubscribe.c | 32 +++++++++++--------- source/dnode/mnode/impl/src/mndTopic.c | 8 ++--- source/dnode/mnode/impl/src/mndTrans.c | 34 +++++++++++----------- source/dnode/vnode/src/inc/tq.h | 2 +- source/dnode/vnode/src/tq/tq.c | 29 ++++++++++-------- source/dnode/vnode/src/tq/tqMeta.c | 6 ++-- 8 files changed, 67 insertions(+), 63 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 5c21e9b22b..81772635fc 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -102,8 +102,8 @@ typedef enum { TRN_CONFLICT_GLOBAL = 1, TRN_CONFLICT_DB = 2, TRN_CONFLICT_DB_INSIDE = 3, - TRN_CONFLICT_TOPIC = 4, - TRN_CONFLICT_TOPIC_INSIDE = 5, +// TRN_CONFLICT_TOPIC = 4, +// TRN_CONFLICT_TOPIC_INSIDE = 5, TRN_CONFLICT_ARBGROUP = 6, } ETrnConflct; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 3eef2afcc1..9a7a8155ec 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -91,7 +91,7 @@ void mndSendConsumerMsg(SMnode *pMnode, int64_t consumerId, uint16_t msgType, SR } } -static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode *pMnode, const char *pUser, +static int32_t validateTopics(const SArray *pTopicList, SMnode *pMnode, const char *pUser, bool enableReplay) { SMqTopicObj *pTopic = NULL; int32_t code = 0; @@ -135,11 +135,6 @@ static int32_t validateTopics(STrans *pTrans, const SArray *pTopicList, SMnode * } } - mndTransSetDbName(pTrans, pOneTopic, NULL); - if (mndTransCheckConflict(pMnode, pTrans) != 0) { - code = -1; - goto FAILED; - } mndReleaseTopic(pMnode, pTopic); } @@ -177,12 +172,12 @@ static int32_t mndProcessConsumerRecoverMsg(SRpcMsg *pMsg) { goto END; } - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC, pMsg, "recover-csm"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "recover-csm"); if (pTrans == NULL) { code = -1; goto END; } - code = validateTopics(pTrans, pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false); + code = validateTopics(pConsumer->assignedTopics, pMnode, pMsg->info.conn.user, false); if (code != 0) { goto END; } @@ -675,13 +670,13 @@ int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto _over; } - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "subscribe"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pMsg, "subscribe"); if (pTrans == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _over; } - code = validateTopics(pTrans, subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay); + code = validateTopics(subscribe.topicNames, pMnode, pMsg->info.conn.user, subscribe.enableReplay); if (code != TSDB_CODE_SUCCESS) { goto _over; } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 9f84e25c9f..ffb723756c 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -618,13 +618,13 @@ static int32_t mndPersistRebResult(SMnode *pMnode, SRpcMsg *pMsg, const SMqRebOu char cgroup[TSDB_CGROUP_LEN] = {0}; mndSplitSubscribeKey(pOutput->pSub->key, topic, cgroup, true); - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "tmq-reb"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "tmq-reb"); if (pTrans == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto END; } - mndTransSetDbName(pTrans, topic, cgroup); + mndTransSetDbName(pTrans, pOutput->pSub->dbName, cgroup); code = mndTransCheckConflict(pMnode, pTrans); if (code != 0) { goto END; @@ -908,22 +908,26 @@ END: } static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STrans *pTrans){ - // iter all vnode to delete handle - int32_t sz = taosArrayGetSize(pSub->unassignedVgs); - for (int32_t i = 0; i < sz; i++) { - SMqVgEp *pVgEp = taosArrayGetP(pSub->unassignedVgs, i); - SVgObj *pVgObj = mndAcquireVgroup(pMnode, pVgEp->vgId); - if (pVgObj == NULL) { - mError("sendDeleteSubToVnode %s failed since vg %d doesn't exist", pSub->key, pVgEp->vgId); + void* pIter = NULL; + SVgObj* pVgObj = NULL; + while (1) { + pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void**)&pVgObj); + if (pIter == NULL) { + break; + } + + if (!mndVgroupInDb(pVgObj, pSub->dbUid)) { + sdbRelease(pMnode->pSdb, pVgObj); continue; } SMqVDeleteReq *pReq = taosMemoryCalloc(1, sizeof(SMqVDeleteReq)); if(pReq == NULL){ terrno = TSDB_CODE_OUT_OF_MEMORY; + sdbRelease(pMnode->pSdb, pVgObj); return -1; } - pReq->head.vgId = htonl(pVgEp->vgId); - pReq->vgId = pVgEp->vgId; + pReq->head.vgId = htonl(pVgObj->vgId); + pReq->vgId = pVgObj->vgId; pReq->consumerId = -1; memcpy(pReq->subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); @@ -934,7 +938,7 @@ static int32_t sendDeleteSubToVnode(SMnode *pMnode, SMqSubscribeObj *pSub, STran action.msgType = TDMT_VND_TMQ_DELETE_SUB; action.acceptableCode = TSDB_CODE_MND_VGROUP_NOT_EXIST; - mndReleaseVgroup(pMnode, pVgObj); + sdbRelease(pMnode->pSdb, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; @@ -996,7 +1000,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { goto end; } - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC_INSIDE, pMsg, "drop-cgroup"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pMsg, "drop-cgroup"); if (pTrans == NULL) { mError("cgroup: %s on topic:%s, failed to drop since %s", dropReq.cgroup, dropReq.topic, terrstr()); code = -1; @@ -1004,7 +1008,7 @@ static int32_t mndProcessDropCgroupReq(SRpcMsg *pMsg) { } mInfo("trans:%d, used to drop cgroup:%s on topic %s", pTrans->id, dropReq.cgroup, dropReq.topic); - mndTransSetDbName(pTrans, dropReq.topic, dropReq.cgroup); + mndTransSetDbName(pTrans, pSub->dbName, dropReq.cgroup); code = mndTransCheckConflict(pMnode, pTrans); if (code != 0) { goto end; diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 8a06b4a613..bcb38a3902 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -422,14 +422,14 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * SQueryPlan *pPlan = NULL; SMqTopicObj topicObj = {0}; - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "create-topic"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB, pReq, "create-topic"); if (pTrans == NULL) { mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); code = -1; goto _OUT; } - mndTransSetDbName(pTrans, pCreate->name, NULL); + mndTransSetDbName(pTrans, pDb->name, NULL); code = mndTransCheckConflict(pMnode, pTrans); if (code != 0) { goto _OUT; @@ -779,14 +779,14 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } } - pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_TOPIC, pReq, "drop-topic"); + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-topic"); if (pTrans == NULL) { mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); code = -1; goto end; } - mndTransSetDbName(pTrans, pTopic->name, NULL); + mndTransSetDbName(pTrans, pTopic->db, NULL); code = mndTransCheckConflict(pMnode, pTrans); if (code != 0) { goto end; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 8b01d296a3..d80164bcad 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -830,26 +830,26 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { } } - if (pNew->conflict == TRN_CONFLICT_TOPIC) { - if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; - if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { - if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; - } - } - if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) { - if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; - if (pTrans->conflict == TRN_CONFLICT_TOPIC) { - if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; - } - if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { - if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0) - conflict = true; - } - } +// if (pNew->conflict == TRN_CONFLICT_TOPIC) { +// if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; +// if (pTrans->conflict == TRN_CONFLICT_TOPIC || pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { +// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; +// } +// } +// if (pNew->conflict == TRN_CONFLICT_TOPIC_INSIDE) { +// if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; +// if (pTrans->conflict == TRN_CONFLICT_TOPIC) { +// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0) conflict = true; +// } +// if (pTrans->conflict == TRN_CONFLICT_TOPIC_INSIDE) { +// if (strcasecmp(pNew->dbname, pTrans->dbname) == 0 && strcasecmp(pNew->stbname, pTrans->stbname) == 0) +// conflict = true; +// } +// } if (pNew->conflict == TRN_CONFLICT_ARBGROUP) { if (pTrans->conflict == TRN_CONFLICT_GLOBAL) conflict = true; if (pTrans->conflict == TRN_CONFLICT_ARBGROUP) { - void *pIter = taosHashIterate(pNew->arbGroupIds, NULL); + pIter = taosHashIterate(pNew->arbGroupIds, NULL); while (pIter != NULL) { int32_t groupId = *(int32_t *)pIter; if (taosHashGet(pTrans->arbGroupIds, &groupId, sizeof(int32_t)) != NULL) { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 68d966add6..08d32b2b81 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -133,7 +133,7 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaRestoreCheckInfo(STQ* pTq); int32_t tqMetaGetHandle(STQ* pTq, const char* key); -int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle, int64_t snapshotVer); +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle); STqOffsetStore* tqOffsetOpen(STQ* pTq); int32_t tqMetaTransform(STQ* pTq); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 58085e2cc2..083e0b28f3 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -649,7 +649,19 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg tqInfo("vgId:%d, tq process sub req:%s, Id:0x%" PRIx64 " -> Id:0x%" PRIx64, pTq->pVnode->config.vgId, req.subKey, req.oldConsumerId, req.newConsumerId); - STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + STqHandle* pHandle = NULL; + while (1) { + pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle) { + break; + } + taosRLockLatch(&pTq->lock); + ret = tqMetaGetHandle(pTq, req.subKey); + taosRUnLockLatch(&pTq->lock); + if (ret < 0) { + break; + } + } if (pHandle == NULL) { if (req.oldConsumerId != -1) { tqError("vgId:%d, build new consumer handle %s for consumer:0x%" PRIx64 ", but old consumerId:0x%" PRIx64, @@ -660,7 +672,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg goto end; } STqHandle handle = {0}; - ret = tqCreateHandle(pTq, &req, &handle, walGetCommittedVer(pTq->pVnode->pWal)); + ret = tqCreateHandle(pTq, &req, &handle); if (ret < 0) { tqDestroyTqHandle(&handle); goto end; @@ -684,17 +696,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } else { tqInfo("vgId:%d switch consumer from Id:0x%" PRIx64 " to Id:0x%" PRIx64, req.vgId, pHandle->consumerId, req.newConsumerId); + atomic_store_64(&pHandle->consumerId, req.newConsumerId); + atomic_store_32(&pHandle->epoch, 0); tqUnregisterPushHandle(pTq, pHandle); - - // update handle to avoid req->qmsg changed if spilt vnode is failed - STqHandle handle = {0}; - ret = tqCreateHandle(pTq, &req, &handle, pHandle->snapshotVer); - if (ret < 0) { - tqDestroyTqHandle(&handle); - taosWUnLockLatch(&pTq->lock); - goto end; - } - ret = tqMetaSaveHandle(pTq, req.subKey, &handle); + ret = tqMetaSaveHandle(pTq, req.subKey, pHandle); } taosWUnLockLatch(&pTq->lock); break; diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 5162136591..ce3308f0ac 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -346,7 +346,7 @@ end: return code; } -int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle, int64_t snapshotVer){ +int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle){ int32_t vgId = TD_VID(pTq->pVnode); memcpy(handle->subKey, req->subKey, TSDB_SUBSCRIBE_KEY_LEN); @@ -364,12 +364,12 @@ int32_t tqCreateHandle(STQ* pTq, SMqRebVgReq* req, STqHandle* handle, int64_t sn handle->execHandle.execTb.qmsg = taosStrdup(req->qmsg); } - handle->snapshotVer = snapshotVer; + handle->snapshotVer = walGetCommittedVer(pTq->pVnode->pWal); if(buildHandle(pTq, handle) < 0){ return -1; } - tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d", handle->subKey, handle->consumerId, vgId); + tqInfo("tqCreateHandle %s consumer 0x%" PRIx64 " vgId:%d, snapshotVer:%" PRId64, handle->subKey, handle->consumerId, vgId, handle->snapshotVer); return taosHashPut(pTq->pHandle, handle->subKey, strlen(handle->subKey), handle, sizeof(STqHandle)); } From 5b0bb82ed046b97972adcaed59bc0820483be78d Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 11 Jun 2024 17:46:53 +0800 Subject: [PATCH 034/103] fix: reserve log file name --- source/util/src/tlog.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 0c09174970..1dad1d9519 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -267,21 +267,31 @@ static void taosUnLockLogFile(TdFilePtr pFile) { } } -static void taosKeepOldLog(char *oldName) { - if (tsLogKeepDays == 0) return; - - int64_t fileSec = taosGetTimestampSec(); - char fileName[LOG_FILE_NAME_LEN + 20]; - snprintf(fileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); - - (void)taosRenameFile(oldName, fileName); - - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64 ".gz", tsLogObj.logName, fileSec); - if (taosCompressFile(fileName, compressFileName) == 0) { - (void)taosRemoveFile(fileName); +static void taosReserveOldLog(char *oldName, char *keepName) { + if (tsLogKeepDays == 0) { + keepName[0] = 0; + return; } + int32_t code = 0; + int64_t fileSec = taosGetTimestampSec(); + snprintf(keepName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); + if ((code = taosRenameFile(oldName, keepName))) { + keepName[0] = 0; + uError("failed to rename file:%s to %s since %s", oldName, keepName, tstrerror(code)); + } +} + +static void taosKeepOldLog(char *oldName) { + if (oldName[0] == '\0') goto _end; + + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.gz", oldName); + if (taosCompressFile(oldName, compressFileName) == 0) { + (void)taosRemoveFile(oldName); + } + +_end: if (tsLogKeepDays > 0) { taosRemoveOldFiles(tsLogDir, tsLogKeepDays); } @@ -316,13 +326,13 @@ static OldFileKeeper *taosOpenNewFile() { tsLogObj.logHandle->pFile = pFile; tsLogObj.lines = 0; tsLogObj.openInProgress = 0; - OldFileKeeper* oldFileKeeper = taosMemoryMalloc(sizeof(OldFileKeeper)); + OldFileKeeper *oldFileKeeper = taosMemoryMalloc(sizeof(OldFileKeeper)); if (oldFileKeeper == NULL) { uError("create old log keep info faild! mem is not enough."); return NULL; } oldFileKeeper->pOldFile = pOldFile; - memcpy(oldFileKeeper->keepName, keepName, LOG_FILE_NAME_LEN + 20); + taosReserveOldLog(keepName, oldFileKeeper->keepName); uInfo(" new log file:%d is opened", tsLogObj.flag); uInfo("=================================="); From e17cdf84c29aee1048fc37930ed7c39d13ccf9b8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jun 2024 18:08:34 +0800 Subject: [PATCH 035/103] fix: count always true for group by tbname performance issue --- source/libs/executor/src/executil.c | 24 +++++++++++++++++++----- source/libs/executor/src/scanoperator.c | 4 ++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index d06beebd6b..eb549db6d2 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2178,9 +2178,25 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* return code; } if (group == NULL || groupByTbname) { - for (int32_t i = 0; i < numOfTables; i++) { - STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - info->groupId = groupByTbname ? info->uid : 0; + if (groupByTbname && tsCountAlwaysReturnValue && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) { + pTableListInfo->remainGroups = + taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pTableListInfo->remainGroups == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + for (int i = 0; i < numOfTables; i++) { + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + info->groupId = info->uid; + + taosHashPut(pTableListInfo->remainGroups, &(info->groupId), sizeof(info->groupId), &(info->uid), + sizeof(info->uid)); + } + } else { + for (int32_t i = 0; i < numOfTables; i++) { + STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); + info->groupId = groupByTbname ? info->uid : 0; + } } pTableListInfo->oneTableForEachGroup = groupByTbname; @@ -2193,8 +2209,6 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* pTableListInfo->numOfOuputGroups = numOfTables; } else if (groupByTbname && pScanNode->groupOrderScan) { pTableListInfo->numOfOuputGroups = numOfTables; - } else if (groupByTbname && tsCountAlwaysReturnValue && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) { - pTableListInfo->numOfOuputGroups = numOfTables; } else { pTableListInfo->numOfOuputGroups = 1; } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index eef8b06ac5..ec40bceb5e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -725,7 +725,7 @@ void markGroupProcessed(STableScanInfo* pInfo, uint64_t groupId) { if (pInfo->countState == TABLE_COUNT_STATE_END) { return; } - if (pInfo->base.pTableListInfo->oneTableForEachGroup || pInfo->base.pTableListInfo->groupOffset) { + if (pInfo->base.pTableListInfo->groupOffset) { pInfo->countState = TABLE_COUNT_STATE_PROCESSED; } else { taosHashRemove(pInfo->base.pTableListInfo->remainGroups, &groupId, sizeof(groupId)); @@ -890,7 +890,7 @@ static SSDataBlock* doGroupedTableScan(SOperatorInfo* pOperator) { if (pTableScanInfo->countState < TABLE_COUNT_STATE_END) { STableListInfo* pTableListInfo = pTableScanInfo->base.pTableListInfo; - if (pTableListInfo->oneTableForEachGroup || pTableListInfo->groupOffset) { // group by tbname, group by tag + sort + if (pTableListInfo->groupOffset) { // group by tbname, group by tag + sort if (pTableScanInfo->countState < TABLE_COUNT_STATE_PROCESSED) { pTableScanInfo->countState = TABLE_COUNT_STATE_PROCESSED; STableKeyInfo* pStart = From 581f5e79d9a539d92f32a2eb5d177beb9bc14a36 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 11 Jun 2024 18:20:00 +0800 Subject: [PATCH 036/103] fix: reserve log file name --- source/util/src/tlog.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 1dad1d9519..ca809e1d5d 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -66,6 +66,7 @@ typedef struct { int32_t lines; int32_t flag; int32_t openInProgress; + int64_t lastFileSec; pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; @@ -275,6 +276,11 @@ static void taosReserveOldLog(char *oldName, char *keepName) { int32_t code = 0; int64_t fileSec = taosGetTimestampSec(); + if (tsLogObj.lastFileSec < fileSec) { + tsLogObj.lastFileSec = fileSec; + } else { + fileSec = ++tsLogObj.lastFileSec; + } snprintf(keepName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); if ((code = taosRenameFile(oldName, keepName))) { keepName[0] = 0; From 1a5cd763516453cf2aed3067d7d624ae502e04cc Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 11 Jun 2024 18:26:09 +0800 Subject: [PATCH 037/103] docs: release 3.3.1.0 --- cmake/cmake.version | 2 +- docs/en/28-releases/01-tdengine.md | 4 ++++ docs/zh/28-releases/01-tdengine.md | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmake/cmake.version b/cmake/cmake.version index 156e99bd03..44ac989200 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.3.1.0.alpha") + SET(TD_VER_NUMBER "3.3.2.0.alpha") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index f295e57bb5..119f22d96b 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://t import Release from "/components/ReleaseV3"; +## 3.3.1.0 + + + ## 3.3.0.3 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index f69e1fd4a8..cddb8160f6 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.3.1.0 + + + ## 3.3.0.3 From f1fa948594b4d85fd2fd89304271961d4aa913d9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 11 Jun 2024 18:27:46 +0800 Subject: [PATCH 038/103] fix: reserve log file name --- source/util/src/tlog.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index ca809e1d5d..e2a5d39888 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -66,7 +66,7 @@ typedef struct { int32_t lines; int32_t flag; int32_t openInProgress; - int64_t lastFileSec; + int64_t lastKeepFileSec; pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; @@ -276,10 +276,10 @@ static void taosReserveOldLog(char *oldName, char *keepName) { int32_t code = 0; int64_t fileSec = taosGetTimestampSec(); - if (tsLogObj.lastFileSec < fileSec) { - tsLogObj.lastFileSec = fileSec; + if (tsLogObj.lastKeepFileSec < fileSec) { + tsLogObj.lastKeepFileSec = fileSec; } else { - fileSec = ++tsLogObj.lastFileSec; + fileSec = ++tsLogObj.lastKeepFileSec; } snprintf(keepName, LOG_FILE_NAME_LEN + 20, "%s.%" PRId64, tsLogObj.logName, fileSec); if ((code = taosRenameFile(oldName, keepName))) { From a41a04dc37d4445f69ed3be983818e17c7666944 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 18:35:59 +0800 Subject: [PATCH 039/103] fix(stream): remove the related history task correctly. --- include/common/tmsg.h | 2 -- include/libs/stream/tstream.h | 3 +++ source/dnode/vnode/src/tqCommon/tqCommon.c | 9 +++++---- source/libs/stream/src/streamCheckpoint.c | 12 +++++++----- source/libs/stream/src/streamMeta.c | 11 +++++++++++ 5 files changed, 26 insertions(+), 11 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 98ca414466..387df52f16 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3473,8 +3473,6 @@ typedef struct SVUpdateCheckpointInfoReq { int64_t checkpointTs; int32_t transId; int8_t dropRelHTask; - int64_t hStreamId; - int64_t hTaskId; } SVUpdateCheckpointInfoReq; typedef struct { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1996c2ed63..bcf081dbfb 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -734,6 +734,9 @@ int32_t streamMetaAddFailedTask(SStreamMeta* pMeta, int64_t streamId, int32 void streamMetaAddFailedTaskSelf(SStreamTask* pTask, int64_t failedTs); void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SStreamTask* pHTask, int32_t transId, int64_t startTs); +void streamMetaClearUpdateTaskList(SStreamMeta* pMeta); +void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId); + void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 7c8ad159bc..44aef4a91e 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -185,7 +185,7 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM SStreamTask* pTask = *ppTask; const char* idstr = pTask->id.idStr; - if (pMeta->updateInfo.transId != req.transId) { + if ((pMeta->updateInfo.transId != req.transId) && (pMeta->updateInfo.transId != -1)) { if (req.transId < pMeta->updateInfo.transId) { tqError("s-task:%s vgId:%d disorder update nodeEp msg recv, discarded, newest transId:%d, recv:%d", idstr, vgId, pMeta->updateInfo.transId, req.transId); @@ -197,13 +197,12 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } else { tqInfo("s-task:%s vgId:%d receive new trans to update nodeEp msg from mnode, transId:%d, prev transId:%d", idstr, vgId, req.transId, pMeta->updateInfo.transId); - // info needs to be kept till the new trans to update the nodeEp arrived. - taosHashClear(pMeta->updateInfo.pTasks); - pMeta->updateInfo.transId = req.transId; + streamMetaInitUpdateTaskList(pMeta, req.transId); } } else { tqDebug("s-task:%s vgId:%d recv trans to update nodeEp from mnode, transId:%d", idstr, vgId, req.transId); + streamMetaInitUpdateTaskList(pMeta, req.transId); } // duplicate update epset msg received, discard this redundant message @@ -280,6 +279,8 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM // persist to disk } + streamMetaClearUpdateTaskList(pMeta); + if (!restored) { tqDebug("vgId:%d vnode restore not completed, not start the tasks, clear the start after nodeUpdate flag", vgId); pMeta->startInfo.tasksWillRestart = 0; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 1583734a99..6696c9f8c2 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -418,6 +418,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin int32_t code = 0; const char* id = pTask->id.idStr; SCheckpointInfo* pInfo = &pTask->chkInfo; + STaskId hTaskId = {0}; taosThreadMutexLock(&pTask->lock); @@ -433,12 +434,11 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin // drop task should not in the meta-lock, and drop the related fill-history task now streamMetaWUnLock(pMeta); if (pReq->dropRelHTask) { - streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); + streamMetaUnregisterTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d", id, vgId, pReq->taskId, numOfTasks); } - streamMetaWLock(pMeta); } @@ -476,8 +476,9 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin } if (pReq->dropRelHTask) { + hTaskId = pTask->hTaskInfo.id; stDebug("s-task:0x%x vgId:%d drop the related fill-history task:0x%" PRIx64 " after update checkpoint", - pReq->taskId, vgId, pReq->hTaskId); + pReq->taskId, vgId, hTaskId.taskId); CLEAR_RELATED_FILLHISTORY_TASK(pTask); } @@ -498,9 +499,10 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin // drop task should not in the meta-lock, and drop the related fill-history task now if (pReq->dropRelHTask) { - streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); + streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); - stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, (int32_t) pReq->hTaskId, numOfTasks); + stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, + (int32_t)hTaskId.taskId, numOfTasks); } streamMetaWLock(pMeta); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index f6449829a3..da1fec5565 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -372,6 +372,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->expandFunc = expandFunc; pMeta->stage = stage; pMeta->role = (vgId == SNODE_HANDLE) ? NODE_ROLE_LEADER : NODE_ROLE_UNINIT; + pMeta->updateInfo.transId = -1; pMeta->startInfo.completeFn = fn; pMeta->pTaskDbUnique = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); @@ -1740,4 +1741,14 @@ void streamMetaAddIntoUpdateTaskList(SStreamMeta* pMeta, SStreamTask* pTask, SSt stDebug("s-task:%s vgId:%d transId:%d task nodeEp update completed, streamTask closed, elapsed time:%" PRId64 "ms", id, vgId, transId, el); } +} + +void streamMetaClearUpdateTaskList(SStreamMeta* pMeta) { + taosHashClear(pMeta->updateInfo.pTasks); + pMeta->updateInfo.transId = -1; +} + +void streamMetaInitUpdateTaskList(SStreamMeta* pMeta, int32_t transId) { + taosHashClear(pMeta->updateInfo.pTasks); + pMeta->updateInfo.transId = transId; } \ No newline at end of file From 9406be9aa681738d78117efb4c47da05f2939b31 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 18:58:16 +0800 Subject: [PATCH 040/103] fix(stream): remove invalid reset. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 44aef4a91e..cabccfc0c8 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -202,7 +202,6 @@ int32_t tqStreamTaskProcessUpdateReq(SStreamMeta* pMeta, SMsgCb* cb, SRpcMsg* pM } } else { tqDebug("s-task:%s vgId:%d recv trans to update nodeEp from mnode, transId:%d", idstr, vgId, req.transId); - streamMetaInitUpdateTaskList(pMeta, req.transId); } // duplicate update epset msg received, discard this redundant message From 90e54d0603d970e738b79a48f7d31b39f1a2f01b Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Tue, 11 Jun 2024 19:38:31 +0800 Subject: [PATCH 041/103] test case: mem overflow --- source/dnode/mnode/impl/test/func/func.cpp | 27 ++++++++++++++-------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index ee60556639..0fc903f5db 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -169,7 +169,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - taosArrayPush(retrieveReq.pFuncNames, "f1"); + char name[TSDB_FUNC_NAME_LEN] = "f1"; + taosArrayPush(retrieveReq.pFuncNames, name); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); @@ -220,7 +221,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { retrieveReq.numOfFuncs = TSDB_FUNC_MAX_RETRIEVE + 1; retrieveReq.pFuncNames = taosArrayInit(TSDB_FUNC_MAX_RETRIEVE + 1, TSDB_FUNC_NAME_LEN); for (int32_t i = 0; i < TSDB_FUNC_MAX_RETRIEVE + 1; ++i) { - taosArrayPush(retrieveReq.pFuncNames, "1"); + char name[TSDB_FUNC_NAME_LEN] = "1"; + taosArrayPush(retrieveReq.pFuncNames, name); } int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); @@ -237,7 +239,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - taosArrayPush(retrieveReq.pFuncNames, "f2"); + char name[TSDB_FUNC_NAME_LEN] = "f2"; + taosArrayPush(retrieveReq.pFuncNames, name); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); @@ -279,7 +282,8 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - taosArrayPush(retrieveReq.pFuncNames, "f2"); + char name[TSDB_FUNC_NAME_LEN] = "f2"; + taosArrayPush(retrieveReq.pFuncNames, name); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); @@ -316,8 +320,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 2; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - taosArrayPush(retrieveReq.pFuncNames, "f2"); - taosArrayPush(retrieveReq.pFuncNames, "f1"); + char name1[TSDB_FUNC_NAME_LEN] = "f2"; + taosArrayPush(retrieveReq.pFuncNames, name1); + char name2[TSDB_FUNC_NAME_LEN] = "f1"; + taosArrayPush(retrieveReq.pFuncNames, name2); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); @@ -367,8 +373,10 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 2; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - taosArrayPush(retrieveReq.pFuncNames, "f2"); - taosArrayPush(retrieveReq.pFuncNames, "f3"); + char name1[TSDB_FUNC_NAME_LEN] = "f2"; + taosArrayPush(retrieveReq.pFuncNames, name1); + char name2[TSDB_FUNC_NAME_LEN] = "f3"; + taosArrayPush(retrieveReq.pFuncNames, name2); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); @@ -483,7 +491,8 @@ TEST_F(MndTestFunc, 05_Actual_code) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); - taosArrayPush(retrieveReq.pFuncNames, "udf1"); + char name[TSDB_FUNC_NAME_LEN] = "udf1"; + taosArrayPush(retrieveReq.pFuncNames, name); int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); void* pReq = rpcMallocCont(contLen); From 80eb88d6ac9c7eebaa2a82ce017e4c591fb713d8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jun 2024 19:54:38 +0800 Subject: [PATCH 042/103] fix(stream): wait for more 5 sec. --- tests/system-test/8-stream/stream_multi_agg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/8-stream/stream_multi_agg.py b/tests/system-test/8-stream/stream_multi_agg.py index 4e8d965f32..32c2648b46 100644 --- a/tests/system-test/8-stream/stream_multi_agg.py +++ b/tests/system-test/8-stream/stream_multi_agg.py @@ -42,7 +42,7 @@ class TDTestCase: tdSql.execute("use test", queryTimes=100) tdSql.query("create stream if not exists s1 trigger at_once ignore expired 0 ignore update 0 fill_history 1 into st1 as select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s)") tdLog.debug("========create stream and insert data ok========") - time.sleep(15) + time.sleep(20) tdSql.query("select _wstart,sum(voltage),groupid from meters partition by groupid interval(2s) order by groupid,_wstart") rowCnt = tdSql.getRows() From db44febadf551982d6c9c54096d5150e331f94ab Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Jun 2024 08:27:46 +0800 Subject: [PATCH 043/103] fix: reserve log file name --- source/util/src/tlog.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index e2a5d39888..8023541af3 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -289,15 +289,14 @@ static void taosReserveOldLog(char *oldName, char *keepName) { } static void taosKeepOldLog(char *oldName) { - if (oldName[0] == '\0') goto _end; - - char compressFileName[LOG_FILE_NAME_LEN + 20]; - snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.gz", oldName); - if (taosCompressFile(oldName, compressFileName) == 0) { - (void)taosRemoveFile(oldName); + if (oldName[0] != 0) { + char compressFileName[LOG_FILE_NAME_LEN + 20]; + snprintf(compressFileName, LOG_FILE_NAME_LEN + 20, "%s.gz", oldName); + if (taosCompressFile(oldName, compressFileName) == 0) { + (void)taosRemoveFile(oldName); + } } -_end: if (tsLogKeepDays > 0) { taosRemoveOldFiles(tsLogDir, tsLogKeepDays); } From dc2497791a89d99b6efaf657f635bec3c707aa4b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jun 2024 10:11:55 +0800 Subject: [PATCH 044/103] fix(stream): accept invalid vgroup id error code when handling the update checkpoint info. --- source/dnode/mnode/impl/inc/mndStream.h | 2 +- source/dnode/mnode/impl/src/mndStream.c | 43 ++++++++++---------- source/dnode/mnode/impl/src/mndStreamTrans.c | 12 +++--- source/dnode/mnode/impl/src/mndStreamUtil.c | 14 +++---- source/os/src/osTimer.c | 5 ++- 5 files changed, 40 insertions(+), 36 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 9289909b19..dade7b4bdf 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -110,7 +110,7 @@ int32_t mndGetNumOfStreamTasks(const SStreamObj *pStream); SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); void mndKillTransImpl(SMnode *pMnode, int32_t transId, const char *pDbName); int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, - int32_t retryCode); + int32_t retryCode, int32_t acceptCode); STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnConflct conflict, const char *name, const char *pMsg); int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status); SSdbRaw *mndStreamActionEncode(SStreamObj *pStream); diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 136839451f..c9598c4b38 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -515,7 +515,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, SStreamTask *pTask) { tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); - int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0); + int32_t code = setTransAction(pTrans, buf, tlen, TDMT_STREAM_TASK_DEPLOY, &pTask->info.epSet, 0, 0); if (code != 0) { taosMemoryFree(buf); return -1; @@ -959,7 +959,7 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask return -1; } - code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY); + code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, 0); if (code != 0) { taosMemoryFree(buf); } @@ -2554,27 +2554,28 @@ int32_t mndProcessCheckpointReport(SRpcMsg *pReq) { int32_t total = taosArrayGetSize(*pReqTaskList); if (total == numOfTasks) { // all tasks has send the reqs - mInfo("stream:0x%" PRIx64 - " %s all %d tasks send checkpoint-report, start to update checkpoint meta-info for checkpointId:%" PRId64, + mInfo("stream:0x%" PRIx64 " %s all %d tasks send checkpoint-report, checkpoint meta-info for checkpointId:%" PRId64 + " will be issued soon", req.streamId, pStream->name, total, req.checkpointId); - if (pStream != NULL) { - bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); - if (conflict) { - mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId); - } else { - int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, *pReqTaskList); - if (code == TSDB_CODE_SUCCESS) { // remove this entry - taosHashRemove(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId)); - - int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); - mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", req.streamId, numOfStreams); - } else { - mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet", - req.streamId); - } - } - } + // if (pStream != NULL) { + // bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_CHKPT_UPDATE_NAME, false); + // if (conflict) { + // mDebug("stream:0x%"PRIx64" active checkpoint trans not finished yet, wait", req.streamId); + // } else { + // int32_t code = mndCreateStreamChkptInfoUpdateTrans(pMnode, pStream, *pReqTaskList); + // if (code == TSDB_CODE_SUCCESS) { // remove this entry + // taosHashRemove(execInfo.pChkptStreams, &req.streamId, sizeof(req.streamId)); + // + // int32_t numOfStreams = taosHashGetSize(execInfo.pChkptStreams); + // mDebug("stream:0x%" PRIx64 " removed, remain streams:%d in checkpoint procedure", req.streamId, + // numOfStreams); + // } else { + // mDebug("stream:0x%" PRIx64 " not launch chkpt update trans, due to checkpoint not finished yet", + // req.streamId); + // } + // } + // } } if (pStream != NULL) { diff --git a/source/dnode/mnode/impl/src/mndStreamTrans.c b/source/dnode/mnode/impl/src/mndStreamTrans.c index 8137081631..0daa383d3e 100644 --- a/source/dnode/mnode/impl/src/mndStreamTrans.c +++ b/source/dnode/mnode/impl/src/mndStreamTrans.c @@ -127,7 +127,7 @@ bool mndStreamTransConflictCheck(SMnode* pMnode, int64_t streamId, const char* p return false; } -int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) { +int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamId) { taosThreadMutexLock(&execInfo.lock); int32_t num = taosHashGetSize(execInfo.transMgmt.pDBTrans); if (num <= 0) { @@ -136,12 +136,13 @@ int32_t mndStreamGetRelTrans(SMnode* pMnode, int64_t streamUid) { } mndStreamClearFinishedTrans(pMnode, NULL); - SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamUid, sizeof(streamUid)); + SStreamTransInfo* pEntry = taosHashGet(execInfo.transMgmt.pDBTrans, &streamId, sizeof(streamId)); if (pEntry != NULL) { SStreamTransInfo tInfo = *pEntry; taosThreadMutexUnlock(&execInfo.lock); - if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0) { + if (strcmp(tInfo.name, MND_STREAM_CHECKPOINT_NAME) == 0 || strcmp(tInfo.name, MND_STREAM_TASK_UPDATE_NAME) == 0 || + strcmp(tInfo.name, MND_STREAM_CHKPT_UPDATE_NAME) == 0) { return tInfo.transId; } } else { @@ -246,8 +247,9 @@ int32_t mndPersistTransLog(SStreamObj *pStream, STrans *pTrans, int32_t status) } int32_t setTransAction(STrans *pTrans, void *pCont, int32_t contLen, int32_t msgType, const SEpSet *pEpset, - int32_t retryCode) { - STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode}; + int32_t retryCode, int32_t acceptCode) { + STransAction action = {.epSet = *pEpset, .contLen = contLen, .pCont = pCont, .msgType = msgType, .retryCode = retryCode, + .acceptableCode = acceptCode}; return mndTransAppendRedoAction(pTrans, &action); } diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index 6745acac34..a0731833e6 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -230,7 +230,7 @@ static int32_t doSetResumeAction(STrans *pTrans, SMnode *pMnode, SStreamTask *pT return -1; } - code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0); + code = setTransAction(pTrans, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); return -1; @@ -308,7 +308,7 @@ static int32_t doSetPauseAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa epsetToStr(&epset, buf, tListLen(buf)); mDebug("pause stream task in node:%d, epset:%s", pTask->info.nodeId, buf); - code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0); + code = setTransAction(pTrans, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); return -1; @@ -356,7 +356,7 @@ static int32_t doSetDropAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTas } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. - code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0); + code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); return -1; @@ -400,7 +400,7 @@ static int32_t doSetDropActionFromId(SMnode *pMnode, STrans *pTrans, SOrphanTask } // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. - code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0); + code = setTransAction(pTrans, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0, 0); if (code != 0) { taosMemoryFree(pReq); return -1; @@ -484,7 +484,7 @@ static int32_t doSetUpdateTaskAction(SMnode *pMnode, STrans *pTrans, SStreamTask return code; } - code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID); + code = setTransAction(pTrans, pBuf, len, TDMT_VND_STREAM_TASK_UPDATE, &epset, TSDB_CODE_VND_INVALID_VGROUP_ID, 0); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pBuf); } @@ -534,7 +534,7 @@ static int32_t doSetResetAction(SMnode *pMnode, STrans *pTrans, SStreamTask *pTa return code; } - code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0); + code = setTransAction(pTrans, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0, 0); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pReq); } @@ -727,7 +727,7 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas return code; } - code = setTransAction(pTrans, pReq, sizeof(SVUpdateCheckpointInfoReq), TDMT_STREAM_TASK_UPDATE_CHKPT, &epset, 0); + code = setTransAction(pTrans, pReq, sizeof(SVUpdateCheckpointInfoReq), TDMT_STREAM_TASK_UPDATE_CHKPT, &epset, 0, TSDB_CODE_VND_INVALID_VGROUP_ID); if (code != TSDB_CODE_SUCCESS) { taosMemoryFree(pReq); } diff --git a/source/os/src/osTimer.c b/source/os/src/osTimer.c index b8eb54e10e..36d364382a 100644 --- a/source/os/src/osTimer.c +++ b/source/os/src/osTimer.c @@ -86,8 +86,9 @@ static void taosDeleteTimer(void *tharg) { static TdThread timerThread; static timer_t timerId; static volatile bool stopTimer = false; -static void *taosProcessAlarmSignal(void *tharg) { - // Block the signal + +static void *taosProcessAlarmSignal(void *tharg) { + // Block the signal sigset_t sigset; sigemptyset(&sigset); sigaddset(&sigset, SIGALRM); From 2e582155e6927b152eed3808cda5325dbdb613c5 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Wed, 12 Jun 2024 03:07:50 +0000 Subject: [PATCH 045/103] Update the libuv version to prevent compilation errors in higher versions of GCC --- cmake/libuv_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index 9c48ddefef..dfca7ee018 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -2,7 +2,7 @@ # libuv ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git - GIT_TAG v1.44.2 + GIT_TAG v1.47.0 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" CONFIGURE_COMMAND "" From fa05b2c8b634c359726d6f20e7460b4cd60418df Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Jun 2024 11:17:08 +0800 Subject: [PATCH 046/103] fix: reserve log file name --- source/util/src/tlog.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 8023541af3..b224eac8c2 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -269,7 +269,7 @@ static void taosUnLockLogFile(TdFilePtr pFile) { } static void taosReserveOldLog(char *oldName, char *keepName) { - if (tsLogKeepDays == 0) { + if (tsLogKeepDays <= 0) { keepName[0] = 0; return; } From 5f347a0b223590f85b2e00ac4eaf015b74413fa0 Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 12 Jun 2024 11:50:03 +0800 Subject: [PATCH 047/103] test case --- tests/parallel_test/cases.task | 5 + .../2-query/partition_limit_interval.py | 105 ++++++++++++++++++ tests/system-test/runAllOne.sh | 5 + 3 files changed, 115 insertions(+) create mode 100755 tests/system-test/2-query/partition_limit_interval.py diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 61687eeccd..bed1c20421 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -535,6 +535,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_limit_interval.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_limit_interval.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py @@ -779,6 +781,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_limit_interval.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 @@ -874,6 +877,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_limit_interval.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 @@ -971,6 +975,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/partition_limit_interval.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_min_last_interval.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row_interval.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4 diff --git a/tests/system-test/2-query/partition_limit_interval.py b/tests/system-test/2-query/partition_limit_interval.py new file mode 100755 index 0000000000..287c7b7619 --- /dev/null +++ b/tests/system-test/2-query/partition_limit_interval.py @@ -0,0 +1,105 @@ +from util.log import * +from util.sql import * +from util.cases import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + self.row_nums = 1000 + self.tb_nums = 10 + self.ts = 1537146000000 + self.dbname = "db1" + self.stable = "meters" + + def prepare_datas(self, stb_name , tb_nums , row_nums, dbname="db" ): + tdSql.execute(f'''create database db1 MAXROWS 4096 MINROWS 100''') + tdSql.execute(f'''use {self.dbname}''') + tdSql.execute(f'''CREATE STABLE {self.stable} (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` TINYINT, `location` VARCHAR(16))''') + + for i in range(self.tb_nums): + tbname = f"{self.dbname}.sub_{self.stable}_{i}" + ts = self.ts + i*10000 + tdSql.execute(f"create table {tbname} using {self.dbname}.{self.stable} tags({i} ,'nchar_{i}')") + tdLog.info(f"create table {tbname} using {self.dbname}.{self.stable} tags({i} ,'nchar_{i}')") + if i < (self.tb_nums - 2): + for row in range(row_nums): + ts = self.ts + row*1000 + tdSql.execute(f"insert into {tbname} values({ts} , {row/10}, {215 + (row % 100)})") + + for null in range(5): + ts = self.ts + row_nums*1000 + null*1000 + tdSql.execute(f"insert into {tbname} values({ts} , NULL , NULL)") + + def basic_query(self): + tdSql.query(f"select groupid, count(*) from {self.dbname}.{self.stable} partition by groupid interval(1d) limit 100") + tdSql.checkRows(8) + tdSql.checkData(0, 1, 1005) + + tdSql.query(f"select groupid, count(*) from {self.dbname}.{self.stable} partition by tbname interval(1d) order by groupid limit 100;") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, 1005) + tdSql.checkData(7, 0, 7) + tdSql.checkData(7, 1, 1005) + + tdSql.query(f"select groupid, count(*) from {self.dbname}.{self.stable} partition by tbname, groupid interval(5d) order by groupid limit 10") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, 1005) + tdSql.checkData(7, 0, 7) + tdSql.checkData(7, 1, 1005) + + tdSql.query(f"select groupid, count(*), min(current) from {self.dbname}.{self.stable} partition by groupid interval(5d) order by groupid limit 10;") + tdSql.checkRows(8) + tdSql.checkData(0, 0, 0) + tdSql.checkData(0, 1, 1005) + tdSql.checkData(0, 2, 0) + tdSql.checkData(7, 0, 7) + tdSql.checkData(7, 1, 1005) + tdSql.checkData(7, 2, 0) + + tdSql.query(f"select groupid, min(current) from {self.dbname}.{self.stable} partition by groupid interval(5d) limit 100;") + tdSql.checkRows(8) + tdSql.checkData(0, 1, 0) + + tdSql.query(f"select groupid, avg(current) from {self.dbname}.{self.stable} partition by groupid interval(5d) limit 10000;") + tdSql.checkRows(8) + tdSql.checkData(0, 1, tdSql.getData(7, 1)) + + tdSql.query(f"select current, avg(current) from {self.dbname}.{self.stable} partition by current interval(5d) limit 100;") + tdSql.checkData(0, 0, tdSql.getData(0, 1)) + + tdSql.query(f"select groupid, last(voltage), min(current) from {self.dbname}.{self.stable} partition by groupid interval(5d) limit 10") + tdSql.checkRows(8) + tdSql.checkData(0, 1, tdSql.getData(7, 1)) + tdSql.checkData(0, 2, tdSql.getData(7, 2)) + + tdSql.query(f"select groupid, min(current), min(voltage) from {self.dbname}.{self.stable} partition by tbname, groupid interval(5d) limit 100;") + tdSql.checkRows(8) + tdSql.checkData(0, 1, 0) + tdSql.checkData(0, 2, 215) + tdSql.checkData(7, 1, 0) + tdSql.checkData(7, 2, 215) + + tdSql.query(f"select groupid, min(voltage), min(current) from {self.dbname}.{self.stable} partition by tbname, groupid interval(5d) limit 100;") + tdSql.checkRows(8) + tdSql.checkData(0, 2, 0) + tdSql.checkData(0, 1, 215) + tdSql.checkData(7, 2, 0) + tdSql.checkData(7, 1, 215) + + def run(self): + tdSql.prepare() + self.prepare_datas("stb",self.tb_nums,self.row_nums) + self.basic_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/runAllOne.sh b/tests/system-test/runAllOne.sh index 099ae1bbd3..79fc2cd363 100644 --- a/tests/system-test/runAllOne.sh +++ b/tests/system-test/runAllOne.sh @@ -235,6 +235,8 @@ python3 ./test.py -f 2-query/mavg.py -P python3 ./test.py -f 2-query/mavg.py -P -R python3 ./test.py -f 2-query/max_partition.py -P python3 ./test.py -f 2-query/max_partition.py -P -R +python3 ./test.py -f 2-query/partition_limit_interval.py -P +python3 ./test.py -f 2-query/partition_limit_interval.py -P -R python3 ./test.py -f 2-query/max_min_last_interval.py -P python3 ./test.py -f 2-query/last_row_interval.py -P python3 ./test.py -f 2-query/max.py -P @@ -481,6 +483,7 @@ python3 ./test.py -f 2-query/irate.py -P -Q 2 python3 ./test.py -f 2-query/function_null.py -P -Q 2 python3 ./test.py -f 2-query/count_partition.py -P -Q 2 python3 ./test.py -f 2-query/max_partition.py -P -Q 2 +python3 ./test.py -f 2-query/partition_limit_interval.py -P -Q 2 python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 2 python3 ./test.py -f 2-query/last_row_interval.py -P -Q 2 python3 ./test.py -f 2-query/last_row.py -P -Q 2 @@ -576,6 +579,7 @@ python3 ./test.py -f 2-query/irate.py -P -Q 3 python3 ./test.py -f 2-query/function_null.py -P -Q 3 python3 ./test.py -f 2-query/count_partition.py -P -Q 3 python3 ./test.py -f 2-query/max_partition.py -P -Q 3 +python3 ./test.py -f 2-query/partition_limit_interval.py -P -Q 3 python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 3 python3 ./test.py -f 2-query/last_row_interval.py -P -Q 3 python3 ./test.py -f 2-query/last_row.py -P -Q 3 @@ -673,6 +677,7 @@ python3 ./test.py -f 2-query/irate.py -P -Q 4 python3 ./test.py -f 2-query/function_null.py -P -Q 4 python3 ./test.py -f 2-query/count_partition.py -P -Q 4 python3 ./test.py -f 2-query/max_partition.py -P -Q 4 +python3 ./test.py -f 2-query/partition_limit_interval.py -P -Q 4 python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 4 python3 ./test.py -f 2-query/last_row_interval.py -P -Q 4 python3 ./test.py -f 2-query/last_row.py -P -Q 4 From 5e4107df9324220d600fa4fa44fc5d320b9edbcd Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Wed, 12 Jun 2024 13:22:03 +0800 Subject: [PATCH 048/103] test case --- tests/system-test/2-query/partition_limit_interval.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/system-test/2-query/partition_limit_interval.py b/tests/system-test/2-query/partition_limit_interval.py index 287c7b7619..dc0aabbfdd 100755 --- a/tests/system-test/2-query/partition_limit_interval.py +++ b/tests/system-test/2-query/partition_limit_interval.py @@ -15,9 +15,9 @@ class TDTestCase: self.stable = "meters" def prepare_datas(self, stb_name , tb_nums , row_nums, dbname="db" ): - tdSql.execute(f'''create database db1 MAXROWS 4096 MINROWS 100''') + tdSql.execute(f'''create database {self.dbname} MAXROWS 4096 MINROWS 100''') tdSql.execute(f'''use {self.dbname}''') - tdSql.execute(f'''CREATE STABLE {self.stable} (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` TINYINT, `location` VARCHAR(16))''') + tdSql.execute(f'''CREATE STABLE {self.dbname}.{self.stable} (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` TINYINT, `location` VARCHAR(16))''') for i in range(self.tb_nums): tbname = f"{self.dbname}.sub_{self.stable}_{i}" From 89b97eff60104caf3c8b4be30fcb2f5c4f70c767 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Wed, 12 Jun 2024 07:22:40 +0000 Subject: [PATCH 049/103] fix compile error --- cmake/libuv_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index dfca7ee018..3e259024b2 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -2,7 +2,7 @@ # libuv ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git - GIT_TAG v1.47.0 + GIT_TAG v1.46.0 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" CONFIGURE_COMMAND "" From d06e1549f122aba59ea80bfba69495a33c7277c6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jun 2024 15:27:19 +0800 Subject: [PATCH 050/103] fix(stream): add fill-history task id in msg. --- include/common/tmsg.h | 2 ++ source/dnode/mnode/impl/src/mndStreamUtil.c | 2 ++ source/dnode/vnode/src/tqCommon/tqCommon.c | 2 -- source/libs/stream/src/streamCheckpoint.c | 10 ++++------ 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 387df52f16..fb8a96124d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -3472,6 +3472,8 @@ typedef struct SVUpdateCheckpointInfoReq { int64_t checkpointVer; int64_t checkpointTs; int32_t transId; + int64_t hStreamId; // add encode/decode + int64_t hTaskId; int8_t dropRelHTask; } SVUpdateCheckpointInfoReq; diff --git a/source/dnode/mnode/impl/src/mndStreamUtil.c b/source/dnode/mnode/impl/src/mndStreamUtil.c index a0731833e6..e47f28c309 100644 --- a/source/dnode/mnode/impl/src/mndStreamUtil.c +++ b/source/dnode/mnode/impl/src/mndStreamUtil.c @@ -716,6 +716,8 @@ static int32_t doSetUpdateChkptAction(SMnode *pMnode, STrans *pTrans, SStreamTas pReq->checkpointTs = pInfo->ts; pReq->dropRelHTask = pInfo->dropHTask; pReq->transId = pInfo->transId; + pReq->hStreamId = pTask->hTaskInfo.id.streamId; + pReq->hTaskId = pTask->hTaskInfo.id.taskId; } } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index cabccfc0c8..d16c41ec1e 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -646,8 +646,6 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen } streamMetaWUnLock(pMeta); - -// tqStreamRemoveTaskBackend(pMeta, &id); return 0; } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 6696c9f8c2..eedd8f20d6 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -418,7 +418,6 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin int32_t code = 0; const char* id = pTask->id.idStr; SCheckpointInfo* pInfo = &pTask->chkInfo; - STaskId hTaskId = {0}; taosThreadMutexLock(&pTask->lock); @@ -434,7 +433,7 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin // drop task should not in the meta-lock, and drop the related fill-history task now streamMetaWUnLock(pMeta); if (pReq->dropRelHTask) { - streamMetaUnregisterTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); + streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped in update checkpointInfo, remain tasks:%d", id, vgId, pReq->taskId, numOfTasks); @@ -476,9 +475,8 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin } if (pReq->dropRelHTask) { - hTaskId = pTask->hTaskInfo.id; stDebug("s-task:0x%x vgId:%d drop the related fill-history task:0x%" PRIx64 " after update checkpoint", - pReq->taskId, vgId, hTaskId.taskId); + pReq->taskId, vgId, pReq->hTaskId); CLEAR_RELATED_FILLHISTORY_TASK(pTask); } @@ -499,10 +497,10 @@ int32_t streamTaskUpdateTaskCheckpointInfo(SStreamTask* pTask, SVUpdateCheckpoin // drop task should not in the meta-lock, and drop the related fill-history task now if (pReq->dropRelHTask) { - streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); + streamMetaUnregisterTask(pMeta, pReq->hStreamId, pReq->hTaskId); int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); stDebug("s-task:%s vgId:%d related fill-history task:0x%x dropped, remain tasks:%d", id, vgId, - (int32_t)hTaskId.taskId, numOfTasks); + (int32_t)pReq->hTaskId, numOfTasks); } streamMetaWLock(pMeta); From 13a0bf3fdf5b480da5dd87624279729dd6afca57 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 12 Jun 2024 16:02:05 +0800 Subject: [PATCH 051/103] fix: count empty table with group by issue --- source/common/src/tdatablock.c | 2 ++ source/libs/executor/src/executil.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index ac4811fb1b..8e8c9d9a85 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1362,6 +1362,8 @@ void blockDataEmpty(SSDataBlock* pDataBlock) { return; } + taosMemoryFreeClear(pDataBlock->pBlockAgg); + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index eb549db6d2..9ca681779d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -2178,7 +2178,7 @@ int32_t buildGroupIdMapForAllTables(STableListInfo* pTableListInfo, SReadHandle* return code; } if (group == NULL || groupByTbname) { - if (groupByTbname && tsCountAlwaysReturnValue && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) { + if (tsCountAlwaysReturnValue && QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN == nodeType(pScanNode) && ((STableScanPhysiNode*)pScanNode)->needCountEmptyTable) { pTableListInfo->remainGroups = taosHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); if (pTableListInfo->remainGroups == NULL) { From 6ff106cd0af1c93613571e02461a4dfc36b826e3 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Wed, 12 Jun 2024 08:39:35 +0000 Subject: [PATCH 052/103] fix compile error --- cmake/libuv_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index 3e259024b2..673c771fb0 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -2,7 +2,7 @@ # libuv ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git - GIT_TAG v1.46.0 + GIT_TAG v1.48.0 SOURCE_DIR "${TD_CONTRIB_DIR}/libuv" BINARY_DIR "${TD_CONTRIB_DIR}/libuv" CONFIGURE_COMMAND "" From e060535c1342b69ae393f17a04f49a6132ba9f88 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 12 Jun 2024 17:23:57 +0800 Subject: [PATCH 053/103] feat: data migration speed limit --- docs/en/14-reference/12-config/index.md | 2 +- docs/zh/14-reference/12-config/index.md | 2 +- include/common/tglobal.h | 1 + source/common/src/tglobal.c | 3 ++ source/dnode/vnode/src/tsdb/tsdbRetention.c | 31 ++++++++++++++++++++- 5 files changed, 36 insertions(+), 3 deletions(-) diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 5e4eadcceb..f50551b5de 100755 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -432,7 +432,7 @@ The charset that takes effect is UTF-8. | Applicable | Server Only | | Meaning | Maximum number of threads to commit | | Value Range | 0-1024 | -| Default Value | | +| Default Value | 4 | ## Log Parameters diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index 6fce985927..effa72099a 100755 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -430,7 +430,7 @@ charset 的有效值是 UTF-8。 | 适用范围 | 仅服务端适用 | | 含义 | 设置写入线程的最大数量 | | 取值范围 | 0-1024 | -| 缺省值 | | +| 缺省值 | 4 | ## 日志相关 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 90ee6f7cc0..e7035fe297 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -86,6 +86,7 @@ extern int32_t tsNumOfQnodeFetchThreads; extern int32_t tsNumOfSnodeStreamThreads; extern int32_t tsNumOfSnodeWriteThreads; extern int64_t tsRpcQueueMemoryAllowed; +extern int32_t tsRetentionSpeedLimitMB; // sync raft extern int32_t tsElectInterval; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c68dc85c29..da692e78fd 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -74,6 +74,7 @@ int32_t tsNumOfSnodeStreamThreads = 4; int32_t tsNumOfSnodeWriteThreads = 1; int32_t tsMaxStreamBackendCache = 128; // M int32_t tsPQSortMemThreshold = 16; // M +int32_t tsRetentionSpeedLimitMB = 0; // unlimited // sync raft int32_t tsElectInterval = 25 * 1000; @@ -667,6 +668,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "retentionSpeedLimitMB", tsRetentionSpeedLimitMB, 0, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -1117,6 +1119,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; + tsRetentionSpeedLimitMB = cfgGetItem(pCfg, "retentionSpeedLimitMB")->i32; tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; tsRatioOfVnodeStreamThreads = cfgGetItem(pCfg, "ratioOfVnodeStreamThreads")->fval; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 3d53d1ada3..0d3994d78e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -38,6 +38,34 @@ static int32_t tsdbDoRemoveFileObject(SRTNer *rtner, const STFileObj *fobj) { return TARRAY2_APPEND(&rtner->fopArr, op); } +static int64_t tsdbCopyFileWithLimitedSpeed(TdFilePtr from, TdFilePtr to, int64_t size, uint32_t limitMB) { + int64_t total = 0; + int64_t interval = 1000; // 1s + int64_t limit = limitMB ? limitMB * 1024 * 1024 : INT64_MAX; + int64_t offset = 0; + int64_t remain = size; + + while (remain > 0) { + int64_t n; + int64_t last = taosGetTimestampMs(); + if ((n = taosFSendFile(to, from, &offset, TMIN(limit, remain))) < 0) { + return -1; + } + + total += n; + remain -= n; + + if (remain > 0) { + int64_t elapsed = taosGetTimestampMs() - last; + if (elapsed < interval) { + taosMsleep(interval - elapsed); + } + } + } + + return total; +} + static int32_t tsdbDoCopyFileLC(SRTNer *rtner, const STFileObj *from, const STFile *to) { int32_t code = 0; int32_t lino = 0; @@ -98,7 +126,8 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile if (fdTo == NULL) code = terrno; TSDB_CHECK_CODE(code, lino, _exit); - int64_t n = taosFSendFile(fdTo, fdFrom, 0, tsdbLogicToFileSize(from->f->size, rtner->szPage)); + int64_t n = tsdbCopyFileWithLimitedSpeed(fdFrom, fdTo, tsdbLogicToFileSize(from->f->size, rtner->szPage), + tsRetentionSpeedLimitMB); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); TSDB_CHECK_CODE(code, lino, _exit); From 7723c24a219d9a21b6dcf5bbcef5721ed9acb58f Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 12 Jun 2024 18:22:49 +0800 Subject: [PATCH 054/103] fix industry package update issue --- packaging/tools/install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 5a83cdc6a8..03e0a0b5f5 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -795,10 +795,10 @@ function is_version_compatible() { if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=$(cat ${script_dir}/driver/vercomp.txt) else - min_compatible_version=$(${script_dir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 5) + min_compatible_version=$(${script_dir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 5) fi - exist_version=$(${installDir}/bin/${serverName} -V | head -1 | cut -d ' ' -f 3) + exist_version=$(${installDir}/bin/${serverName} -V | grep version | head -1 | cut -d ' ' -f 3) vercomp $exist_version "3.0.0.0" case $? in 2) From 95a6cbf8e082db7cbd14dcc623fda221084eb7f9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Wed, 12 Jun 2024 18:56:29 +0800 Subject: [PATCH 055/103] enh: support get origin string of taos errors --- include/util/taoserror.h | 7 +++++++ source/util/src/terror.c | 21 ++++++++++----------- source/util/test/CMakeLists.txt | 10 +++++++++- source/util/test/terrorTest.cpp | 29 +++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 12 deletions(-) create mode 100644 source/util/test/terrorTest.cpp diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 8f8434dfc1..63b733a337 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -24,6 +24,12 @@ extern "C" { // clang-format off +typedef struct { + int32_t val; + const char* str; + const char* origin; +} STaosError; + #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) @@ -38,6 +44,7 @@ const char* terrstr(); char* taosGetErrMsgReturn(); char* taosGetErrMsg(); int32_t* taosGetErrno(); +int32_t taosGetErrSize(); #define terrno (*taosGetErrno()) #define terrMsg (taosGetErrMsg()) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 0f594af0e9..f2ae43bc94 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -21,10 +21,10 @@ #define TAOS_ERROR_C -typedef struct { - int32_t val; - const char* str; -} STaosError; +// typedef struct { +// int32_t val; +// const char* str; +// } STaosError; static threadlocal int32_t tsErrno; static threadlocal char tsErrMsgDetail[ERR_MSG_LEN] = {0}; @@ -35,7 +35,9 @@ char* taosGetErrMsg() { return tsErrMsgDetail; } char* taosGetErrMsgReturn() { return tsErrMsgReturn; } #ifdef TAOS_ERROR_C -#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg)}, +#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg), .origin = #name}, +STaosError errors[] = { + TAOS_DEFINE_ERROR(TSDB_CODE_SUCCESS, "success") #else #define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = TAOS_DEF_ERROR_CODE(mod, code); #endif @@ -44,11 +46,6 @@ char* taosGetErrMsgReturn() { return tsErrMsgReturn; } #define TAOS_SUCCEEDED(err) ((err) >= 0) #define TAOS_FAILED(err) ((err) < 0) -#ifdef TAOS_ERROR_C -STaosError errors[] = { - {.val = 0, .str = "success"}, -#endif - // rpc TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") @@ -784,7 +781,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open TAOS_DEFINE_ERROR(TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY, "Queue out of memory") -#ifdef TAOS_ERROR_C +#if defined(TAOS_ERROR_INFO) || defined(TAOS_ERROR_C) }; #endif @@ -837,3 +834,5 @@ const char* tstrerror(int32_t err) { } const char* terrstr() { return tstrerror(terrno); } + +int32_t taosGetErrSize() { return sizeof(errors)/sizeof(errors[0]); } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index e8e3348343..89978fd5aa 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -123,4 +123,12 @@ add_test( #add_test( # NAME decompressTest # COMMAND decompressTest -#) \ No newline at end of file +#) + +# terrorTest +add_executable(terrorTest "terrorTest.cpp") +target_link_libraries(terrorTest os util common gtest_main) +add_test( + NAME terrorTest + COMMAND terrorTest +) \ No newline at end of file diff --git a/source/util/test/terrorTest.cpp b/source/util/test/terrorTest.cpp new file mode 100644 index 0000000000..db2f9641ed --- /dev/null +++ b/source/util/test/terrorTest.cpp @@ -0,0 +1,29 @@ +#include +#include + +#define TAOS_ERROR_INFO + +#include +#include "os.h" +#include "osTime.h" +#include "taos.h" +#include "taoserror.h" +#include "tglobal.h" + +extern STaosError errors[]; + +using namespace std; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +TEST(TAOS_ERROR_TEST, terror_test) { + int32_t errSize = taosGetErrSize(); + for (int32_t i = 0; i < errSize; ++i) { + STaosError *pInfo = &errors[i]; + std::cout << i + 1 << " " << pInfo->origin << " " << pInfo->val << std::endl; + } +} \ No newline at end of file From ad0dd88ba1d948e3e0af950acff11f36ecc085cf Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 12 Jun 2024 22:31:08 +0800 Subject: [PATCH 056/103] fix:[TD-30579]compile error in macOS 14.5 and m3 chip --- source/common/src/tvariant.c | 6 +++--- source/libs/function/src/tpercentile.c | 2 +- source/libs/function/src/tudf.c | 4 ++++ source/util/src/tunit.c | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index ae2c8c0c14..06ba3a5a59 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -88,7 +88,7 @@ static int32_t parseSignAndUInteger(const char *z, int32_t n, bool *is_neg, uint if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { return TSDB_CODE_FAILED; } - if (val > UINT64_MAX) { + if (val > (double)UINT64_MAX) { errno = ERANGE; return TSDB_CODE_FAILED; } @@ -172,7 +172,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(z, &endPtr)); - if (!IS_VALID_INT64(val)) { + if(val >= (double)INT64_MIN && val <= (double)INT64_MAX){ return TSDB_CODE_FAILED; } if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { @@ -271,7 +271,7 @@ int32_t toUIntegerEx(const char *z, int32_t n, uint32_t type, uint64_t *value) { } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(p, &endPtr)); - if (!IS_VALID_UINT64(val)) { + if (val < 0 || val > (double)UINT64_MAX) { return TSDB_CODE_FAILED; } if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index c671e7717c..776a7fb95a 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -64,7 +64,7 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) static void resetBoundingBox(MinMaxEntry *range, int32_t type) { if (IS_SIGNED_NUMERIC_TYPE(type)) { range->dMaxVal = INT64_MIN; - range->dMinVal = INT64_MAX; + range->dMinVal = (double)INT64_MAX; } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { range->u64MaxVal = 0; range->u64MinVal = UINT64_MAX; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 7e344866a5..5f7764f342 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -26,6 +26,10 @@ #include "tudf.h" #include "tudfInt.h" +#ifdef _TD_DARWIN_64 +#include +#endif + typedef struct SUdfdData { bool startCalled; bool needCleanUp; diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c index 09f59f1e40..4ec9e39fde 100644 --- a/source/util/src/tunit.c +++ b/source/util/src/tunit.c @@ -24,7 +24,7 @@ #define UNIT_ONE_EXBIBYTE (UNIT_ONE_PEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) static int32_t parseCfgIntWithUnit(const char* str, double *res) { - double val, temp = INT64_MAX; + double val, temp = (double)INT64_MAX; char* endPtr; errno = 0; val = taosStr2Int64(str, &endPtr, 0); From 3bdbb10608555645f44cd8f9a0bc809b92bca8ca Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Wed, 12 Jun 2024 22:39:02 +0800 Subject: [PATCH 057/103] fix:[TD-30579]compile error in macOS 14.5 and m3 chip --- source/common/src/tvariant.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 06ba3a5a59..aad877312b 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -172,7 +172,7 @@ int32_t toIntegerEx(const char *z, int32_t n, uint32_t type, int64_t *value) { } break; case TK_NK_FLOAT: { double val = round(taosStr2Double(z, &endPtr)); - if(val >= (double)INT64_MIN && val <= (double)INT64_MAX){ + if(val < (double)INT64_MIN || val > (double)INT64_MAX){ return TSDB_CODE_FAILED; } if (errno == ERANGE || errno == EINVAL || endPtr - z != n) { From 247183b6fc061c135443e8e2b7e76d409dd0e554 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 13 Jun 2024 06:57:13 +0800 Subject: [PATCH 058/103] enh: support get macro string of taos errors --- include/util/taoserror.h | 2 +- source/util/src/terror.c | 14 ++++++-------- source/util/test/terrorTest.cpp | 12 +----------- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 63b733a337..3207d498d3 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -27,7 +27,7 @@ extern "C" { typedef struct { int32_t val; const char* str; - const char* origin; + const char* macro; } STaosError; #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index f2ae43bc94..56cca26f6e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -21,11 +21,6 @@ #define TAOS_ERROR_C -// typedef struct { -// int32_t val; -// const char* str; -// } STaosError; - static threadlocal int32_t tsErrno; static threadlocal char tsErrMsgDetail[ERR_MSG_LEN] = {0}; static threadlocal char tsErrMsgReturn[ERR_MSG_LEN] = {0}; @@ -35,9 +30,7 @@ char* taosGetErrMsg() { return tsErrMsgDetail; } char* taosGetErrMsgReturn() { return tsErrMsgReturn; } #ifdef TAOS_ERROR_C -#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg), .origin = #name}, -STaosError errors[] = { - TAOS_DEFINE_ERROR(TSDB_CODE_SUCCESS, "success") +#define TAOS_DEFINE_ERROR(name, msg) {.val = (name), .str = (msg), .macro = #name}, #else #define TAOS_DEFINE_ERROR(name, mod, code, msg) static const int32_t name = TAOS_DEF_ERROR_CODE(mod, code); #endif @@ -46,6 +39,11 @@ STaosError errors[] = { #define TAOS_SUCCEEDED(err) ((err) >= 0) #define TAOS_FAILED(err) ((err) < 0) +#ifdef TAOS_ERROR_C +STaosError errors[] = { + TAOS_DEFINE_ERROR(TSDB_CODE_SUCCESS, "success") +#endif + // rpc TAOS_DEFINE_ERROR(TSDB_CODE_RPC_NETWORK_UNAVAIL, "Unable to establish connection") TAOS_DEFINE_ERROR(TSDB_CODE_RPC_FQDN_ERROR, "Unable to resolve FQDN") diff --git a/source/util/test/terrorTest.cpp b/source/util/test/terrorTest.cpp index db2f9641ed..5ef5f43216 100644 --- a/source/util/test/terrorTest.cpp +++ b/source/util/test/terrorTest.cpp @@ -4,26 +4,16 @@ #define TAOS_ERROR_INFO #include -#include "os.h" -#include "osTime.h" -#include "taos.h" #include "taoserror.h" -#include "tglobal.h" extern STaosError errors[]; using namespace std; -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wwrite-strings" -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wsign-compare" - TEST(TAOS_ERROR_TEST, terror_test) { int32_t errSize = taosGetErrSize(); for (int32_t i = 0; i < errSize; ++i) { STaosError *pInfo = &errors[i]; - std::cout << i + 1 << " " << pInfo->origin << " " << pInfo->val << std::endl; + std::cout << i + 1 << " " << pInfo->macro << " " << pInfo->val << std::endl; } } \ No newline at end of file From 962a78d6a399ae8b937a5b36e05bead811d27790 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 13 Jun 2024 06:59:04 +0800 Subject: [PATCH 059/103] enh: support get macro string of taos errors --- source/util/src/terror.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 56cca26f6e..a4067b942b 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -779,7 +779,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDLITE_IVLD_OPEN_DIR, "Invalid TDLite open TAOS_DEFINE_ERROR(TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY, "Queue out of memory") -#if defined(TAOS_ERROR_INFO) || defined(TAOS_ERROR_C) +#ifdef TAOS_ERROR_C }; #endif From b7ef054b38dee1152356fc7239366579981b109c Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 13 Jun 2024 02:11:41 +0000 Subject: [PATCH 060/103] add compress to child table --- source/common/src/tcol.c | 4 ++- source/dnode/vnode/src/vnd/vnodeQuery.c | 36 ++++++++++++++----------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/source/common/src/tcol.c b/source/common/src/tcol.c index ba36558587..a949d0793a 100644 --- a/source/common/src/tcol.c +++ b/source/common/src/tcol.c @@ -327,7 +327,9 @@ int32_t setColCompressByOption(uint8_t type, uint8_t encode, uint16_t compressTy return TSDB_CODE_SUCCESS; } -bool useCompress(uint8_t tableType) { return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType; } +bool useCompress(uint8_t tableType) { + return TSDB_SUPER_TABLE == tableType || TSDB_NORMAL_TABLE == tableType || TSDB_CHILD_TABLE == tableType; +} int8_t validColCompressLevel(uint8_t type, uint8_t level) { if (level == TSDB_COLVAL_LEVEL_DISABLED) return 1; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 19be7e7ebd..5b17e0f1da 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -120,7 +120,8 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { memcpy(metaRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); } if (metaRsp.pSchemaExt) { - code = fillTableColCmpr(&mer1, metaRsp.pSchemaExt, metaRsp.numOfColumns); + SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1; + code = fillTableColCmpr(pReader, metaRsp.pSchemaExt, metaRsp.numOfColumns); if (code < 0) { code = TSDB_CODE_INVALID_MSG; goto _exit; @@ -254,15 +255,18 @@ int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); } - if (useCompress(cfgRsp.tableType)) { - SColCmprWrapper *pColCmpr = &mer1.me.colCmpr; - for (int32_t i = 0; i < cfgRsp.numOfColumns; i++) { - SColCmpr *pCmpr = &pColCmpr->pColCmpr[i]; - SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i; - pSchExt->colId = pCmpr->id; - pSchExt->compress = pCmpr->alg; - } + // if (useCompress(cfgRsp.tableType)) { + + SMetaReader *pReader = mer1.me.type == TSDB_CHILD_TABLE ? &mer2 : &mer1; + SColCmprWrapper *pColCmpr = &pReader->me.colCmpr; + + for (int32_t i = 0; i < cfgRsp.numOfColumns; i++) { + SColCmpr *pCmpr = &pColCmpr->pColCmpr[i]; + SSchemaExt *pSchExt = cfgRsp.pSchemaExt + i; + pSchExt->colId = pCmpr->id; + pSchExt->compress = pCmpr->alg; } + //} // encode and send response rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); @@ -752,13 +756,13 @@ int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64 return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid); } -int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct) { - int32_t code = 0; - SStreamProgressReq req; - SStreamProgressRsp rsp = {0}; - SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0}; - char * buf = NULL; - int32_t rspLen = 0; +int32_t vnodeGetStreamProgress(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { + int32_t code = 0; + SStreamProgressReq req; + SStreamProgressRsp rsp = {0}; + SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0}; + char *buf = NULL; + int32_t rspLen = 0; code = tDeserializeStreamProgressReq(pMsg->pCont, pMsg->contLen, &req); if (code == TSDB_CODE_SUCCESS) { From 6f667d93145a29a0ddeff971a27101ab26f5dd41 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 13 Jun 2024 10:13:24 +0800 Subject: [PATCH 061/103] enh: support get macro string of taos errors --- include/util/taoserror.h | 2 ++ source/util/test/terrorTest.cpp | 4 ---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3207d498d3..95d028a47e 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -30,6 +30,8 @@ typedef struct { const char* macro; } STaosError; +extern STaosError errors[]; + #define TAOS_DEF_ERROR_CODE(mod, code) ((int32_t)((0x80000000 | ((mod)<<16) | (code)))) #define TAOS_SYSTEM_ERROR(code) (0x80ff0000 | (code)) diff --git a/source/util/test/terrorTest.cpp b/source/util/test/terrorTest.cpp index 5ef5f43216..fbb698f780 100644 --- a/source/util/test/terrorTest.cpp +++ b/source/util/test/terrorTest.cpp @@ -1,13 +1,9 @@ #include #include -#define TAOS_ERROR_INFO - #include #include "taoserror.h" -extern STaosError errors[]; - using namespace std; TEST(TAOS_ERROR_TEST, terror_test) { From 849f64d557a744fe6c0dac1f84eb2bdad862e7f1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jun 2024 13:25:29 +0800 Subject: [PATCH 062/103] fix(tsdb): check null for tsdb block load info --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 18 ++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbRead2.c | 3 ++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 2a49c1ba0a..d02ef92429 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -107,15 +107,21 @@ void *destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoa SArray *pList = taosArrayGetP(pLDataIterArray, i); for (int32_t j = 0; j < taosArrayGetSize(pList); ++j) { SLDataIter *pIter = taosArrayGetP(pList, j); + if (pIter->pBlockLoadInfo == NULL) { + continue; + } + + SSttBlockLoadCostInfo* pCost = &pIter->pBlockLoadInfo->cost; if (pLoadCost != NULL) { - pLoadCost->loadBlocks += pIter->pBlockLoadInfo->cost.loadBlocks; - pLoadCost->loadStatisBlocks += pIter->pBlockLoadInfo->cost.loadStatisBlocks; - pLoadCost->blockElapsedTime += pIter->pBlockLoadInfo->cost.blockElapsedTime; - pLoadCost->statisElapsedTime += pIter->pBlockLoadInfo->cost.statisElapsedTime; + pLoadCost->loadBlocks += pCost->loadBlocks; + pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks; + pLoadCost->blockElapsedTime += pCost->blockElapsedTime; + pLoadCost->statisElapsedTime += pCost->statisElapsedTime; } destroyLDataIter(pIter); } + taosArrayDestroy(pList); } @@ -903,6 +909,10 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf, SSttDataInfoF if (pLoadInfo == NULL) { pLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); + if (pLoadInfo == NULL) { + code = terrno; + goto _end; + } } memset(pIter, 0, sizeof(SLDataIter)); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index e1cfcba070..fe44f01917 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -2240,7 +2240,8 @@ static bool initSttBlockReader(SSttBlockReader* pSttBlockReader, STableBlockScan }; SSttDataInfoForTable info = {.pKeyRangeList = taosArrayInit(4, sizeof(SSttKeyRange))}; - int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); + + int32_t code = tMergeTreeOpen2(&pSttBlockReader->mergeTree, &conf, &info); if (code != TSDB_CODE_SUCCESS) { return false; } From 5d2d5c3eee2b279bcdf370f1321b254900aaf2ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jun 2024 13:43:57 +0800 Subject: [PATCH 063/103] fix(tsdb): check for null ptr for block load info when retrieving block distribution. --- source/dnode/vnode/src/tsdb/tsdbReadUtil.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index 8ab7f7eeef..273ec6d797 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -985,6 +985,10 @@ int32_t tsdbGetRowsInSttFiles(STFileSet* pFileSet, SArray* pSttFileBlockIterArra if (pIter->pBlockLoadInfo == NULL) { pIter->pBlockLoadInfo = tCreateSttBlockLoadInfo(pConf->pSchema, pConf->pCols, pConf->numOfCols); + if (pIter->pBlockLoadInfo == NULL) { + tsdbError("failed to create block load info, code: out of memory, %s", pstr); + continue; + } } // load stt blocks statis for all stt-blocks, to decide if the data of queried table exists in current stt file From 6f32a617f709c514b3ff9f2cd4f67d725a6f314a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jun 2024 13:45:42 +0800 Subject: [PATCH 064/103] fix(tsdb): check for malloc failure. --- source/dnode/vnode/src/tsdb/tsdbMergeTree.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index d02ef92429..57b8a99fb1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -46,6 +46,12 @@ SSttBlockLoadInfo *tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, } pLoadInfo->aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); + if (pLoadInfo->aSttBlk == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFreeClear(pLoadInfo); + return NULL; + } + pLoadInfo->pSchema = pSchema; pLoadInfo->colIds = colList; pLoadInfo->numOfCols = numOfCols; From ff24eaf94df39bace0398bd50241e6898d5a97c1 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 13 Jun 2024 14:13:44 +0800 Subject: [PATCH 065/103] fix: code review --- source/libs/executor/src/groupoperator.c | 50 ++++++++++-------------- 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 63be7eaa67..e4c650a50b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -570,45 +570,39 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc SSDataBlock* pDstBlock = createDataBlock(); pDstBlock->info = pDataBlock->info; + pDstBlock->info.id = pOperator->resultDataBlockId; pDstBlock->info.capacity = 0; pDstBlock->info.rowSize = 0; - size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; - int32_t slotId = pExpr->base.pParam[0].pCol->slotId; - SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId); - SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info}; - blockDataAppendColInfo(pDstBlock, &colInfo); - } - - int32_t code = blockDataEnsureCapacity(pDstBlock, pDataBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - blockDataDestroy(pDstBlock); - return NULL; - } - - if (pDataBlock->pBlockAgg != NULL) { + size_t numOfCols = pOperator->exprSupp.numOfExprs; + if (pDataBlock->pBlockAgg) { pDstBlock->pBlockAgg = taosMemoryCalloc(numOfCols, sizeof(SColumnDataAgg)); if (pDstBlock->pBlockAgg == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; blockDataDestroy(pDstBlock); return NULL; } - for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { - SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; - int32_t slotId = pExpr->base.pParam[0].pCol->slotId; - pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId]; - } } for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; int32_t slotId = pExpr->base.pParam[0].pCol->slotId; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, slotId); + SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info}; + blockDataAppendColInfo(pDstBlock, &colInfo); + SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); + int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pDstBlock); + return NULL; + } colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); + + if (pDataBlock->pBlockAgg) { + pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId]; + } } return pDstBlock; @@ -707,13 +701,13 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { T_LONG_JMP(pTaskInfo->env, terrno); } if (pGroupInfo->blockForNotLoaded == NULL) { - pGroupInfo->blockForNotLoaded = taosArrayInit(1, sizeof(SSDataBlock)); + pGroupInfo->blockForNotLoaded = taosArrayInit(0, sizeof(SSDataBlock*)); pGroupInfo->offsetForNotLoaded = 0; } dataNotLoadBlock->info.id.groupId = pGroupInfo->groupId; dataNotLoadBlock->info.dataLoad = 0; pInfo->binfo.pRes->info.rows = pBlock->info.rows; - taosArrayInsert(pGroupInfo->blockForNotLoaded, pGroupInfo->blockForNotLoaded->size, &dataNotLoadBlock); + taosArrayPush(pGroupInfo->blockForNotLoaded, &dataNotLoadBlock); break; } } @@ -863,12 +857,10 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { T_LONG_JMP(pTaskInfo->env, terrno); } if (*(int32_t*)page == 0) { + releaseBufPage(pInfo->pBuf, page); SSDataBlock* ret = buildPartitionResultForNotLoadBlock(pGroupInfo); if (ret != NULL) return ret; - releaseBufPage(pInfo->pBuf, page); - if (pInfo->pageIndex < taosArrayGetSize(pGroupInfo->pPageList)) { - pInfo->pageIndex += 1; - } else if (pInfo->groupIndex + 1 < taosArrayGetSize(pInfo->sortedGroupArray)) { + if (pInfo->groupIndex + 1 < taosArrayGetSize(pInfo->sortedGroupArray)) { pInfo->groupIndex++; pInfo->pageIndex = 0; } else { @@ -951,8 +943,6 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { while (pGroupIter != NULL) { SDataGroupInfo* pGroupInfo = pGroupIter; taosArrayPush(groupArray, pGroupInfo); - static int i = 0; - qInfo("groupArray push %p %p %d times", pGroupInfo, pGroupInfo->blockForNotLoaded, ++i); pGroupIter = taosHashIterate(pInfo->pGroupSet, pGroupIter); } From 6da1215573e5f258951fce1248cab038719712b2 Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 13 Jun 2024 14:47:21 +0800 Subject: [PATCH 066/103] fix blockid --- source/libs/executor/src/groupoperator.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index e4c650a50b..88d5ac4c3c 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -570,7 +570,7 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc SSDataBlock* pDstBlock = createDataBlock(); pDstBlock->info = pDataBlock->info; - pDstBlock->info.id = pOperator->resultDataBlockId; + pDstBlock->info.id.blockId = pOperator->resultDataBlockId; pDstBlock->info.capacity = 0; pDstBlock->info.rowSize = 0; From 0f8c0fa8cbb6c7764e17c9dfbcdd7f15e5e76175 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jun 2024 14:56:20 +0800 Subject: [PATCH 067/103] fix(stream): add dispatch data monitor to handle the network broken problem that may cause the stream process frozen. --- include/libs/stream/tstream.h | 22 +- source/dnode/vnode/src/tq/tq.c | 10 - source/dnode/vnode/src/tqCommon/tqCommon.c | 4 + source/libs/stream/inc/streamInt.h | 47 +- source/libs/stream/src/streamDispatch.c | 557 ++++++++++++--------- source/libs/stream/src/streamExec.c | 2 +- source/libs/stream/src/streamTask.c | 12 +- 7 files changed, 384 insertions(+), 270 deletions(-) diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index bcf081dbfb..f928ba6314 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -205,7 +205,6 @@ typedef struct { typedef struct { char stbFullName[TSDB_TABLE_FNAME_LEN]; - int32_t waitingRspCnt; SUseDbRsp dbInfo; } STaskDispatcherShuffle; @@ -312,15 +311,18 @@ typedef struct SMetaHbInfo SMetaHbInfo; typedef struct SDispatchMsgInfo { SStreamDispatchReq* pData; // current dispatch data - int8_t dispatchMsgType; - int64_t checkpointId;// checkpoint id msg - int32_t transId; // transId for current checkpoint - int16_t msgType; // dispatch msg type - int32_t retryCount; // retry send data count - int64_t startTs; // dispatch start time, record total elapsed time for dispatch - SArray* pRetryList; // current dispatch successfully completed node of downstream - void* pRetryTmr; // used to dispatch data after a given time duration - void* pRspTmr; // used to dispatch data after a given time duration + + int8_t dispatchMsgType; + int64_t checkpointId; // checkpoint id msg + int32_t transId; // transId for current checkpoint + int16_t msgType; // dispatch msg type + int32_t msgId; + int64_t startTs; // dispatch start time, record total elapsed time for dispatch + int64_t rspTs; // latest rsp time + void* pRetryTmr; // used to dispatch data after a given time duration + TdThreadMutex lock; + int8_t inMonitor; + SArray* pSendInfo; // SArray } SDispatchMsgInfo; typedef struct STaskQueue { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a2ca3662d7..2d7c36c6ec 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1012,16 +1012,6 @@ int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { } int32_t tqProcessTaskUpdateCheckpointReq(STQ* pTq, char* msg, int32_t msgLen) { - int32_t vgId = TD_VID(pTq->pVnode); - SVUpdateCheckpointInfoReq* pReq = (SVUpdateCheckpointInfoReq*)msg; - -// if (!pTq->pVnode->restored) { -// tqDebug("vgId:%d update-checkpoint-info msg received during restoring, checkpointId:%" PRId64 -// ", transId:%d s-task:0x%x ignore it", -// vgId, pReq->checkpointId, pReq->transId, pReq->taskId); -// return TSDB_CODE_SUCCESS; -// } - return tqStreamTaskProcessUpdateCheckpointReq(pTq->pStreamMeta, msg, msgLen); } diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index d16c41ec1e..7beee71be5 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -362,6 +362,7 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t vgId = pMeta->vgId; + pRsp->upstreamNodeId = htonl(pRsp->upstreamNodeId); pRsp->upstreamTaskId = htonl(pRsp->upstreamTaskId); pRsp->streamId = htobe64(pRsp->streamId); pRsp->downstreamTaskId = htonl(pRsp->downstreamTaskId); @@ -369,6 +370,9 @@ int32_t tqStreamTaskProcessDispatchRsp(SStreamMeta* pMeta, SRpcMsg* pMsg) { pRsp->stage = htobe64(pRsp->stage); pRsp->msgId = htonl(pRsp->msgId); + tqDebug("s-task:0x%x vgId:%d recv dispatch-rsp from 0x%x vgId:%d", pRsp->upstreamTaskId, pRsp->upstreamNodeId, + pRsp->downstreamTaskId, pRsp->downstreamNodeId); + SStreamTask* pTask = streamMetaAcquireTask(pMeta, pRsp->streamId, pRsp->upstreamTaskId); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index f3ec01cf7a..8bdc0d2343 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -26,20 +26,16 @@ extern "C" { #endif -#define CHECK_RSP_CHECK_INTERVAL 300 -#define LAUNCH_HTASK_INTERVAL 100 -#define WAIT_FOR_MINIMAL_INTERVAL 100.00 -#define MAX_RETRY_LAUNCH_HISTORY_TASK 40 -#define RETRY_LAUNCH_INTERVAL_INC_RATE 1.2 - -#define MAX_BLOCK_NAME_NUM 1024 -#define DISPATCH_RETRY_INTERVAL_MS 300 -#define MAX_CONTINUE_RETRY_COUNT 5 - -#define META_HB_CHECK_INTERVAL 200 -#define META_HB_SEND_IDLE_COUNTER 25 // send hb every 5 sec -#define STREAM_TASK_KEY_LEN ((sizeof(int64_t)) << 1) - +#define CHECK_RSP_CHECK_INTERVAL 300 +#define LAUNCH_HTASK_INTERVAL 100 +#define WAIT_FOR_MINIMAL_INTERVAL 100.00 +#define MAX_RETRY_LAUNCH_HISTORY_TASK 40 +#define RETRY_LAUNCH_INTERVAL_INC_RATE 1.2 +#define MAX_BLOCK_NAME_NUM 1024 +#define DISPATCH_RETRY_INTERVAL_MS 300 +#define META_HB_CHECK_INTERVAL 200 +#define META_HB_SEND_IDLE_COUNTER 25 // send hb every 5 sec +#define STREAM_TASK_KEY_LEN ((sizeof(int64_t)) << 1) #define STREAM_TASK_QUEUE_CAPACITY 20480 #define STREAM_TASK_QUEUE_CAPACITY_IN_SIZE (30) @@ -118,6 +114,14 @@ typedef struct { int32_t taskId; } STaskTriggerSendInfo; +typedef struct { + int32_t nodeId; + int32_t status; + int64_t sendTs; + int64_t rspTs; + int32_t retryCount; +} SDispatchEntry; + typedef struct { int64_t streamId; int64_t recvTs; @@ -143,6 +147,12 @@ typedef enum { EXEC_AFTER_IDLE = 0x1, } EExtractDataCode; +typedef enum ECHECKPOINT_BACKUP_TYPE { + DATA_UPLOAD_DISABLE = -1, + DATA_UPLOAD_S3 = 0, + DATA_UPLOAD_RSYNC = 1, +} ECHECKPOINT_BACKUP_TYPE; + extern void* streamTimer; extern int32_t streamBackendId; extern int32_t streamBackendCfWrapperId; @@ -153,10 +163,9 @@ void streamTimerCleanUp(); void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen); -void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration); +void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration); int32_t streamDispatchStreamBlock(SStreamTask* pTask); void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups); -int32_t getNumOfDispatchBranch(SStreamTask* pTask); void clearBufferedDispatchMsg(SStreamTask* pTask); int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock* pBlock); @@ -204,12 +213,6 @@ int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); void streamMetaRemoveDB(void* arg, char* key); -typedef enum ECHECKPOINT_BACKUP_TYPE { - DATA_UPLOAD_DISABLE = -1, - DATA_UPLOAD_S3 = 0, - DATA_UPLOAD_RSYNC = 1, -} ECHECKPOINT_BACKUP_TYPE; - ECHECKPOINT_BACKUP_TYPE streamGetCheckpointBackupType(); int32_t streamTaskDownloadCheckpointData(const char* id, char* path); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9cdc668ec4..c10d716881 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -23,13 +23,15 @@ typedef struct SBlockName { char parTbName[TSDB_TABLE_NAME_LEN]; } SBlockName; -static void doRetryDispatchData(void* param, void* tmrId); +static void doMonitorDispatchData(void* param, void* tmrId); static int32_t doSendDispatchMsg(SStreamTask* pTask, const SStreamDispatchReq* pReq, int32_t vgId, SEpSet* pEpSet); static int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq); static int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, - int32_t vgSz, int64_t groupId); + int64_t groupId, int64_t now); static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTask* pTask, int32_t vgId, int32_t numOfBlocks, int64_t dstTaskId, int32_t type); +static int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now); +static bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now); void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen) { pMsg->msgType = msgType; @@ -42,7 +44,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas pReq->streamId = pTask->id.streamId; pReq->srcVgId = vgId; pReq->stage = pTask->pMeta->stage; - pReq->msgId = pTask->execInfo.dispatch; + pReq->msgId = pTask->msgInfo.msgId; pReq->upstreamTaskId = pTask->id.taskId; pReq->upstreamChildId = pTask->info.selfChildId; pReq->upstreamNodeId = pTask->info.nodeId; @@ -65,6 +67,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas void streamTaskSendRetrieveRsp(SStreamRetrieveReq *pReq, SRpcMsg* pRsp){ void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp)); ((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId); + SStreamRetrieveRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead)); pCont->streamId = pReq->streamId; pCont->rspToTaskId = pReq->srcTaskId; @@ -216,26 +219,66 @@ void destroyDispatchMsg(SStreamDispatchReq* pReq, int32_t numOfVgroups) { taosMemoryFree(pReq); } -int32_t getNumOfDispatchBranch(SStreamTask* pTask) { - return (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) - ? 1 - : taosArrayGetSize(pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos); -} - void clearBufferedDispatchMsg(SStreamTask* pTask) { SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; if (pMsgInfo->pData != NULL) { - destroyDispatchMsg(pMsgInfo->pData, getNumOfDispatchBranch(pTask)); + destroyDispatchMsg(pMsgInfo->pData, streamTaskGetNumOfDownstream(pTask)); } pMsgInfo->checkpointId = -1; pMsgInfo->transId = -1; pMsgInfo->pData = NULL; pMsgInfo->dispatchMsgType = 0; + + taosThreadMutexLock(&pMsgInfo->lock); + taosArrayClear(pTask->msgInfo.pSendInfo); + taosThreadMutexUnlock(&pMsgInfo->lock); +} + +static SStreamDispatchReq* createDispatchDataReq(SStreamTask* pTask, const SStreamDataBlock* pData) { + int32_t code = 0; + int32_t type = pTask->outputInfo.type; + int32_t num = streamTaskGetNumOfDownstream(pTask); + + ASSERT(type == TASK_OUTPUT__SHUFFLE_DISPATCH || type == TASK_OUTPUT__FIXED_DISPATCH); + + SStreamDispatchReq* pReqs = taosMemoryCalloc(num, sizeof(SStreamDispatchReq)); + if (pReqs == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + if (type == TASK_OUTPUT__SHUFFLE_DISPATCH) { + SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t numOfVgroups = taosArrayGetSize(vgInfo); + + for (int32_t i = 0; i < numOfVgroups; i++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); + code = tInitStreamDispatchReq(&pReqs[i], pTask, pData->srcVgId, 0, pVgInfo->taskId, pData->type); + if (code != TSDB_CODE_SUCCESS) { + destroyDispatchMsg(pReqs, numOfVgroups); + terrno = code; + return NULL; + } + } + } else { + int32_t numOfBlocks = taosArrayGetSize(pData->blocks); + int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; + + code = tInitStreamDispatchReq(pReqs, pTask, pData->srcVgId, numOfBlocks, downstreamTaskId, pData->type); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(pReqs); + terrno = code; + return NULL; + } + } + + return pReqs; } static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = 0; + int64_t now = taosGetTimestampMs(); int32_t numOfBlocks = taosArrayGetSize(pData->blocks); ASSERT(numOfBlocks != 0 && pTask->msgInfo.pData == NULL); @@ -247,48 +290,28 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD pTask->msgInfo.transId = p->info.window.ekey; } + SStreamDispatchReq* pReqs = createDispatchDataReq(pTask, pData); + if (pReqs == NULL) { + stError("s-task:%s failed to create dispatch req", pTask->id.idStr); + return terrno; + } + if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { - SStreamDispatchReq* pReq = taosMemoryCalloc(1, sizeof(SStreamDispatchReq)); - - int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; - code = tInitStreamDispatchReq(pReq, pTask, pData->srcVgId, numOfBlocks, downstreamTaskId, pData->type); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFree(pReq); - return code; - } - for (int32_t i = 0; i < numOfBlocks; i++) { SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); - code = streamAddBlockIntoDispatchMsg(pDataBlock, pReq); + code = streamAddBlockIntoDispatchMsg(pDataBlock, pReqs); if (code != TSDB_CODE_SUCCESS) { - destroyDispatchMsg(pReq, 1); + destroyDispatchMsg(pReqs, 1); return code; } } - pTask->msgInfo.pData = pReq; + pTask->msgInfo.pData = pReqs; } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - int32_t rspCnt = atomic_load_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt); - ASSERT(rspCnt == 0); SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgroups = taosArrayGetSize(vgInfo); - SStreamDispatchReq* pReqs = taosMemoryCalloc(numOfVgroups, sizeof(SStreamDispatchReq)); - if (pReqs == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - for (int32_t i = 0; i < numOfVgroups; i++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, i); - code = tInitStreamDispatchReq(&pReqs[i], pTask, pData->srcVgId, 0, pVgInfo->taskId, pData->type); - if (code != TSDB_CODE_SUCCESS) { - destroyDispatchMsg(pReqs, numOfVgroups); - return code; - } - } - for (int32_t i = 0; i < numOfBlocks; i++) { SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); @@ -304,7 +327,12 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD // it's a new vnode to receive dispatch msg, so add one if (pReqs[j].blockNum == 0) { - atomic_add_fetch_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 1); + SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); + SDispatchEntry entry = {.nodeId = pDstVgroupInfo->vgId, .rspTs = -1, .status = 0, .sendTs = now}; + + taosThreadMutexLock(&pTask->msgInfo.lock); + taosArrayPush(pTask->msgInfo.pSendInfo, &entry); + taosThreadMutexUnlock(&pTask->msgInfo.lock); } pReqs[j].blockNum++; @@ -313,7 +341,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD continue; } - code = streamSearchAndAddBlock(pTask, pReqs, pDataBlock, numOfVgroups, pDataBlock->info.id.groupId); + code = streamSearchAndAddBlock(pTask, pReqs, pDataBlock, pDataBlock->info.id.groupId, now); if (code != 0) { destroyDispatchMsg(pReqs, numOfVgroups); return code; @@ -327,9 +355,9 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64 " %p", pTask->id.idStr, pTask->execInfo.dispatch, pTask->pMeta->stage, pTask->msgInfo.pData); } else { + int32_t numOfBranches = taosArrayGetSize(pTask->msgInfo.pSendInfo); stDebug("s-task:%s build dispatch msg success, msgId:%d, stage:%" PRId64 " dstVgNum:%d %p", pTask->id.idStr, - pTask->execInfo.dispatch, pTask->pMeta->stage, pTask->outputInfo.shuffleDispatcher.waitingRspCnt, - pTask->msgInfo.pData); + pTask->execInfo.dispatch, pTask->pMeta->stage, numOfBranches, pTask->msgInfo.pData); } return code; @@ -337,8 +365,8 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatchMsg) { int32_t code = 0; - int32_t msgId = pTask->execInfo.dispatch; const char* id = pTask->id.idStr; + int32_t msgId = pTask->msgInfo.msgId; if (pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH) { int32_t vgId = pTask->outputInfo.fixedDispatcher.nodeId; @@ -352,10 +380,10 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch } else { SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; int32_t numOfVgroups = taosArrayGetSize(vgInfo); + int32_t numOfBranches = taosArrayGetSize(pTask->msgInfo.pSendInfo); - int32_t actualVgroups = pTask->outputInfo.shuffleDispatcher.waitingRspCnt; stDebug("s-task:%s (child taskId:%d) start to shuffle-dispatch blocks to %d/%d vgroup(s), msgId:%d", id, - pTask->info.selfChildId, actualVgroups, numOfVgroups, msgId); + pTask->info.selfChildId, numOfBranches, numOfVgroups, msgId); int32_t numOfSend = 0; for (int32_t i = 0; i < numOfVgroups; i++) { @@ -370,7 +398,7 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch } // no need to try remain, all already send. - if (++numOfSend == actualVgroups) { + if (++numOfSend == numOfBranches) { break; } } @@ -382,102 +410,154 @@ static int32_t sendDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pDispatch return code; } -static void doRetryDispatchData(void* param, void* tmrId) { - SStreamTask* pTask = param; - const char* id = pTask->id.idStr; - int32_t msgId = pTask->execInfo.dispatch; +static void setNotInDispatchMonitor(SDispatchMsgInfo* pMsgInfo) { + taosThreadMutexLock(&pMsgInfo->lock); + pMsgInfo->inMonitor = 0; + taosThreadMutexUnlock(&pMsgInfo->lock); +} + +static void setResendInfo(SDispatchEntry* pEntry, int64_t now) { + pEntry->sendTs = now; + pEntry->rspTs = -1; + pEntry->retryCount += 1; +} + +static void doSendFailedDispatch(SStreamTask* pTask, SDispatchEntry* pEntry, int64_t now, const char* pMsg) { + SStreamDispatchReq* pReq = pTask->msgInfo.pData; + + int32_t msgId = pTask->msgInfo.msgId; + SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; + int32_t numOfVgroups = taosArrayGetSize(vgInfo); + + setResendInfo(pEntry, now); + for (int32_t j = 0; j < numOfVgroups; ++j) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); + if (pVgInfo->vgId == pEntry->nodeId) { + int32_t code = doSendDispatchMsg(pTask, &pReq[j], pVgInfo->vgId, &pVgInfo->epSet); + stDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d for %s, msgId:%d, code:%s", + pTask->id.idStr, pTask->info.selfChildId, pReq[j].blockNum, pVgInfo->vgId, pMsg, msgId, tstrerror(code)); + break; + } + } +} + +static void doMonitorDispatchData(void* param, void* tmrId) { + SStreamTask* pTask = param; + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; + SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; + int32_t msgId = pMsgInfo->msgId; + int32_t code = 0; + int64_t now = taosGetTimestampMs(); + + stDebug("s-task:%s start monitor dispatch data", id); if (streamTaskShouldStop(pTask)) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); stDebug("s-task:%s should stop, abort from timer, ref:%d", pTask->id.idStr, ref); + setNotInDispatchMonitor(pMsgInfo); return; } - ASSERT(pTask->outputq.status == TASK_OUTPUT_STATUS__WAIT); + // slave task not handle the dispatch, downstream not ready will break the monitor timer + // follower not handle the dispatch rsp + if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stError("s-task:%s vgId:%d follower or downstream not ready, jump out of monitor tmr, ref:%d", id, vgId, ref); + setNotInDispatchMonitor(pMsgInfo); + return; + } - int32_t code = 0; + taosThreadMutexLock(&pMsgInfo->lock); + if (pTask->outputq.status == TASK_OUTPUT_STATUS__NORMAL) { + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s not in dispatch procedure, abort from timer, ref:%d", pTask->id.idStr, ref); + + pTask->msgInfo.inMonitor = 0; + taosThreadMutexUnlock(&pMsgInfo->lock); + return; + } + taosThreadMutexUnlock(&pMsgInfo->lock); + + int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now); + if (numOfFailed == 0) { + stDebug("s-task:%s no error occurs, check again in %dms", id, DISPATCH_RETRY_INTERVAL_MS); + streamStartMonitorDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); + return; + } { - SArray* pList = taosArrayDup(pTask->msgInfo.pRetryList, NULL); - taosArrayClear(pTask->msgInfo.pRetryList); - SStreamDispatchReq* pReq = pTask->msgInfo.pData; if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; - int32_t numOfVgroups = taosArrayGetSize(vgInfo); + stDebug("s-task:%s (child taskId:%d) retry shuffle-dispatch to down streams, msgId:%d", id, + pTask->info.selfChildId, msgId); - int32_t numOfFailed = taosArrayGetSize(pList); - stDebug("s-task:%s (child taskId:%d) retry shuffle-dispatch blocks to %d vgroup(s), msgId:%d", id, - pTask->info.selfChildId, numOfFailed, msgId); + int32_t numOfRetry = 0; + for (int32_t i = 0; i < taosArrayGetSize(pTask->msgInfo.pSendInfo); ++i) { + SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, i); + if (pEntry->status == TSDB_CODE_SUCCESS && pEntry->rspTs > 0) { + continue; + } - for (int32_t i = 0; i < numOfFailed; i++) { - int32_t vgId = *(int32_t*)taosArrayGet(pList, i); + // downstream not rsp yet beyond threshold that is 10s + if (isDispatchRspTimeout(pEntry, now)) { // not respond yet beyonds 30s, re-send data + doSendFailedDispatch(pTask, pEntry, now, "timeout"); + numOfRetry += 1; + continue; + } - for (int32_t j = 0; j < numOfVgroups; ++j) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); - if (pVgInfo->vgId == vgId) { - stDebug("s-task:%s (child taskId:%d) shuffle-dispatch blocks:%d to vgId:%d", pTask->id.idStr, - pTask->info.selfChildId, pReq[j].blockNum, pVgInfo->vgId); + // downstream inputQ is closed + if (pEntry->status == TASK_INPUT_STATUS__BLOCKED) { + doSendFailedDispatch(pTask, pEntry, now, "downstream inputQ blocked"); + numOfRetry += 1; + continue; + } - code = doSendDispatchMsg(pTask, &pReq[j], pVgInfo->vgId, &pVgInfo->epSet); - if (code < 0) { - break; - } - } + // handle other errors + if (pEntry->status != TSDB_CODE_SUCCESS) { + doSendFailedDispatch(pTask, pEntry, now, "downstream error"); + numOfRetry += 1; } } stDebug("s-task:%s complete retry shuffle-dispatch blocks to all %d vnodes, msgId:%d", pTask->id.idStr, - numOfFailed, msgId); + numOfRetry, msgId); } else { - int32_t vgId = pTask->outputInfo.fixedDispatcher.nodeId; + int32_t dstVgId = pTask->outputInfo.fixedDispatcher.nodeId; SEpSet* pEpSet = &pTask->outputInfo.fixedDispatcher.epSet; int32_t downstreamTaskId = pTask->outputInfo.fixedDispatcher.taskId; - stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d", id, - pTask->info.selfChildId, 1, downstreamTaskId, vgId, msgId); + ASSERT(taosArrayGetSize(pTask->msgInfo.pSendInfo) == 1); + SDispatchEntry* pEntry = taosArrayGet(pTask->msgInfo.pSendInfo, 0); - code = doSendDispatchMsg(pTask, pReq, vgId, pEpSet); + setResendInfo(pEntry, now); + code = doSendDispatchMsg(pTask, pReq, dstVgId, pEpSet); + + stDebug("s-task:%s (child taskId:%d) fix-dispatch %d block(s) to s-task:0x%x (vgId:%d), msgId:%d, code:%s", id, + pTask->info.selfChildId, 1, downstreamTaskId, dstVgId, msgId, tstrerror(code)); } - - taosArrayDestroy(pList); } - if (code != TSDB_CODE_SUCCESS) { - if (!streamTaskShouldStop(pTask)) { - // stDebug("s-task:%s reset the waitRspCnt to be 0 before launch retry dispatch", pTask->id.idStr); - // atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); - if (streamTaskShouldPause(pTask)) { - streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS * 10); - } else { - streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); - } - } else { - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s should stop, abort from timer, ref:%d", pTask->id.idStr, ref); - } - } else { + if (streamTaskShouldStop(pTask)) { int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s send success, jump out of timer, ref:%d", pTask->id.idStr, ref); - } -} - -void streamRetryDispatchData(SStreamTask* pTask, int64_t waitDuration) { - pTask->msgInfo.retryCount++; - - stTrace("s-task:%s retry send dispatch data in %" PRId64 "ms, in timer msgId:%d, retryTimes:%d", pTask->id.idStr, - waitDuration, pTask->execInfo.dispatch, pTask->msgInfo.retryCount); - - if (pTask->msgInfo.pRetryTmr != NULL) { - taosTmrReset(doRetryDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr); + stDebug("s-task:%s should stop, abort from timer, ref:%d", pTask->id.idStr, ref); + setNotInDispatchMonitor(pMsgInfo); } else { - pTask->msgInfo.pRetryTmr = taosTmrStart(doRetryDispatchData, waitDuration, pTask, streamTimer); + streamStartMonitorDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); } } -int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz, - int64_t groupId) { +void streamStartMonitorDispatchData(SStreamTask* pTask, int64_t waitDuration) { + if (pTask->msgInfo.pRetryTmr != NULL) { + taosTmrReset(doMonitorDispatchData, waitDuration, pTask, streamTimer, &pTask->msgInfo.pRetryTmr); + } else { + pTask->msgInfo.pRetryTmr = taosTmrStart(doMonitorDispatchData, waitDuration, pTask, streamTimer); + } +} + +int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, + int64_t groupId, int64_t now) { uint32_t hashValue = 0; SArray* vgInfo = pTask->outputInfo.shuffleDispatcher.dbInfo.pVgroupInfos; if (pTask->pNameMap == NULL) { @@ -495,23 +575,24 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } else { char ctbName[TSDB_TABLE_FNAME_LEN] = {0}; if (pDataBlock->info.parTbName[0]) { - if(pTask->subtableWithoutMd5 != 1 && - !isAutoTableName(pDataBlock->info.parTbName) && - !alreadyAddGroupId(pDataBlock->info.parTbName, groupId) && - groupId != 0){ - if(pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER){ + if (pTask->subtableWithoutMd5 != 1 && !isAutoTableName(pDataBlock->info.parTbName) && + !alreadyAddGroupId(pDataBlock->info.parTbName, groupId) && groupId != 0) { + if (pTask->ver == SSTREAM_TASK_SUBTABLE_CHANGED_VER) { buildCtbNameAddGroupId(NULL, pDataBlock->info.parTbName, groupId); - }else if(pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER) { + } else if (pTask->ver > SSTREAM_TASK_SUBTABLE_CHANGED_VER) { buildCtbNameAddGroupId(pTask->outputInfo.shuffleDispatcher.stbFullName, pDataBlock->info.parTbName, groupId); } } } else { buildCtbNameByGroupIdImpl(pTask->outputInfo.shuffleDispatcher.stbFullName, groupId, pDataBlock->info.parTbName); } - snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, pDataBlock->info.parTbName); + + snprintf(ctbName, TSDB_TABLE_NAME_LEN, "%s.%s", pTask->outputInfo.shuffleDispatcher.dbInfo.db, + pDataBlock->info.parTbName); /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ SUseDbRsp* pDbInfo = &pTask->outputInfo.shuffleDispatcher.dbInfo; - hashValue = taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); + hashValue = + taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); SBlockName bln = {0}; bln.hashValue = hashValue; memcpy(bln.parTbName, pDataBlock->info.parTbName, strlen(pDataBlock->info.parTbName)); @@ -520,20 +601,25 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S } } - bool found = false; + bool found = false; + int32_t numOfVgroups = taosArrayGetSize(vgInfo); + // TODO: optimize search - int32_t j; - for (j = 0; j < vgSz; j++) { + taosThreadMutexLock(&pTask->msgInfo.lock); + + for (int32_t j = 0; j < numOfVgroups; j++) { SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); - ASSERT(pVgInfo->vgId > 0); if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { if (streamAddBlockIntoDispatchMsg(pDataBlock, &pReqs[j]) < 0) { + taosThreadMutexUnlock(&pTask->msgInfo.lock); return -1; } if (pReqs[j].blockNum == 0) { - atomic_add_fetch_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 1); + SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); + SDispatchEntry entry = {.nodeId = pDstVgroupInfo->vgId, .rspTs = -1, .status = 0, .sendTs = now}; + taosArrayPush(pTask->msgInfo.pSendInfo, &entry); } pReqs[j].blockNum++; @@ -541,10 +627,28 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S break; } } + + taosThreadMutexUnlock(&pTask->msgInfo.lock); ASSERT(found); return 0; } +static void initDispatchInfo(SDispatchMsgInfo* pInfo, int32_t msgId) { + pInfo->startTs = taosGetTimestampMs(); + pInfo->rspTs = -1; + pInfo->msgId = msgId; +} + +static void clearDispatchInfo(SDispatchMsgInfo* pInfo) { + pInfo->startTs = -1; + pInfo->msgId = -1; + pInfo->rspTs = -1; +} + +static void updateDispatchInfo(SDispatchMsgInfo* pInfo, int64_t recvTs) { + pInfo->rspTs = recvTs; +} + int32_t streamDispatchStreamBlock(SStreamTask* pTask) { ASSERT((pTask->outputInfo.type == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH)); @@ -587,7 +691,7 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { type == STREAM_INPUT__TRANS_STATE); pTask->execInfo.dispatch += 1; - pTask->msgInfo.startTs = taosGetTimestampMs(); + initDispatchInfo(&pTask->msgInfo, pTask->execInfo.dispatch); int32_t code = doBuildDispatchMsg(pTask, pBlock); if (code == 0) { @@ -599,34 +703,21 @@ int32_t streamDispatchStreamBlock(SStreamTask* pTask) { streamTaskInitTriggerDispatchInfo(pTask); } - int32_t retryCount = 0; - while (1) { - code = sendDispatchMsg(pTask, pTask->msgInfo.pData); - if (code == TSDB_CODE_SUCCESS) { - break; - } + code = sendDispatchMsg(pTask, pTask->msgInfo.pData); - stDebug("s-task:%s failed to dispatch msg:%d to downstream, code:%s, output status:%d, retry cnt:%d", id, - pTask->execInfo.dispatch, tstrerror(terrno), pTask->outputq.status, retryCount); - - // todo deal with only partially success dispatch case - atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 0); - if (terrno == TSDB_CODE_APP_IS_STOPPING) { // in case of this error, do not retry anymore - clearBufferedDispatchMsg(pTask); - return code; - } - - if (++retryCount > MAX_CONTINUE_RETRY_COUNT) { // add to timer to retry - int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug( - "s-task:%s failed to dispatch msg to downstream for %d times, code:%s, add timer to retry in %dms, ref:%d", - pTask->id.idStr, retryCount, tstrerror(terrno), DISPATCH_RETRY_INTERVAL_MS, ref); - - streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); - break; - } + taosThreadMutexLock(&pTask->msgInfo.lock); + if (pTask->msgInfo.inMonitor == 0) { + int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s start dispatch monitor tmr in %dms, ref:%d, dispatch code:%s", id, DISPATCH_RETRY_INTERVAL_MS, ref, + tstrerror(code)); + streamStartMonitorDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); + pTask->msgInfo.inMonitor = 1; + } else { + stDebug("s-task:%s already in dispatch monitor tmr", id); } + taosThreadMutexUnlock(&pTask->msgInfo.lock); + // this block can not be deleted until it has been sent to downstream task successfully. return TSDB_CODE_SUCCESS; } @@ -817,8 +908,10 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; ASSERT(dataStrLen > 0); - void* buf = taosMemoryCalloc(1, dataStrLen); - if (buf == NULL) return -1; + void* buf = taosMemoryCalloc(1, dataStrLen); + if (buf == NULL) { + return -1; + } SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)buf; pRetrieve->useconds = 0; @@ -1031,23 +1124,6 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId stDebug("s-task:%s destroy dispatch msg:%p", pTask->id.idStr, pTask->msgInfo.pData); bool delayDispatch = (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); - if (delayDispatch) { - taosThreadMutexLock(&pTask->lock); - // we only set the dispatch msg info for current checkpoint trans - if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK && - pTask->chkInfo.pActiveInfo->activeId == pTask->msgInfo.checkpointId) { - ASSERT(pTask->chkInfo.pActiveInfo->transId == pTask->msgInfo.transId); - stDebug("s-task:%s checkpoint-trigger msg to 0x%x rsp for checkpointId:%" PRId64 " transId:%d confirmed", - pTask->id.idStr, downstreamId, pTask->msgInfo.checkpointId, pTask->msgInfo.transId); - - streamTaskSetTriggerDispatchConfirmed(pTask, downstreamNodeId); - } else { - stWarn("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 " transId:%d discard, since expired", - pTask->id.idStr, pTask->msgInfo.checkpointId, pTask->msgInfo.transId); - } - taosThreadMutexUnlock(&pTask->lock); - } - clearBufferedDispatchMsg(pTask); int64_t el = taosGetTimestampMs() - pTask->msgInfo.startTs; @@ -1074,17 +1150,55 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId return 0; } -int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { - const char* id = pTask->id.idStr; - int32_t vgId = pTask->pMeta->vgId; - int32_t msgId = pTask->execInfo.dispatch; +static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, const char* id) { + int32_t numOfRsp = 0; + bool alreadySet = false; -#if 0 - // for test purpose, build the failure case - if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER) { - pRsp->inputStatus = TASK_INPUT_STATUS__REFUSED; + taosThreadMutexLock(&pMsgInfo->lock); + for(int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { + SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); + if (pEntry->nodeId == vgId) { + ASSERT(!alreadySet); + pEntry->rspTs = now; + pEntry->status = code; + alreadySet = true; + stDebug("s-task:%s record the rps recv, ts:%"PRId64" code:%d, idx:%d", id, now, code, j); + } + + if (pEntry->rspTs != -1) { + numOfRsp += 1; + } } -#endif + + taosThreadMutexUnlock(&pMsgInfo->lock); + return numOfRsp; +} + +bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now) { + return (pEntry->rspTs == -1) && (now - pEntry->sendTs) > 30 * 1000; +} + +int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now) { + int32_t numOfFailed = 0; + taosThreadMutexLock(&pMsgInfo->lock); + + for (int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { + SDispatchEntry* pEntry = taosArrayGet(pMsgInfo->pSendInfo, j); + if (pEntry->status != TSDB_CODE_SUCCESS || isDispatchRspTimeout(pEntry, now)) { + numOfFailed += 1; + } + } + taosThreadMutexUnlock(&pMsgInfo->lock); + return numOfFailed; +} + +int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { + const char* id = pTask->id.idStr; + int32_t vgId = pTask->pMeta->vgId; + SDispatchMsgInfo* pMsgInfo = &pTask->msgInfo; + int32_t msgId = pMsgInfo->msgId; + int64_t now = taosGetTimestampMs(); + int32_t totalRsp = 0; // follower not handle the dispatch rsp if ((pTask->pMeta->role == NODE_ROLE_FOLLOWER) || (pTask->status.downstreamReady != 1)) { @@ -1109,53 +1223,61 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i if (code == TSDB_CODE_STREAM_TASK_NOT_EXIST) { // destination task does not exist, not retry anymore stError("s-task:%s failed to dispatch msg to task:0x%x(vgId:%d), msgId:%d no retry, since task destroyed already", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId, msgId); + totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); } else { stError("s-task:%s failed to dispatch msgId:%d to task:0x%x(vgId:%d), code:%s, add to retry list", id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, tstrerror(code)); - taosThreadMutexLock(&pTask->lock); - taosArrayPush(pTask->msgInfo.pRetryList, &pRsp->downstreamNodeId); - taosThreadMutexUnlock(&pTask->lock); + totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, code, now, id); } } else { // code == 0 if (pRsp->inputStatus == TASK_INPUT_STATUS__BLOCKED) { pTask->inputq.status = TASK_INPUT_STATUS__BLOCKED; // block the input of current task, to push pressure to upstream - taosThreadMutexLock(&pTask->lock); - taosArrayPush(pTask->msgInfo.pRetryList, &pRsp->downstreamNodeId); - taosThreadMutexUnlock(&pTask->lock); + totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, pRsp->inputStatus, now, id); + stTrace("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for retry dispatch", id, + pRsp->downstreamTaskId, pRsp->downstreamNodeId); + } else { + if (pRsp->inputStatus == TASK_INPUT_STATUS__REFUSED) { + // todo handle the role-changed during checkpoint generation, add test case + stError( + "s-task:%s downstream task:0x%x(vgId:%d) refused the dispatch msg, downstream may become follower or " + "restart already, treat it as success", + id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); + } - stTrace("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for %dms and retry dispatch", id, - pRsp->downstreamTaskId, pRsp->downstreamNodeId, DISPATCH_RETRY_INTERVAL_MS); - } else if (pRsp->inputStatus == TASK_INPUT_STATUS__REFUSED) { - // todo handle the agg task failure, add test case - if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER && - pTask->info.taskLevel == TASK_LEVEL__SOURCE) { - stError("s-task:%s failed to dispatch checkpoint-trigger msg, checkpointId:%" PRId64 - ", set the current checkpoint failed, and send rsp to mnode", - id, pTask->chkInfo.pActiveInfo->activeId); - { // send checkpoint failure msg to mnode directly - pTask->chkInfo.pActiveInfo->failedId = pTask->chkInfo.pActiveInfo->activeId; // record the latest failed checkpoint id - pTask->chkInfo.pActiveInfo->activeId = pTask->chkInfo.pActiveInfo->activeId; - streamTaskSendCheckpointSourceRsp(pTask); + totalRsp = setDispatchRspInfo(pMsgInfo, pRsp->downstreamNodeId, TSDB_CODE_SUCCESS, now, id); + + { + bool delayDispatch = (pMsgInfo->dispatchMsgType == STREAM_INPUT__CHECKPOINT_TRIGGER); + if (delayDispatch) { + taosThreadMutexLock(&pTask->lock); + // we only set the dispatch msg info for current checkpoint trans + if (streamTaskGetStatus(pTask)->state == TASK_STATUS__CK && + pTask->chkInfo.pActiveInfo->activeId == pMsgInfo->checkpointId) { + ASSERT(pTask->chkInfo.pActiveInfo->transId == pMsgInfo->transId); + stDebug("s-task:%s checkpoint-trigger msg to 0x%x rsp for checkpointId:%" PRId64 " transId:%d confirmed", + pTask->id.idStr, pRsp->downstreamTaskId, pMsgInfo->checkpointId, pMsgInfo->transId); + + streamTaskSetTriggerDispatchConfirmed(pTask, pRsp->downstreamNodeId); + } else { + stWarn("s-task:%s checkpoint-trigger msg rsp for checkpointId:%" PRId64 + " transId:%d discard, since expired", + pTask->id.idStr, pMsgInfo->checkpointId, pMsgInfo->transId); + } + taosThreadMutexUnlock(&pTask->lock); } - } else { - stError("s-task:%s downstream task:0x%x(vgId:%d) refused the dispatch msg, treat it as success", id, - pRsp->downstreamTaskId, pRsp->downstreamNodeId); } } } - int32_t leftRsp = 0; + int32_t notRsp = taosArrayGetSize(pMsgInfo->pSendInfo) - totalRsp; if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - leftRsp = atomic_sub_fetch_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, 1); - ASSERT(leftRsp >= 0); - - if (leftRsp > 0) { + if (notRsp > 0) { stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, waiting " "for %d rsp", - id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), leftRsp); + id, msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code), notRsp); } else { stDebug( "s-task:%s recv dispatch rsp, msgId:%d from 0x%x(vgId:%d), downstream task input status:%d code:%s, all rsp", @@ -1166,31 +1288,17 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i msgId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->inputStatus, tstrerror(code)); } - ASSERT(leftRsp >= 0); - // all msg rsp already, continue - if (leftRsp == 0) { + if (notRsp == 0) { ASSERT(pTask->outputq.status == TASK_OUTPUT_STATUS__WAIT); // we need to re-try send dispatch msg to downstream tasks - int32_t numOfFailed = taosArrayGetSize(pTask->msgInfo.pRetryList); - if (numOfFailed > 0) { - if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { - atomic_store_32(&pTask->outputInfo.shuffleDispatcher.waitingRspCnt, numOfFailed); - stDebug("s-task:%s waiting rsp set to be %d", id, pTask->outputInfo.shuffleDispatcher.waitingRspCnt); - } - - int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s failed to dispatch msg to downstream, add into timer to retry in %dms, ref:%d", - pTask->id.idStr, DISPATCH_RETRY_INTERVAL_MS, ref); - - streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); - } else { // this message has been sent successfully, let's try next one. - pTask->msgInfo.retryCount = 0; - + int32_t numOfFailed = getFailedDispatchInfo(pMsgInfo, now); + if (numOfFailed == 0) { // this message has been sent successfully, let's try next one. // trans-state msg has been sent to downstream successfully. let's transfer the fill-history task state - if (pTask->msgInfo.dispatchMsgType == STREAM_INPUT__TRANS_STATE) { - stDebug("s-task:%s dispatch trans-state msgId:%d to downstream successfully, start to prepare transfer state", id, msgId); + if (pMsgInfo->dispatchMsgType == STREAM_INPUT__TRANS_STATE) { + stDebug("s-task:%s dispatch trans-state msgId:%d to downstream successfully, start to prepare transfer state", + id, msgId); ASSERT(pTask->info.fillHistory == 1); code = streamTransferStatePrepare(pTask); @@ -1312,4 +1420,3 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S return 0; } - diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 98168abae1..bacab3ac7a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -643,7 +643,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } if (taosGetTimestampMs() - pTask->status.lastExecTs < MIN_INVOKE_INTERVAL) { - stDebug("s-task:%s invoke with high frequency, idle and retry exec in 50ms", id); + stDebug("s-task:%s invoke exec too fast, idle and retry in 50ms", id); streamTaskSetIdleInfo(pTask, MIN_INVOKE_INTERVAL); return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f8a4deb6b1..e96e44c19b 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -283,10 +283,12 @@ void tFreeStreamTask(SStreamTask* pTask) { pTask->status.pSM = streamDestroyStateMachine(pTask->status.pSM); streamTaskDestroyUpstreamInfo(&pTask->upstreamInfo); - pTask->msgInfo.pRetryList = taosArrayDestroy(pTask->msgInfo.pRetryList); taosMemoryFree(pTask->outputInfo.pTokenBucket); taosThreadMutexDestroy(&pTask->lock); + pTask->msgInfo.pSendInfo = taosArrayDestroy(pTask->msgInfo.pSendInfo); + taosThreadMutexDestroy(&pTask->msgInfo.lock); + pTask->outputInfo.pNodeEpsetUpdateList = taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); if ((pTask->status.removeBackendFiles) && (pTask->pMeta != NULL)) { @@ -373,7 +375,13 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i pTask->pMeta = pMeta; pTask->pMsgCb = pMsgCb; - pTask->msgInfo.pRetryList = taosArrayInit(4, sizeof(int32_t)); + pTask->msgInfo.pSendInfo = taosArrayInit(4, sizeof(SDispatchEntry)); + if (pTask->msgInfo.pSendInfo == NULL) { + stError("s-task:%s failed to create sendInfo struct for stream task, code:Out of memory", pTask->id.idStr); + return terrno; + } + + taosThreadMutexInit(&pTask->msgInfo.lock, NULL); TdThreadMutexAttr attr = {0}; From f59b88ea936f0867be00e153aa9994df3400c45e Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 13 Jun 2024 07:03:15 +0000 Subject: [PATCH 068/103] double check enableWhiteList --- source/libs/transport/src/transSvr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 049554a7c9..c205c1412c 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -363,7 +363,7 @@ static bool uvHandleReq(SSvrConn* pConn) { memcpy(pConn->user, pHead->user, strlen(pHead->user)); int8_t forbiddenIp = 0; - if (pThrd->enableIpWhiteList) { + if (pThrd->enableIpWhiteList && tsEnableWhiteList) { forbiddenIp = !uvWhiteListCheckConn(pThrd->pWhiteList, pConn) ? 1 : 0; if (forbiddenIp == 0) { uvWhiteListSetConnVer(pThrd->pWhiteList, pConn); From 367b3b153fa717c0ef7b7c69ee30180e4cec5e62 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Thu, 13 Jun 2024 15:35:26 +0800 Subject: [PATCH 069/103] check cursor for count window --- source/libs/stream/src/streamSessionState.c | 12 +++++++----- source/libs/stream/src/streamState.c | 1 + 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 4c61e6da1d..61f44e9b79 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -516,24 +516,25 @@ SStreamStateCur* countWinStateSeekKeyPrev(SStreamFileState* pFileState, const SS return pBuffCur; } winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); + taosMemoryFreeClear(pVal); + streamStateFreeCur(pBuffCur); if (sessionRangeKeyCmpr(pWinKey, &key) != 0 && winCount == count) { - streamStateFreeCur(pCur); - return pBuffCur; + streamStateCurNext(pFileStore, pCur); + return pCur; } streamStateCurPrev(pFileStore, pCur); while (1) { code = streamStateSessionGetKVByCur_rocksdb(pCur, &key, &pVal, &len); if (code == TSDB_CODE_FAILED) { streamStateCurNext(pFileStore, pCur); - streamStateFreeCur(pBuffCur); return pCur; } winCount = *((COUNT_TYPE*) ((char*)pVal + (resSize - sizeof(COUNT_TYPE)))); + taosMemoryFreeClear(pVal); if (sessionRangeKeyCmpr(pWinKey, &key) == 0 || winCount < count) { streamStateCurPrev(pFileStore, pCur); } else { streamStateCurNext(pFileStore, pCur); - streamStateFreeCur(pBuffCur); return pCur; } } @@ -568,7 +569,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void void* pData = NULL; code = streamStateSessionGetKVByCur_rocksdb(pCur, pKey, &pData, pVLen); if (taosArrayGetSize(pWinStates) > 0 && - (code == TSDB_CODE_FAILED || sessionStateKeyCompare(pKey, pWinStates, 0) >= 0)) { + (code == TSDB_CODE_FAILED || sessionStateRangeKeyCompare(pKey, pWinStates, 0) >= 0)) { transformCursor(pCur->pStreamFileState, pCur); SRowBuffPos* pPos = taosArrayGetP(pWinStates, pCur->buffIndex); if (pVal) { @@ -590,6 +591,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void } int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { + qDebug("move cursor to next"); if (pCur && pCur->buffIndex >= 0) { pCur->buffIndex++; } else { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 47324bd8c9..38d6a5c372 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -654,6 +654,7 @@ int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { #ifdef USE_ROCKSDB + qDebug("move cursor to next"); return streamStateCurPrev_rocksdb(pCur); #else if (!pCur) { From 1f646713fe0ffc3254bec29db350f726d6f26151 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 13 Jun 2024 16:36:18 +0800 Subject: [PATCH 070/103] fix: check range option of alter user --- source/libs/parser/src/parTranslater.c | 16 ++++++++-------- tests/script/tsim/user/basic.sim | 4 ++++ tests/system-test/0-others/user_control.py | 2 +- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 3dbba397ba..5f55841a55 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6605,8 +6605,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS } static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int64_t val, int64_t minVal, - int64_t maxVal) { - if (val >= 0 && (val < minVal || val > maxVal)) { + int64_t maxVal, bool skipMinus) { + if (skipMinus ? ((val >= 0) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 ", valid range: [%" PRId64 ", %" PRId64 "]", pName, val, minVal, maxVal); @@ -6616,12 +6616,12 @@ static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const cha static int32_t checkDbRangeOption(STranslateContext* pCxt, const char* pName, int32_t val, int32_t minVal, int32_t maxVal) { - return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal); + return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_DB_OPTION, pName, val, minVal, maxVal, true); } static int32_t checkTableRangeOption(STranslateContext* pCxt, const char* pName, int64_t val, int64_t minVal, int64_t maxVal) { - return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal); + return checkRangeOption(pCxt, TSDB_CODE_PAR_INVALID_TABLE_OPTION, pName, val, minVal, maxVal, true); } static int32_t checkDbS3KeepLocalOption(STranslateContext* pCxt, SDatabaseOptions* pOptions) { @@ -8485,7 +8485,7 @@ static int32_t translateUseDatabase(STranslateContext* pCxt, SUseDatabaseStmt* p static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pStmt) { int32_t code = 0; SCreateUserReq createReq = {0}; - if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1))) { + if ((code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false))) { return code; } strcpy(createReq.user, pStmt->userName); @@ -8509,13 +8509,13 @@ static int32_t checkAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt) { int32_t code = 0; switch (pStmt->alterType) { case TSDB_ALTER_USER_ENABLE: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "enable", pStmt->enable, 0, 1); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "enable", pStmt->enable, 0, 1, false); break; case TSDB_ALTER_USER_SYSINFO: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "sysinfo", pStmt->sysinfo, 0, 1, false); break; case TSDB_ALTER_USER_CREATEDB: - code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "createdb", pStmt->createdb, 0, 1); + code = checkRangeOption(pCxt, TSDB_CODE_INVALID_OPTION, "createdb", pStmt->createdb, 0, 1, false); break; } return code; diff --git a/tests/script/tsim/user/basic.sim b/tests/script/tsim/user/basic.sim index 353e1d080a..0e1fbb5b40 100644 --- a/tests/script/tsim/user/basic.sim +++ b/tests/script/tsim/user/basic.sim @@ -217,14 +217,18 @@ endi sql_error CREATE USER u100 PASS 'taosdata' SYSINFO -1; sql_error CREATE USER u101 PASS 'taosdata' SYSINFO 2; sql_error CREATE USER u102 PASS 'taosdata' SYSINFO 20000; +sql_error CREATE USER u103 PASS 'taosdata' SYSINFO 1000; sql_error ALTER USER u1 enable -1 sql_error ALTER USER u1 enable 2 +sql_error ALTER USER u1 enable 1000 sql_error ALTER USER u1 enable 10000 sql_error ALTER USER u1 sysinfo -1 sql_error ALTER USER u1 sysinfo 2 +sql_error ALTER USER u1 sysinfo 1000 sql_error ALTER USER u1 sysinfo -20000 sql_error ALTER USER u1 createdb -1 sql_error ALTER USER u1 createdb 3 +sql_error ALTER USER u1 createdb 1000 sql_error ALTER USER u1 createdb 100000 system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index c29170e112..c4d24582e4 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -514,7 +514,7 @@ class TDTestCase: def test_alter_user(self): options = ["enable", "sysinfo", "createdb"] - optionErrVals = [-10000, -128, -1, 2, 127, 10000] + optionErrVals = [-10000, -128, -1, 2, 127, 1000, 10000] for optionErrVal in optionErrVals: tdSql.error("create user user_alter pass 'taosdata' sysinfo %d" % optionErrVal) tdSql.execute("create user user_alter pass 'taosdata'") From d9043d6984659ebe0d5e484a01ec0242a725d96d Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 13 Jun 2024 16:57:49 +0800 Subject: [PATCH 071/103] fix: unexpected udf function --- source/libs/executor/inc/executorInt.h | 2 +- source/libs/executor/src/aggregateoperator.c | 5 ++++- source/libs/executor/src/eventwindowoperator.c | 3 +-- source/libs/executor/src/executorInt.c | 9 +++++++-- source/libs/executor/src/groupoperator.c | 7 +++---- source/libs/executor/src/streamtimewindowoperator.c | 7 +++---- source/libs/executor/src/timewindowoperator.c | 7 ++----- source/libs/function/test/udf2.c | 1 + 8 files changed, 22 insertions(+), 19 deletions(-) diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 592231f043..6c2327d692 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -861,7 +861,7 @@ void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, void setVgIdColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int32_t vgId); void setVgVerColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, int64_t vgVer); -void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); +int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput); SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index b5a49831c5..a7b4532bd0 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -406,7 +406,10 @@ void doSetTableGroupOutputBuf(SOperatorInfo* pOperator, int32_t numOfOutput, uin } } - setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); + int32_t ret = setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); + if (ret != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, ret); + } } // a new buffer page for each table. Needs to opt this design diff --git a/source/libs/executor/src/eventwindowoperator.c b/source/libs/executor/src/eventwindowoperator.c index 29907e6f1f..d73274f85e 100644 --- a/source/libs/executor/src/eventwindowoperator.c +++ b/source/libs/executor/src/eventwindowoperator.c @@ -221,8 +221,7 @@ static int32_t setSingleOutputTupleBufv1(SResultRowInfo* pResultRowInfo, STimeWi (*pResult)->win = *win; - setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); - return TSDB_CODE_SUCCESS; + return setResultRowInitCtx(*pResult, pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); } static void doEventWindowAggImpl(SEventWindowOperatorInfo* pInfo, SExprSupp* pSup, int32_t startIndex, int32_t endIndex, diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 43c04ca8d9..92a777a5cc 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -468,7 +468,7 @@ STimeWindow getAlignQueryTimeWindow(const SInterval* pInterval, int64_t key) { return win; } -void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) { +int32_t setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) { bool init = false; for (int32_t i = 0; i < numOfOutput; ++i) { pCtx[i].resultInfo = getResultEntryInfo(pResult, i, rowEntryInfoOffset); @@ -487,7 +487,11 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO if (!pResInfo->initialized) { if (pCtx[i].functionId != -1) { - pCtx[i].fpSet.init(&pCtx[i], pResInfo); + bool ini = pCtx[i].fpSet.init(&pCtx[i], pResInfo); + if (!ini && fmIsUserDefinedFunc(pCtx[i].functionId)){ + pResInfo->initialized = false; + return TSDB_CODE_UDF_FUNC_EXEC_FAILURE; + } } else { pResInfo->initialized = true; } @@ -495,6 +499,7 @@ void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numO init = true; } } + return TSDB_CODE_SUCCESS; } void clearResultRowInitFlag(SqlFunctionCtx* pCtx, int32_t numOfOutput) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 9a31e993b2..d36c8ddf9c 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -319,7 +319,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, ret); } int32_t rowIndex = j - num; @@ -337,7 +337,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR); + T_LONG_JMP(pTaskInfo->env, ret); } int32_t rowIndex = pBlock->info.rows - num; @@ -1009,8 +1009,7 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, SResultRow* pResultRow = doSetResultOutBufByKey(pBuf, pResultRowInfo, (char*)pData, bytes, true, groupId, pTaskInfo, false, pAggSup, false); - setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); - return TSDB_CODE_SUCCESS; + return setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); } uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId) { diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 2224942893..013b0f2f02 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -548,8 +548,8 @@ int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResu // set time window for current result res->win = (*win); - setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); - return code; + if(code != TSDB_CODE_SUCCESS) return code; + return setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); } bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, @@ -1975,8 +1975,7 @@ static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pR *pResult = (SResultRow*)pWinInfo->pStatePos->pRowBuff; // set time window for current result (*pResult)->win = pWinInfo->sessionWin.win; - setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); - return TSDB_CODE_SUCCESS; + return setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); } int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index b72811cdcc..cdcdc6ddd1 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -84,9 +84,7 @@ static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindo pResultRow->win = (*win); *pResult = pResultRow; - setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); - - return TSDB_CODE_SUCCESS; + return setResultRowInitCtx(pResultRow, pCtx, numOfOutput, rowEntryInfoOffset); } static void doKeepTuple(SWindowRowsSup* pRowSup, int64_t ts, uint64_t groupId) { @@ -1647,8 +1645,7 @@ static int32_t setSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, STimeWind // set time window for current result (*pResult)->win = (*win); - setResultRowInitCtx((*pResult), pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); - return TSDB_CODE_SUCCESS; + return setResultRowInitCtx((*pResult), pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); } static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index faf4daa4e5..e0b08f227f 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -10,6 +10,7 @@ DLL_EXPORT int32_t udf2_init() { return 0; } DLL_EXPORT int32_t udf2_destroy() { return 0; } DLL_EXPORT int32_t udf2_start(SUdfInterBuf* buf) { + if(buf->buf == NULL || buf->bufLen < (sizeof(int64_t))) return TSDB_CODE_UDF_INVALID_BUFSIZE; *(int64_t*)(buf->buf) = 0; buf->bufLen = sizeof(double); buf->numOfResult = 1; From 94d5acba33b93d73229e64aa647c94ee7eea7b06 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 13 Jun 2024 17:36:02 +0800 Subject: [PATCH 072/103] chore: order of error definition --- include/util/taoserror.h | 2 +- source/util/src/terror.c | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 95d028a47e..c3afd2df5a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -325,7 +325,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) #define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C) -#define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) +// #define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) // used #define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E) // #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x diff --git a/source/util/src/terror.c b/source/util/src/terror.c index a4067b942b..239090b8f4 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -245,15 +245,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, "Invalid database name TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, "Too many databases for account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, "Database in dropping status") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_EXIST, "Database not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO, "WAL retention period is zero") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_ACCT, "Invalid database account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_OPTION_UNCHANGED, "Database options not changed") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_INDEX_NOT_EXIST, "Index not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating status") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY, "Inconsistent encryption key") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO, "WAL retention period is zero") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ENCRYPT_KEY, "The cluster has not been set properly for database encryption") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_CREATING, "Database in creating status") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_ENCRYPT_NOT_ALLOW_CHANGE, "Encryption is not allowed to be changed after database is created") // mnode-node TAOS_DEFINE_ERROR(TSDB_CODE_MND_MNODE_ALREADY_EXIST, "Mnode already exists") From 6f1662c2be4ea0cb11ad4d2b4b7e6f600e69521a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jun 2024 17:38:39 +0800 Subject: [PATCH 073/103] fix(test): wait for a little bit more time. --- tests/script/sh/deploy.sh | 4 ++-- tests/script/tsim/stream/checkStreamSTable1.sim | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index 76d890b26a..b69f1eba4f 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -121,7 +121,7 @@ echo "tmrDebugFlag 131" >> $TAOS_CFG echo "uDebugFlag 131" >> $TAOS_CFG echo "rpcDebugFlag 135" >> $TAOS_CFG echo "jniDebugFlag 131" >> $TAOS_CFG -echo "qDebugFlag 131" >> $TAOS_CFG +echo "qDebugFlag 135" >> $TAOS_CFG echo "cDebugFlag 135" >> $TAOS_CFG echo "dDebugFlag 131" >> $TAOS_CFG echo "vDebugFlag 131" >> $TAOS_CFG @@ -136,7 +136,7 @@ echo "idxDebugFlag 135" >> $TAOS_CFG echo "udfDebugFlag 135" >> $TAOS_CFG echo "smaDebugFlag 135" >> $TAOS_CFG echo "metaDebugFlag 135" >> $TAOS_CFG -echo "stDebugFlag 135" >> $TAOS_CFG +echo "stDebugFlag 143" >> $TAOS_CFG echo "numOfLogLines 20000000" >> $TAOS_CFG echo "asyncLog 0" >> $TAOS_CFG echo "locale en_US.UTF-8" >> $TAOS_CFG diff --git a/tests/script/tsim/stream/checkStreamSTable1.sim b/tests/script/tsim/stream/checkStreamSTable1.sim index dd44f5c102..942a947feb 100644 --- a/tests/script/tsim/stream/checkStreamSTable1.sim +++ b/tests/script/tsim/stream/checkStreamSTable1.sim @@ -57,7 +57,7 @@ loop1: sleep 1000 $loop_count = $loop_count + 1 -if $loop_count == 10 then +if $loop_count == 100 then return -1 endi From 7078b5017ca6ad89ec3418564941c7cd52a70eb3 Mon Sep 17 00:00:00 2001 From: kailixu Date: Thu, 13 Jun 2024 17:42:41 +0800 Subject: [PATCH 074/103] chore: order of error definition --- include/util/taoserror.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index c3afd2df5a..a06581bcad 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -325,7 +325,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_MND_DB_OPTION_UNCHANGED TAOS_DEF_ERROR_CODE(0, 0x038A) // #define TSDB_CODE_MND_DB_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x038B) #define TSDB_CODE_MND_DB_RETENTION_PERIOD_ZERO TAOS_DEF_ERROR_CODE(0, 0x038C) -// #define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) // used +// #define TSDB_CODE_MND_INCONSIST_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038D) // unused #define TSDB_CODE_MND_INVALID_ENCRYPT_KEY TAOS_DEF_ERROR_CODE(0, 0x038E) // #define TSDB_CODE_MND_INVALID_DB_OPTION_DAYS TAOS_DEF_ERROR_CODE(0, 0x0390) // 2.x // #define TSDB_CODE_MND_INVALID_DB_OPTION_KEEP TAOS_DEF_ERROR_CODE(0, 0x0391) // 2.x From a707a2921b015e532f6909ef5b23ee39d860f20c Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 13 Jun 2024 17:49:58 +0800 Subject: [PATCH 075/103] fix: udfScalarProcFunc exception --- source/libs/function/src/udfd.c | 2 +- source/libs/function/test/udf2.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 54745951cc..df97e873aa 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -691,7 +691,7 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { convertDataBlockToUdfDataBlock(&call->block, &input); code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx); freeUdfDataDataBlock(&input); - convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); + if(code == 0) convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); freeUdfColumn(&output); break; } diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c index e0b08f227f..273b9c49c2 100644 --- a/source/libs/function/test/udf2.c +++ b/source/libs/function/test/udf2.c @@ -18,6 +18,7 @@ DLL_EXPORT int32_t udf2_start(SUdfInterBuf* buf) { } DLL_EXPORT int32_t udf2(SUdfDataBlock* block, SUdfInterBuf* interBuf, SUdfInterBuf* newInterBuf) { + if(newInterBuf->buf == NULL || newInterBuf->bufLen < (sizeof(double))) return TSDB_CODE_UDF_INVALID_BUFSIZE; double sumSquares = 0; if (interBuf->numOfResult == 1) { sumSquares = *(double*)interBuf->buf; From 325ad5beb8f3917fdfe08851a2746b68b6ff5d82 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 13 Jun 2024 18:25:34 +0800 Subject: [PATCH 076/103] fix(stream):set correct node id --- source/libs/stream/src/streamCheckStatus.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 11fecf7683..b577147171 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -509,7 +509,7 @@ void doSendCheckMsg(SStreamTask* pTask, SDownstreamStatusInfo* p) { STaskOutputInfo* pOutputInfo = &pTask->outputInfo; if (pOutputInfo->type == TASK_OUTPUT__FIXED_DISPATCH) { STaskDispatcherFixed* pDispatch = &pOutputInfo->fixedDispatcher; - setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->taskId); + setCheckDownstreamReqInfo(&req, p->reqId, pDispatch->taskId, pDispatch->nodeId); stDebug("s-task:%s (vgId:%d) stage:%" PRId64 " re-send check downstream task:0x%x(vgId:%d) reqId:0x%" PRIx64, id, pTask->info.nodeId, req.stage, req.downstreamTaskId, req.downstreamNodeId, req.reqId); From 2044cec6ca4e0f6fcb8fff25db5b996f6bec1d8e Mon Sep 17 00:00:00 2001 From: facetosea <25808407@qq.com> Date: Thu, 13 Jun 2024 21:06:58 +0800 Subject: [PATCH 077/103] test --- source/libs/executor/src/streamtimewindowoperator.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 013b0f2f02..2d73cf3cf6 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -548,8 +548,8 @@ int32_t setIntervalOutputBuf(void* pState, STimeWindow* win, SRowBuffPos** pResu // set time window for current result res->win = (*win); - if(code != TSDB_CODE_SUCCESS) return code; - return setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); + setResultRowInitCtx(res, pCtx, numOfOutput, rowEntryInfoOffset); + return code; } bool isDeletedStreamWindow(STimeWindow* pWin, uint64_t groupId, void* pState, STimeWindowAggSupp* pTwSup, From eb72cec8d21176f7bdedae3d16bd09e9bcb8e32e Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 14 Jun 2024 08:10:07 +0800 Subject: [PATCH 078/103] chore: naming optimization --- source/libs/parser/src/parTranslater.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5f55841a55..335324c86f 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -6605,8 +6605,8 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS } static int32_t checkRangeOption(STranslateContext* pCxt, int32_t code, const char* pName, int64_t val, int64_t minVal, - int64_t maxVal, bool skipMinus) { - if (skipMinus ? ((val >= 0) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { + int64_t maxVal, bool skipUndef) { + if (skipUndef ? ((val >= 0) && (val < minVal || val > maxVal)) : (val < minVal || val > maxVal)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, code, "Invalid option %s: %" PRId64 ", valid range: [%" PRId64 ", %" PRId64 "]", pName, val, minVal, maxVal); From 9af6076b4f1aa849b434d91f5117faf49de589f8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jun 2024 14:42:22 +0800 Subject: [PATCH 079/103] fix(stream): add the dispatch entry. --- source/libs/stream/src/streamDispatch.c | 32 ++++++++++++++++++------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index c10d716881..b17d0206f0 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -32,6 +32,7 @@ static int32_t tInitStreamDispatchReq(SStreamDispatchReq* pReq, const SStreamTas int32_t numOfBlocks, int64_t dstTaskId, int32_t type); static int32_t getFailedDispatchInfo(SDispatchMsgInfo* pMsgInfo, int64_t now); static bool isDispatchRspTimeout(SDispatchEntry* pEntry, int64_t now); +static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t now, bool lock); void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen) { pMsg->msgType = msgType; @@ -306,6 +307,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD } } + addDispatchEntry(&pTask->msgInfo, pTask->outputInfo.fixedDispatcher.nodeId, now, true); pTask->msgInfo.pData = pReqs; } else if (pTask->outputInfo.type == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -328,11 +330,7 @@ static int32_t doBuildDispatchMsg(SStreamTask* pTask, const SStreamDataBlock* pD // it's a new vnode to receive dispatch msg, so add one if (pReqs[j].blockNum == 0) { SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); - SDispatchEntry entry = {.nodeId = pDstVgroupInfo->vgId, .rspTs = -1, .status = 0, .sendTs = now}; - - taosThreadMutexLock(&pTask->msgInfo.lock); - taosArrayPush(pTask->msgInfo.pSendInfo, &entry); - taosThreadMutexUnlock(&pTask->msgInfo.lock); + addDispatchEntry(&pTask->msgInfo, pDstVgroupInfo->vgId, now, true); } pReqs[j].blockNum++; @@ -422,6 +420,20 @@ static void setResendInfo(SDispatchEntry* pEntry, int64_t now) { pEntry->retryCount += 1; } +static void addDispatchEntry(SDispatchMsgInfo* pMsgInfo, int32_t nodeId, int64_t now, bool lock) { + SDispatchEntry entry = {.nodeId = nodeId, .rspTs = -1, .status = 0, .sendTs = now}; + + if (lock) { + taosThreadMutexLock(&pMsgInfo->lock); + } + + taosArrayPush(pMsgInfo->pSendInfo, &entry); + + if (lock) { + taosThreadMutexUnlock(&pMsgInfo->lock); + } +} + static void doSendFailedDispatch(SStreamTask* pTask, SDispatchEntry* pEntry, int64_t now, const char* pMsg) { SStreamDispatchReq* pReq = pTask->msgInfo.pData; @@ -618,8 +630,7 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S if (pReqs[j].blockNum == 0) { SVgroupInfo* pDstVgroupInfo = taosArrayGet(vgInfo, j); - SDispatchEntry entry = {.nodeId = pDstVgroupInfo->vgId, .rspTs = -1, .status = 0, .sendTs = now}; - taosArrayPush(pTask->msgInfo.pSendInfo, &entry); + addDispatchEntry(&pTask->msgInfo, pDstVgroupInfo->vgId, now, false); } pReqs[j].blockNum++; @@ -1153,6 +1164,7 @@ static int32_t handleDispatchSuccessRsp(SStreamTask* pTask, int32_t downstreamId static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int32_t code, int64_t now, const char* id) { int32_t numOfRsp = 0; bool alreadySet = false; + bool updated = false; taosThreadMutexLock(&pMsgInfo->lock); for(int32_t j = 0; j < taosArrayGetSize(pMsgInfo->pSendInfo); ++j) { @@ -1162,7 +1174,8 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3 pEntry->rspTs = now; pEntry->status = code; alreadySet = true; - stDebug("s-task:%s record the rps recv, ts:%"PRId64" code:%d, idx:%d", id, now, code, j); + updated = true; + stDebug("s-task:%s record the rsp recv, ts:%"PRId64" code:%d, idx:%d", id, now, code, j); } if (pEntry->rspTs != -1) { @@ -1171,6 +1184,8 @@ static int32_t setDispatchRspInfo(SDispatchMsgInfo* pMsgInfo, int32_t vgId, int3 } taosThreadMutexUnlock(&pMsgInfo->lock); + ASSERT(updated); + return numOfRsp; } @@ -1417,6 +1432,5 @@ int32_t streamProcessDispatchMsg(SStreamTask* pTask, SStreamDispatchReq* pReq, S } streamTrySchedExec(pTask); - return 0; } From b25e6d3250da05e4207cf46c13239b4e42b45446 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jun 2024 14:43:27 +0800 Subject: [PATCH 080/103] fix(stream): always return 0 for enqueuing retrieve block. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index 7beee71be5..c27249cff6 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -418,7 +418,9 @@ int32_t tqStreamTaskProcessRetrieveReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { streamMetaReleaseTask(pMeta, pTask); tCleanupStreamRetrieveReq(&req); - return code; + + // always return success, to disable the auto rsp + return TSDB_CODE_SUCCESS; } int32_t tqStreamTaskProcessCheckReq(SStreamMeta* pMeta, SRpcMsg* pMsg) { From 4490f40c961de428d95d8441d92700e8faa27f3a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jun 2024 14:43:57 +0800 Subject: [PATCH 081/103] fix(stream): do some internal refactor. --- source/libs/stream/inc/streamInt.h | 2 +- source/libs/stream/src/streamData.c | 8 ++++-- source/libs/stream/src/streamTask.c | 44 ++++++++++++++++------------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 8bdc0d2343..be3da64c6a 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -174,7 +174,7 @@ SStreamDataBlock* createStreamBlockFromResults(SStreamQueueItem* pItem, SStreamT SArray* pRes); void destroyStreamDataBlock(SStreamDataBlock* pBlock); -int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); +int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData, const char* idstr); int32_t streamBroadcastToUpTasks(SStreamTask* pTask, const SSDataBlock* pBlock); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index fa4efc3c6e..fae90f4db8 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -104,10 +104,12 @@ void destroyStreamDataBlock(SStreamDataBlock* pBlock) { taosFreeQitem(pBlock); } -int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData) { +int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData, const char* id) { SArray* pArray = taosArrayInit(1, sizeof(SSDataBlock)); if (pArray == NULL) { - return -1; + terrno = TSDB_CODE_OUT_OF_MEMORY; + stError("failed to prepare retrieve block, %s", id); + return terrno; } taosArrayPush(pArray, &(SSDataBlock){0}); @@ -126,7 +128,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock pData->reqId = pReq->reqId; pData->blocks = pArray; - return 0; + return TSDB_CODE_SUCCESS; } SStreamDataSubmit* streamDataSubmitNew(SPackedData* pData, int32_t type) { diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index e96e44c19b..7d869ce538 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -944,32 +944,36 @@ char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { static int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq) { SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, sizeof(SStreamDataBlock)); - int8_t status = TASK_INPUT_STATUS__NORMAL; - - // enqueue - if (pData != NULL) { - stDebug("s-task:%s (child %d) recv retrieve req from task:0x%x(vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, - pTask->info.selfChildId, pReq->srcTaskId, pReq->srcNodeId, pReq->reqId); - - pData->type = STREAM_INPUT__DATA_RETRIEVE; - pData->srcVgId = 0; - streamRetrieveReqToData(pReq, pData); - if (streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pData) == 0) { - status = TASK_INPUT_STATUS__NORMAL; - } else { - status = TASK_INPUT_STATUS__FAILED; - } - } else { // todo handle oom - /*streamTaskInputFail(pTask);*/ - /*status = TASK_INPUT_STATUS__FAILED;*/ + if (pData == NULL) { + stError("s-task:%s failed to allocated retrieve-block", pTask->id.idStr); + return terrno; } - return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; + // enqueue + stDebug("s-task:%s (vgId:%d level:%d) recv retrieve req from task:0x%x(vgId:%d), reqId:0x%" PRIx64, pTask->id.idStr, + pTask->pMeta->vgId, pTask->info.taskLevel, pReq->srcTaskId, pReq->srcNodeId, pReq->reqId); + + pData->type = STREAM_INPUT__DATA_RETRIEVE; + pData->srcVgId = 0; + + int32_t code = streamRetrieveReqToData(pReq, pData, pTask->id.idStr); + if (code != TSDB_CODE_SUCCESS) { + taosFreeQitem(pData); + return code; + } + + code = streamTaskPutDataIntoInputQ(pTask, (SStreamQueueItem*)pData); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s failed to put retrieve-block into inputQ, inputQ is full, discard the retrieve msg", + pTask->id.idStr); + } + + return code; } int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq) { int32_t code = streamTaskEnqueueRetrieve(pTask, pReq); - if(code != 0){ + if (code != 0) { return code; } return streamTrySchedExec(pTask); From 0ef8d3510a4e79bda648c6e2b7aae8b30d1a7cb9 Mon Sep 17 00:00:00 2001 From: kailixu Date: Fri, 14 Jun 2024 15:46:11 +0800 Subject: [PATCH 082/103] fix: use unsigned type for union bit operation --- include/common/tmsg.h | 6 +++--- source/common/src/tmsg.c | 4 ++-- source/dnode/mnode/impl/inc/mndDef.h | 6 +++--- source/dnode/mnode/impl/src/mndUser.c | 4 ++-- source/dnode/mnode/sdb/inc/sdb.h | 4 ++++ source/dnode/mnode/sdb/src/sdbRaw.c | 30 +++++++++++++++++++++++++++ 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fa89ca917a..9301b31f95 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1054,10 +1054,10 @@ typedef struct { int8_t enable; int8_t isView; union { - int8_t flag; + uint8_t flag; struct { - int8_t createdb : 1; - int8_t reserve : 7; + uint8_t createdb : 1; + uint8_t reserve : 7; }; }; char user[TSDB_USER_LEN]; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d465e24d5b..1cbd646413 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1813,7 +1813,7 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) } if (tEncodeI64(&encoder, pReq->privileges) < 0) return -1; ENCODESQL(); - if (tEncodeI8(&encoder, pReq->flag) < 0) return -1; + if (tEncodeU8(&encoder, pReq->flag) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1854,7 +1854,7 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq if (tDecodeI64(&decoder, &pReq->privileges) < 0) return -1; DECODESQL(); if (!tDecodeIsEnd(&decoder)) { - if (tDecodeI8(&decoder, &pReq->flag) < 0) return -1; + if (tDecodeU8(&decoder, &pReq->flag) < 0) return -1; } tEndDecode(&decoder); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index b6fc24a910..46e606d3ea 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -332,10 +332,10 @@ typedef struct { int8_t sysInfo; int8_t enable; union { - int8_t flag; + uint8_t flag; struct { - int8_t createdb : 1; - int8_t reserve : 7; + uint8_t createdb : 1; + uint8_t reserve : 7; }; }; int32_t acctId; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 9a85d405ca..7cde8c5508 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -818,7 +818,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT8(pRaw, dataPos, pUser->superUser, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->sysInfo, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER) - SDB_SET_INT8(pRaw, dataPos, pUser->flag, _OVER) + SDB_SET_UINT8(pRaw, dataPos, pUser->flag, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->passVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) @@ -1002,7 +1002,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pUser->superUser, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->sysInfo, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER) - SDB_GET_INT8(pRaw, dataPos, &pUser->flag, _OVER) + SDB_GET_UINT8(pRaw, dataPos, &pUser->flag, _OVER) if (pUser->superUser) pUser->createdb = 1; SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) if (sver >= 4) { diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 0b2de2b151..fc9b89a141 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -57,6 +57,7 @@ extern "C" { #define SDB_GET_INT32(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt32, int32_t) #define SDB_GET_INT16(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt16, int16_t) #define SDB_GET_INT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawInt8, int8_t) +#define SDB_GET_UINT8(pData, dataPos, val, pos) SDB_GET_VAL(pData, dataPos, val, pos, sdbGetRawUInt8, uint8_t) #define SDB_GET_RESERVE(pRaw, dataPos, valLen, pos) \ { \ @@ -76,6 +77,7 @@ extern "C" { #define SDB_SET_INT32(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt32, int32_t) #define SDB_SET_INT16(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt16, int16_t) #define SDB_SET_INT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawInt8, int8_t) +#define SDB_SET_UINT8(pRaw, dataPos, val, pos) SDB_SET_VAL(pRaw, dataPos, val, pos, sdbSetRawUInt8, uint8_t) #define SDB_SET_BINARY(pRaw, dataPos, val, valLen, pos) \ { \ @@ -388,6 +390,7 @@ void sdbGetCommitInfo(SSdb *pSdb, int64_t *index, int64_t *term, int64_t *config SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen); void sdbFreeRaw(SSdbRaw *pRaw); int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val); +int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val); int32_t sdbSetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t val); int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val); int32_t sdbSetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t val); @@ -395,6 +398,7 @@ int32_t sdbSetRawBinary(SSdbRaw *pRaw, int32_t dataPos, const char *pVal, int32 int32_t sdbSetRawDataLen(SSdbRaw *pRaw, int32_t dataLen); int32_t sdbSetRawStatus(SSdbRaw *pRaw, ESdbStatus status); int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val); +int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val); int32_t sdbGetRawInt16(SSdbRaw *pRaw, int32_t dataPos, int16_t *val); int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val); int32_t sdbGetRawInt64(SSdbRaw *pRaw, int32_t dataPos, int64_t *val); diff --git a/source/dnode/mnode/sdb/src/sdbRaw.c b/source/dnode/mnode/sdb/src/sdbRaw.c index 244e50b52e..4f68139155 100644 --- a/source/dnode/mnode/sdb/src/sdbRaw.c +++ b/source/dnode/mnode/sdb/src/sdbRaw.c @@ -67,6 +67,21 @@ int32_t sdbSetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t val) { return 0; } +int32_t sdbSetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t val) { + if (pRaw == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } + + if (dataPos + sizeof(uint8_t) > pRaw->dataLen) { + terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; + return -1; + } + + *(uint8_t *)(pRaw->pData + dataPos) = val; + return 0; +} + int32_t sdbSetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t val) { if (pRaw == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -174,6 +189,21 @@ int32_t sdbGetRawInt8(SSdbRaw *pRaw, int32_t dataPos, int8_t *val) { return 0; } +int32_t sdbGetRawUInt8(SSdbRaw *pRaw, int32_t dataPos, uint8_t *val) { + if (pRaw == NULL) { + terrno = TSDB_CODE_INVALID_PTR; + return -1; + } + + if (dataPos + sizeof(uint8_t) > pRaw->dataLen) { + terrno = TSDB_CODE_SDB_INVALID_DATA_LEN; + return -1; + } + + *val = *(uint8_t *)(pRaw->pData + dataPos); + return 0; +} + int32_t sdbGetRawInt32(SSdbRaw *pRaw, int32_t dataPos, int32_t *val) { if (pRaw == NULL) { terrno = TSDB_CODE_INVALID_PTR; From 42e0dff589d6e4c2a911563fb3620c52102400bf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jun 2024 17:03:04 +0800 Subject: [PATCH 083/103] fix(stream): move the expansion of the stream task related fill-history task to the place where the fill-history task must have been deoployed already. --- include/dnode/vnode/tqCommon.h | 2 +- include/libs/stream/tstream.h | 10 +++-- source/dnode/snode/src/snode.c | 10 +---- source/dnode/vnode/src/inc/vnodeInt.h | 2 +- source/dnode/vnode/src/tq/tq.c | 7 +++- source/libs/stream/src/streamExec.c | 8 ++-- source/libs/stream/src/streamMeta.c | 46 +++++++-------------- source/libs/stream/src/streamStartHistory.c | 38 +++++++++++++---- 8 files changed, 64 insertions(+), 59 deletions(-) diff --git a/include/dnode/vnode/tqCommon.h b/include/dnode/vnode/tqCommon.h index 0fb690e756..f0ab2a0e95 100644 --- a/include/dnode/vnode/tqCommon.h +++ b/include/dnode/vnode/tqCommon.h @@ -43,7 +43,7 @@ int32_t tqStreamTaskProcessTaskPauseReq(SStreamMeta* pMeta, char* pMsg); int32_t tqStreamTaskProcessTaskResumeReq(void* handle, int64_t sversion, char* pMsg, bool fromVnode); int32_t tqStreamTaskProcessUpdateCheckpointReq(SStreamMeta* pMeta, char* msg, int32_t msgLen); -int32_t tqExpandStreamTask(SStreamTask* pTask, SStreamMeta* pMeta); void tqSetRestoreVersionInfo(SStreamTask* pTask); +int32_t tqExpandStreamTask(SStreamTask* pTask); #endif // TDENGINE_TQ_COMMON_H diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index f928ba6314..03c42e5c7e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -157,7 +157,8 @@ typedef enum EStreamTaskEvent { typedef void FTbSink(SStreamTask* pTask, void* vnode, void* data); typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data); -typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); +typedef int32_t FTaskBuild(void* ahandle, SStreamTask* pTask, int64_t ver); +typedef int32_t FTaskExpand(SStreamTask* pTask); typedef struct { int8_t type; @@ -486,7 +487,8 @@ typedef struct SStreamMeta { SArray* pTaskList; // SArray void* ahandle; TXN* txn; - FTaskExpand* expandFunc; + FTaskBuild* buildTaskFn; + FTaskExpand* expandTaskFn; int32_t vgId; int64_t stage; int32_t role; @@ -710,8 +712,8 @@ SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st); // stream task meta void streamMetaInit(); void streamMetaCleanup(); -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, - startComplete_fn_t fn); +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild expandFunc, FTaskExpand expandTaskFn, + int32_t vgId, int64_t stage, startComplete_fn_t fn); void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); // save to stream meta store int32_t streamMetaRemoveTask(SStreamMeta* pMeta, STaskId* pKey); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index b0d61ebc06..d322eb2977 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -25,14 +25,6 @@ #define sndDebug(...) do { if (sndDebugFlag & DEBUG_DEBUG) { taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__);}} while (0) // clang-format on -static STaskId replaceStreamTaskId(SStreamTask *pTask) { - ASSERT(pTask->info.fillHistory); - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - pTask->id.streamId = pTask->streamTaskId.streamId; - pTask->id.taskId = pTask->streamTaskId.taskId; - return id; -} - static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) { ASSERT(pTask->info.fillHistory); pTask->id.taskId = pId->taskId; @@ -85,7 +77,7 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { startRsync(); pSnode->msgCb = pOption->msgCb; - pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskExpand *)sndExpandTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); + pSnode->pMeta = streamMetaOpen(path, pSnode, (FTaskBuild *)sndExpandTask, tqExpandStreamTask, SNODE_HANDLE, taosGetTimestampMs(), tqStartTaskCompleteCallback); if (pSnode->pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto FAIL; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 7f5ab8b6e6..22b26498e4 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -264,7 +264,7 @@ int32_t tqProcessStreamReqCheckpointRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskChkptReportRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskCheckpointReadyRsp(STQ* pTq, SRpcMsg* pMsg); -int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); +int32_t tqBuildStreamTask(void* pTq, SStreamTask* pTask, int64_t ver); int32_t tqScanWal(STQ* pTq); int tqCommit(STQ*); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 2d7c36c6ec..4963d42f78 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -87,7 +87,8 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { int32_t tqInitialize(STQ* pTq) { int32_t vgId = TD_VID(pTq->pVnode); - pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, vgId, -1, tqStartTaskCompleteCallback); + pTq->pStreamMeta = + streamMetaOpen(pTq->path, pTq, tqBuildStreamTask, tqExpandStreamTask, vgId, -1, tqStartTaskCompleteCallback); if (pTq->pStreamMeta == NULL) { return -1; } @@ -715,7 +716,9 @@ end: static void freePtr(void* ptr) { taosMemoryFree(*(void**)ptr); } -int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t nextProcessVer) { +int32_t tqBuildStreamTask(void* pTqObj, SStreamTask* pTask, int64_t nextProcessVer) { + STQ* pTq = (STQ*) pTqObj; + int32_t vgId = TD_VID(pTq->pVnode); tqDebug("s-task:0x%x start to build task", pTask->id.taskId); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index bacab3ac7a..b0915640cc 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -541,14 +541,14 @@ int32_t streamProcessTransstateBlock(SStreamTask* pTask, SStreamDataBlock* pBloc //static void streamTaskSetIdleInfo(SStreamTask* pTask, int32_t idleTime) { pTask->status.schedIdleTime = idleTime; } static void setLastExecTs(SStreamTask* pTask, int64_t ts) { pTask->status.lastExecTs = ts; } -static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock) { +static void doStreamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pBlock, int32_t num) { const char* id = pTask->id.idStr; int32_t blockSize = 0; int64_t st = taosGetTimestampMs(); SCheckpointInfo* pInfo = &pTask->chkInfo; int64_t ver = pInfo->processedVer; - stDebug("s-task:%s start to process batch of blocks, num:%d, type:%s", id, 1, "checkpoint-trigger"); + stDebug("s-task:%s start to process batch blocks, num:%d, type:%s", id, num, streamQueueItemGetTypeStr(pBlock->type)); doSetStreamInputBlock(pTask, pBlock, &ver, id); @@ -607,7 +607,7 @@ void flushStateDataInExecutor(SStreamTask* pTask, SStreamQueueItem* pCheckpointB } // 2. flush data in executor to K/V store, which should be completed before do checkpoint in the K/V. - doStreamTaskExecImpl(pTask, pCheckpointBlock); + doStreamTaskExecImpl(pTask, pCheckpointBlock, 1); } /** @@ -698,7 +698,7 @@ static int32_t doStreamExecTask(SStreamTask* pTask) { } if (type != STREAM_INPUT__CHECKPOINT) { - doStreamTaskExecImpl(pTask, pInput); + doStreamTaskExecImpl(pTask, pInput, numOfBlocks); streamFreeQitem(pInput); } else { // todo other thread may change the status // do nothing after sync executor state to storage backend, untill the vnode-level checkpoint is completed. diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index da1fec5565..d0f9d40469 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -299,8 +299,8 @@ void streamMetaRemoveDB(void* arg, char* key) { taosThreadMutexUnlock(&pMeta->backendMutex); } -SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId, int64_t stage, - startComplete_fn_t fn) { +SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskBuild buildTaskFn, FTaskExpand expandTaskFn, + int32_t vgId, int64_t stage, startComplete_fn_t fn) { SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); if (pMeta == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -369,7 +369,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->scanInfo.scanCounter = 0; pMeta->vgId = vgId; pMeta->ahandle = ahandle; - pMeta->expandFunc = expandFunc; + pMeta->buildTaskFn = buildTaskFn; + pMeta->expandTaskFn = expandTaskFn; pMeta->stage = stage; pMeta->role = (vgId == SNODE_HANDLE) ? NODE_ROLE_LEADER : NODE_ROLE_UNINIT; pMeta->updateInfo.transId = -1; @@ -602,7 +603,7 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa return 0; } - if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { + if (pMeta->buildTaskFn(pMeta->ahandle, pTask, ver) < 0) { return -1; } @@ -901,7 +902,7 @@ void streamMetaLoadAllTasks(SStreamMeta* pMeta) { STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; void* p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (p == NULL) { - code = pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.checkpointVer + 1); + code = pMeta->buildTaskFn(pMeta->ahandle, pTask, pTask->chkInfo.checkpointVer + 1); if (code < 0) { stError("failed to expand s-task:0x%"PRIx64", code:%s, continue", id.taskId, tstrerror(terrno)); tFreeStreamTask(pTask); @@ -1522,6 +1523,7 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta) { } int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId, __stream_task_expand_fn expandFn) { + int32_t code = 0; int32_t vgId = pMeta->vgId; stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId); @@ -1541,40 +1543,22 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas ASSERT(pTask->status.downstreamReady == 0); if (pTask->pBackend == NULL) { - int32_t code = expandFn(pTask); + code = expandFn(pTask); if (code != TSDB_CODE_SUCCESS) { streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); - streamMetaReleaseTask(pMeta, pTask); - return code; - } - - if (HAS_RELATED_FILLHISTORY_TASK(pTask)) { - SStreamTask* pHTask = streamMetaAcquireTask(pMeta, pTask->hTaskInfo.id.streamId, pTask->hTaskInfo.id.taskId); - if (pHTask != NULL) { - if (pHTask->pBackend == NULL) { - code = expandFn(pHTask); - if (code != TSDB_CODE_SUCCESS) { - streamMetaAddFailedTaskSelf(pHTask, pInfo->readyTs); - - streamMetaReleaseTask(pMeta, pHTask); - streamMetaReleaseTask(pMeta, pTask); - return code; - } - } - - streamMetaReleaseTask(pMeta, pHTask); - } } } - int32_t ret = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); - if (ret != TSDB_CODE_SUCCESS) { - stError("s-task:%s vgId:%d failed to handle event:%d", pTask->id.idStr, pMeta->vgId, TASK_EVENT_INIT); - streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); + if (code == TSDB_CODE_SUCCESS) { + code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); + if (code != TSDB_CODE_SUCCESS) { + stError("s-task:%s vgId:%d failed to handle event:%d", pTask->id.idStr, pMeta->vgId, TASK_EVENT_INIT); + streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); + } } streamMetaReleaseTask(pMeta, pTask); - return ret; + return code; } static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ) { diff --git a/source/libs/stream/src/streamStartHistory.c b/source/libs/stream/src/streamStartHistory.c index 7a864a60d2..050d88aaf1 100644 --- a/source/libs/stream/src/streamStartHistory.c +++ b/source/libs/stream/src/streamStartHistory.c @@ -197,6 +197,9 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { const char* idStr = pTask->id.idStr; int64_t hStreamId = pTask->hTaskInfo.id.streamId; int32_t hTaskId = pTask->hTaskInfo.id.taskId; + int64_t now = taosGetTimestampMs(); + int32_t code = 0; + ASSERT(hTaskId != 0); // check stream task status in the first place. @@ -226,7 +229,18 @@ int32_t streamLaunchFillHistoryTask(SStreamTask* pTask) { stDebug("s-task:%s fill-history task is ready, no need to check downstream", pHisTask->id.idStr); streamMetaAddTaskLaunchResult(pMeta, hStreamId, hTaskId, pExecInfo->checkTs, pExecInfo->readyTs, true); } else { // exist, but not ready, continue check downstream task status - checkFillhistoryTaskStatus(pTask, pHisTask); + if (pHisTask->pBackend == NULL) { + code = pMeta->expandTaskFn(pHisTask); + if (code != TSDB_CODE_SUCCESS) { + streamMetaAddFailedTaskSelf(pHisTask, now); + stError("s-task:%s failed to expand fill-history task, code:%s", pHisTask->id.idStr, tstrerror(code)); + } + } + + if (code == TSDB_CODE_SUCCESS) { + checkFillhistoryTaskStatus(pTask, pHisTask); + } + } streamMetaReleaseTask(pMeta, pHisTask); @@ -306,6 +320,7 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { SLaunchHTaskInfo* pInfo = param; SStreamMeta* pMeta = pInfo->pMeta; int64_t now = taosGetTimestampMs(); + int32_t code = 0; streamMetaWLock(pMeta); @@ -362,13 +377,22 @@ void tryLaunchHistoryTask(void* param, void* tmrId) { streamMetaReleaseTask(pMeta, pTask); return; } else { - checkFillhistoryTaskStatus(pTask, pHTask); - streamMetaReleaseTask(pMeta, pHTask); + if (pHTask->pBackend == NULL) { + code = pMeta->expandTaskFn(pHTask); + if (code != TSDB_CODE_SUCCESS) { + streamMetaAddFailedTaskSelf(pHTask, now); + stError("failed to expand fill-history task:%s, code:%s", pHTask->id.idStr, tstrerror(code)); + } + } - // not in timer anymore - int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:0x%x fill-history task launch completed, retry times:%d, ref:%d", (int32_t)pInfo->id.taskId, - pHTaskInfo->retryTimes, ref); + if (code == TSDB_CODE_SUCCESS) { + checkFillhistoryTaskStatus(pTask, pHTask); + // not in timer anymore + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:0x%x fill-history task launch completed, retry times:%d, ref:%d", (int32_t)pInfo->id.taskId, + pHTaskInfo->retryTimes, ref); + } + streamMetaReleaseTask(pMeta, pHTask); } } From b44d87475d95a2afdb3480384aa5297fc8c9d2f7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 14 Jun 2024 17:54:44 +0800 Subject: [PATCH 084/103] fix(test): fix syntax error. --- source/libs/stream/test/backendTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp index c9b981e5f9..dc506cfbc9 100644 --- a/source/libs/stream/test/backendTest.cpp +++ b/source/libs/stream/test/backendTest.cpp @@ -43,7 +43,7 @@ SStreamState *stateCreate(const char *path) { pTask->ver = 1024; pTask->id.streamId = 1023; pTask->id.taskId = 1111111; - SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, 0, 0, NULL); + SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL); pTask->pMeta = pMeta; SStreamState *p = streamStateOpen((char *)path, pTask, true, 32, 32 * 1024); From 19f6766c9a181aeb27ce80e3dd63c4165e99585e Mon Sep 17 00:00:00 2001 From: factosea <285808407@qq.com> Date: Thu, 13 Jun 2024 19:14:44 +0800 Subject: [PATCH 085/103] fix: blockAgg --- include/common/tdatablock.h | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 7 ++++++- source/libs/executor/src/executorInt.c | 2 +- source/libs/executor/src/groupoperator.c | 21 +++++++++++---------- source/libs/function/src/detail/tminmax.c | 23 ++++++++--------------- tests/army/test.py | 2 ++ 6 files changed, 29 insertions(+), 28 deletions(-) diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 722f18c52d..ec998e9365 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -102,7 +102,7 @@ static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, u return false; } - if (pColAgg != NULL) { + if (pColAgg != NULL && pColAgg->colId != -1) { if (pColAgg->numOfNull == totalRows) { ASSERT(pColumnInfoData->nullbitmap == NULL); return true; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 0771e4d5bf..b4aca0a8a1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4959,6 +4959,12 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, if (pResBlock->pBlockAgg == NULL) { size_t num = taosArrayGetSize(pResBlock->pDataBlock); pResBlock->pBlockAgg = taosMemoryCalloc(num, sizeof(SColumnDataAgg)); + if (pResBlock->pBlockAgg == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for(int i = 0; i < num; ++i) { + pResBlock->pBlockAgg[i].colId = -1; + } } // do fill all null column value SMA info @@ -4976,7 +4982,6 @@ int32_t tsdbRetrieveDatablockSMA2(STsdbReader* pReader, SSDataBlock* pDataBlock, } else if (pAgg->colId < pSup->colId[j]) { i += 1; } else if (pSup->colId[j] < pAgg->colId) { - pResBlock->pBlockAgg[pSup->slotId[j]].colId = -1; *allHave = false; j += 1; } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 03b9a374c1..9326bfec43 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -435,7 +435,7 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB if (pFuncParam->type == FUNC_PARAM_TYPE_COLUMN) { int32_t slotId = pFuncParam->pCol->slotId; pInput->pColumnDataAgg[j] = &pBlock->pBlockAgg[slotId]; - if (pInput->pColumnDataAgg[j] == NULL) { + if (pInput->pColumnDataAgg[j]->colId == -1) { pInput->colDataSMAIsSet = false; } diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 88d5ac4c3c..0feaedaeef 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -591,17 +591,19 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info}; blockDataAppendColInfo(pDstBlock, &colInfo); + pDstBlock->pBlockAgg[i].colId = -1; SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); - int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - blockDataDestroy(pDstBlock); - return NULL; - } - colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); - - if (pDataBlock->pBlockAgg) { + if (pDataBlock->pBlockAgg && pDataBlock->pBlockAgg[slotId].colId != -1) { pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId]; + } else { + int32_t code = doEnsureCapacity(pDst, &pDstBlock->info, pDataBlock->info.rows, false); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pDstBlock); + return NULL; + } + + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } } @@ -706,7 +708,6 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } dataNotLoadBlock->info.id.groupId = pGroupInfo->groupId; dataNotLoadBlock->info.dataLoad = 0; - pInfo->binfo.pRes->info.rows = pBlock->info.rows; taosArrayPush(pGroupInfo->blockForNotLoaded, &dataNotLoadBlock); break; } diff --git a/source/libs/function/src/detail/tminmax.c b/source/libs/function/src/detail/tminmax.c index a6c91a57ce..653b8adad7 100644 --- a/source/libs/function/src/detail/tminmax.c +++ b/source/libs/function/src/detail/tminmax.c @@ -702,23 +702,16 @@ static void doExtractVal(SColumnInfoData* pCol, int32_t i, int32_t end, SqlFunct } } -static int32_t saveRelatedTuple(SqlFunctionCtx* pCtx, SInputColumnInfoData* pInput, int32_t index, void* tval) { +static int32_t saveRelatedTupleTag(SqlFunctionCtx* pCtx, SInputColumnInfoData* pInput, void* tval) { SColumnInfoData* pCol = pInput->pData[0]; SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SMinmaxResInfo* pBuf = GET_ROWCELL_INTERBUF(pResInfo); - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; if (pCtx->subsidiaries.num > 0) { - index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - if (index >= 0) { - code = saveTupleData(pCtx, index, pCtx->pSrcBlock, &pBuf->tuplePos); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } + code = saveTupleData(pCtx, 0, pCtx->pSrcBlock, &pBuf->tuplePos); } - return code; } @@ -758,7 +751,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) pBuf->v = GET_INT64_VAL(tval); } - code = saveRelatedTuple(pCtx, pInput, index, tval); + code = saveRelatedTupleTag(pCtx, pInput, tval); } else { if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t prev = 0; @@ -767,7 +760,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { GET_INT64_VAL(&pBuf->v) = val; - code = saveRelatedTuple(pCtx, pInput, index, tval); + code = saveRelatedTupleTag(pCtx, pInput, tval); } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { uint64_t prev = 0; @@ -776,7 +769,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { GET_UINT64_VAL(&pBuf->v) = val; - code = saveRelatedTuple(pCtx, pInput, index, tval); + code = saveRelatedTupleTag(pCtx, pInput, tval); } } else if (type == TSDB_DATA_TYPE_DOUBLE) { double prev = 0; @@ -785,7 +778,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) double val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { GET_DOUBLE_VAL(&pBuf->v) = val; - code = saveRelatedTuple(pCtx, pInput, index, tval); + code = saveRelatedTupleTag(pCtx, pInput, tval); } } else if (type == TSDB_DATA_TYPE_FLOAT) { float prev = 0; @@ -794,7 +787,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc, int32_t* nElems) float val = GET_DOUBLE_VAL(tval); if ((prev < val) ^ isMinFunc) { GET_FLOAT_VAL(&pBuf->v) = val; - code = saveRelatedTuple(pCtx, pInput, index, tval); + code = saveRelatedTupleTag(pCtx, pInput, tval); } } } diff --git a/tests/army/test.py b/tests/army/test.py index dda5d7d5b0..332d7f29c4 100644 --- a/tests/army/test.py +++ b/tests/army/test.py @@ -657,8 +657,10 @@ if __name__ == "__main__": conn = taos.connect(host=f"{host}", config=tdDnodes.getSimCfgPath()) if fileName == "all": + tdLog.info("Procedures for testing runAllLinux") tdCases.runAllLinux(conn) else: + tdLog.info(f"Procedures for testing runOneLinux {fileName}") tdCases.runOneLinux(conn, fileName, replicaVar) # do restart option From 65044b39643a58baa25863319996ddcef422f2bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Sat, 15 Jun 2024 20:56:58 +0800 Subject: [PATCH 086/103] colId init --- source/libs/executor/src/groupoperator.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 0feaedaeef..710db0045d 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -582,6 +582,9 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc blockDataDestroy(pDstBlock); return NULL; } + for(int i = 0; i < numOfCols; ++i) { + pDstBlock->pBlockAgg[i].colId = -1; + } } for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) { @@ -591,7 +594,6 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc SColumnInfoData colInfo = {.hasNull = true, .info = pSrc->info}; blockDataAppendColInfo(pDstBlock, &colInfo); - pDstBlock->pBlockAgg[i].colId = -1; SColumnInfoData* pDst = taosArrayGet(pDstBlock->pDataBlock, i); if (pDataBlock->pBlockAgg && pDataBlock->pBlockAgg[slotId].colId != -1) { pDstBlock->pBlockAgg[i] = pDataBlock->pBlockAgg[slotId]; From 147ea5c040fb93326a56bb00a782a7290a557ef8 Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Sun, 16 Jun 2024 09:49:17 +0800 Subject: [PATCH 087/103] Update taostools_CMakeLists.txt.in --- cmake/taostools_CMakeLists.txt.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 9bbda8309f..9a6a5329ae 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG main + GIT_TAG 3.0 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE From 2e900f1787d8c94a55d2a12856440654b98aaf8c Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 17 Jun 2024 10:05:03 +0800 Subject: [PATCH 088/103] adj stream doc --- docs/en/12-taos-sql/14-stream.md | 2 +- docs/zh/12-taos-sql/14-stream.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index e7cefc1d7a..bbcffff062 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -47,7 +47,7 @@ window_clause: { } ``` -`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically. +`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.The _wend of this window is the time of the last data plus tol_val. `EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns. diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index cc057c3b72..bf95a29baa 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -54,8 +54,10 @@ window_clause: { } ``` -其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。 +其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的_wend等于最后一条数据的时间加上tol_val。 + EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 + COUNT_WINDOW 是计数窗口,按固定的数据行数来划分窗口。 count_val 是常量，是正整数，必须大于等于2，小于2147483648。 count_val 表示每个 COUNT_WINDOW 包含的最大数据行数，总数据行数不能整除 count_val 时，最后一个窗口的行数会小于 count_val 。 sliding_val 是常量，表示窗口滑动的数量，类似于 INTERVAL 的 SLIDING 。 窗口的定义与时序数据特色查询中的定义完全相同，详见 [TDengine 特色查询](../distinguished) From 6a10cea339417b842761dcf31ed5e5b1b4b911c0 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 17 Jun 2024 10:07:10 +0800 Subject: [PATCH 089/103] adj stream doc --- docs/en/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index bbcffff062..fcd782b429 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -47,7 +47,7 @@ window_clause: { } ``` -`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.The _wend of this window is the time of the last data plus tol_val. +`SESSION` indicates a session window, and `tol_val` indicates the maximum range of the time interval. If the time interval between two continuous rows are within the time interval specified by `tol_val` they belong to the same session window; otherwise a new session window is started automatically.The `_wend` of this window is the time of the last data plus `tol_val`. `EVENT_WINDOW` is determined according to the window start condition and the window close condition. The window is started when `start_trigger_condition` is evaluated to true, the window is closed when `end_trigger_condition` is evaluated to true. `start_trigger_condition` and `end_trigger_condition` can be any conditional expressions supported by TDengine and can include multiple columns. From ffdef2d1c8e1aa9a34aac5e469d982e08345450f Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 17 Jun 2024 10:08:36 +0800 Subject: [PATCH 090/103] adj stream doc --- docs/zh/12-taos-sql/14-stream.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index bf95a29baa..38d913dfaf 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -54,7 +54,7 @@ window_clause: { } ``` -其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的_wend等于最后一条数据的时间加上tol_val。 +其中，SESSION 是会话窗口，tol_val 是时间间隔的最大范围。在 tol_val 时间间隔范围内的数据都属于同一个窗口，如果连续的两条数据的时间超过 tol_val，则自动开启下一个窗口。该窗口的 _wend 等于最后一条数据的时间加上 tol_val。 EVENT_WINDOW 是事件窗口，根据开始条件和结束条件来划定窗口。当 start_trigger_condition 满足时则窗口开始，直到 end_trigger_condition 满足时窗口关闭。 start_trigger_condition 和 end_trigger_condition 可以是任意 TDengine 支持的条件表达式，且可以包含不同的列。 From 2298b4eae8745395b8b4ba41a8469d102a7da0b4 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 17 Jun 2024 11:19:10 +0800 Subject: [PATCH 091/103] adj log --- source/libs/stream/src/streamState.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 38d6a5c372..55e06cdcaf 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -654,7 +654,7 @@ int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur) { int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur) { #ifdef USE_ROCKSDB - qDebug("move cursor to next"); + qTrace("move cursor to next"); return streamStateCurPrev_rocksdb(pCur); #else if (!pCur) { From 4474788e362713e0d27cd0f2c8d718704849e23e Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao> Date: Mon, 17 Jun 2024 11:24:30 +0800 Subject: [PATCH 092/103] adj log --- source/libs/stream/src/streamSessionState.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index 61f44e9b79..005fd1603c 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -591,7 +591,7 @@ int32_t sessionWinStateGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void } int32_t sessionWinStateMoveToNext(SStreamStateCur* pCur) { - qDebug("move cursor to next"); + qTrace("move cursor to next"); if (pCur && pCur->buffIndex >= 0) { pCur->buffIndex++; } else { From 848b1e19b5c740069ceb057a637d08aa39b8eb79 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 14:39:46 +0800 Subject: [PATCH 093/103] fix(stream): fix some typo. --- source/dnode/vnode/src/tq/tqUtil.c | 5 +++-- source/libs/stream/src/streamMeta.c | 12 +++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 33e3414a7d..642de6c3f8 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -605,12 +605,13 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b numOfTasks = taosArrayGetSize(pMeta->pTaskList); for (int32_t i = 0; i < numOfTasks; ++i) { - STaskId* pId = taosArrayGet(pMeta->pTaskList, i); + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); if (pId->streamId != streamId) { continue; } - SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL) { tqError("vgId:%d failed to acquire task:0x%" PRIx64 " in retrieving progress", pMeta->vgId, pId->taskId); continue; diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index d0f9d40469..3c50156fad 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -1007,9 +1007,10 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { hbMsg.pUpdateNodes = taosArrayInit(numOfTasks, sizeof(int32_t)); for (int32_t i = 0; i < numOfTasks; ++i) { - STaskId* pId = taosArrayGet(pMeta->pTaskList, i); + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); - SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, pId, sizeof(*pId)); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask** pTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (pTask == NULL) { continue; } @@ -1020,7 +1021,7 @@ static int32_t metaHeartbeatToMnodeImpl(SStreamMeta* pMeta) { } STaskStatusEntry entry = { - .id = *pId, + .id = id, .status = streamTaskGetStatus(*pTask)->state, .nodeId = hbMsg.vgId, .stage = pMeta->stage, @@ -1508,8 +1509,9 @@ int32_t streamMetaStopAllTasks(SStreamMeta* pMeta) { bool streamMetaAllTasksReady(const SStreamMeta* pMeta) { int32_t num = taosArrayGetSize(pMeta->pTaskList); for (int32_t i = 0; i < num; ++i) { - STaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); - SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, pTaskId, sizeof(*pTaskId)); + SStreamTaskId* pId = taosArrayGet(pMeta->pTaskList, i); + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL) { continue; } From 3aa6e00bee01b22d3689b1e65db67f731645bd8d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 15:58:59 +0800 Subject: [PATCH 094/103] fix(stream): fix some typo. --- source/dnode/vnode/src/tq/tqUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c index 642de6c3f8..5290c39d42 100644 --- a/source/dnode/vnode/src/tq/tqUtil.c +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -613,7 +613,7 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; SStreamTask** ppTask = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask == NULL) { - tqError("vgId:%d failed to acquire task:0x%" PRIx64 " in retrieving progress", pMeta->vgId, pId->taskId); + tqError("vgId:%d failed to acquire task:0x%x in retrieving progress", pMeta->vgId, pId->taskId); continue; } From bdbcdf34655b9f56e1ce5db787e3f283cae5cd99 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 16:58:58 +0800 Subject: [PATCH 095/103] refactor: add two assert. --- source/libs/stream/src/streamMeta.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 3c50156fad..8433531f71 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -744,10 +744,16 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t pTask = *ppTask; // it is an fill-history task, remove the related stream task's id that points to it - atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); + if (pTask->info.fillHistory == 0) { + atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); + } + + ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); + + ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); streamMetaRemoveTask(pMeta, &id); streamMetaWUnLock(pMeta); From 0f4ffdf49a96ab594f7fce16189a89dbf2aae474 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 18:57:31 +0800 Subject: [PATCH 096/103] fix(stream): add some info. --- source/libs/stream/src/streamMeta.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 8433531f71..bca0f06d5d 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -673,13 +673,21 @@ void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) } static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamTaskId* id) { + bool remove = false; for (int32_t i = 0; i < num; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); if (pTaskId->streamId == id->streamId && pTaskId->taskId == id->taskId) { taosArrayRemove(pMeta->pTaskList, i); + stDebug("vgId:%d remove streamId:0x%" PRIx64 " taskId:0x%x", pMeta->vgId, id->streamId, id->taskId); + remove = true; break; + } else { + stDebug("vgId:%d remove streamId:0x%" PRIx64 " taskId:0x%x, entry:0x%" PRIx64 "-0x%x", pMeta->vgId, id->streamId, + id->taskId, pTaskId->streamId, pTaskId->taskId); } } + + ASSERT(remove); } static int32_t streamTaskSendTransSuccessMsg(SStreamTask* pTask, void* param) { From 60fac803459c81fdce2635c8bcd0eec08e14cdad Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 19:34:09 +0800 Subject: [PATCH 097/103] fix(stream): add some logs. --- source/dnode/vnode/src/tqCommon/tqCommon.c | 2 +- source/libs/stream/src/streamMeta.c | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index c27249cff6..d9bc8b74d2 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -635,8 +635,8 @@ int32_t tqStreamTaskProcessDropReq(SStreamMeta* pMeta, char* msg, int32_t msgLen // drop the related fill-history task firstly if (hTaskId.taskId != 0 && hTaskId.streamId != 0) { - streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); tqDebug("s-task:0x%x vgId:%d drop rel fill-history task:0x%x firstly", pReq->taskId, vgId, (int32_t)hTaskId.taskId); + streamMetaUnregisterTask(pMeta, hTaskId.streamId, hTaskId.taskId); } // drop the stream task now diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index bca0f06d5d..df22dff97f 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -677,8 +677,8 @@ static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamTaskId* i for (int32_t i = 0; i < num; ++i) { SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); if (pTaskId->streamId == id->streamId && pTaskId->taskId == id->taskId) { + stDebug("vgId:%d remove streamId:0x%" PRIx64 " taskId:0x%x succ", pMeta->vgId, id->streamId, id->taskId); taosArrayRemove(pMeta->pTaskList, i); - stDebug("vgId:%d remove streamId:0x%" PRIx64 " taskId:0x%x", pMeta->vgId, id->streamId, id->taskId); remove = true; break; } else { @@ -723,7 +723,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t } streamMetaWUnLock(pMeta); - stDebug("s-task:0x%x set task status:dropping and start to unregister it", taskId); + stDebug("s-task:0x%x vgId:%d set task status:dropping and start to unregister it", taskId, pMeta->vgId); while (1) { streamMetaRLock(pMeta); @@ -750,6 +750,7 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { pTask = *ppTask; + ASSERT(pTask->id.taskId == id.taskId && pTask->id.streamId == id.streamId); // it is an fill-history task, remove the related stream task's id that points to it if (pTask->info.fillHistory == 0) { From f57ff9640884e53834e993d90229d00f70c40922 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 19:56:18 +0800 Subject: [PATCH 098/103] fix(stream): do some refactor. --- source/libs/stream/src/streamMeta.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index df22dff97f..05c414078d 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -750,7 +750,8 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { pTask = *ppTask; - ASSERT(pTask->id.taskId == id.taskId && pTask->id.streamId == id.streamId); + SStreamTaskId pxId = pTask->id; + ASSERT((pxId.taskId == id.taskId) && (pxId.streamId == id.streamId)); // it is an fill-history task, remove the related stream task's id that points to it if (pTask->info.fillHistory == 0) { From a3f86abf46a5a35256736d82a8ffe4a3b2aa4b82 Mon Sep 17 00:00:00 2001 From: danielclow <106956386+danielclow@users.noreply.github.com> Date: Mon, 17 Jun 2024 20:26:18 +0800 Subject: [PATCH 099/103] docs: correct float compression description for 3.0 --- docs/en/21-tdinternal/08-compress.md | 14 ++++---------- docs/zh/21-tdinternal/08-compress.md | 10 +++------- 2 files changed, 7 insertions(+), 17 deletions(-) diff --git a/docs/en/21-tdinternal/08-compress.md b/docs/en/21-tdinternal/08-compress.md index a3b073c3a7..10a06a4c91 100644 --- a/docs/en/21-tdinternal/08-compress.md +++ b/docs/en/21-tdinternal/08-compress.md @@ -1,12 +1,8 @@ - --- - title: Configurable Column Compression description: Configurable column storage compression method --- -# Configurable Storage Compression - Since TDengine 3.3.0.0, more advanced compression feature is introduced, you can specify compression or not, the compression method and compression level for each column. ## Compression Terminology Definition @@ -32,16 +28,14 @@ In this article, it specifically refers to the level within the secondary compre - Default compression algorithm list and applicable range for each data type -| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level| +| Data Type | Optional Encoding Algorithm | Default Encoding Algorithm | Optional Compression Algorithm|Default Compression Algorithm| Default Compression Level| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| - tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| | bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| -|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium| +|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| |binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| |bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| -Note: For floating point types, if configured as tsz, its precision is determined by the global configuration of taosd. If configured as tsz, but the lossy compression flag is not configured, lz4 is used for compression by default. - ## SQL ### Create Table with Compression @@ -76,7 +70,7 @@ ALTER TABLE [db_name.]tabName MODIFY COLUMN colName [ENCODE 'ecode_type'] [COMPR - Change the compression method of the column -### View Compression Dethod +### View Compression Method ```sql DESCRIBE [dbname.]tabName diff --git a/docs/zh/21-tdinternal/08-compress.md b/docs/zh/21-tdinternal/08-compress.md index 9653a9366b..dfec63c212 100644 --- a/docs/zh/21-tdinternal/08-compress.md +++ b/docs/zh/21-tdinternal/08-compress.md @@ -3,8 +3,6 @@ title: 可配置压缩算法 description: 可配置压缩算法 --- -# 可配置存储压缩 - 从 TDengine 3.3.0.0 版本开始，TDengine 提供了更高级的压缩功能，用户可以在建表时针对每一列配置是否进行压缩、以及使用的压缩算法和压缩级别。 ## 压缩术语定义 @@ -30,16 +28,14 @@ description: 可配置压缩算法 - 各个数据类型的默认压缩算法列表和适用范围 -| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|可选压缩算法| 压缩等级默认值| +| 数据类型 | 可选编码算法 | 编码算法默认值 | 可选压缩算法|压缩算法默认值| 压缩等级默认值| | :-----------:|:----------:|:-------:|:-------:|:----------:|:----:| - tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| +| tinyint/untinyint/smallint/usmallint/int/uint | simple8b| simple8b | lz4/zlib/zstd/xz| lz4 | medium| | bigint/ubigint/timestamp | simple8b/delta-i | delta-i |lz4/zlib/zstd/xz | lz4| medium| -|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|tsz| medium| +|float/double | delta-d|delta-d |lz4/zlib/zstd/xz/tsz|lz4| medium| |binary/nchar| disabled| disabled|lz4/zlib/zstd/xz| lz4| medium| |bool| bit-packing| bit-packing| lz4/zlib/zstd/xz| lz4| medium| -注意: 针对浮点类型，如果配置为tsz, 其精度由taosd的全局配置决定，如果配置为tsz, 但是没有配置有损压缩标志, 则使用lz4进行压缩 - ## SQL 语法 ### 建表时指定压缩 From c4a27569076a7d5ba418e3c4acec88a013105205 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 23:13:54 +0800 Subject: [PATCH 100/103] fix(stream): not revise the stream id for fill-history task. --- include/libs/stream/streamState.h | 3 ++- source/dnode/snode/src/snode.c | 6 ----- source/dnode/vnode/src/sma/smaRollup.c | 2 +- source/dnode/vnode/src/tqCommon/tqCommon.c | 31 ++++++---------------- source/libs/stream/src/streamMeta.c | 24 +++++------------ source/libs/stream/src/streamState.c | 7 ++--- 6 files changed, 21 insertions(+), 52 deletions(-) diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index ae5a733ae9..1333257dfb 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -29,7 +29,8 @@ extern "C" { #include "storageapi.h" -SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); +SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId, bool specPath, + int32_t szPage, int32_t pages); void streamStateClose(SStreamState* pState, bool remove); int32_t streamStateBegin(SStreamState* pState); int32_t streamStateCommit(SStreamState* pState); diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index d322eb2977..481033508b 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -25,12 +25,6 @@ #define sndDebug(...) do { if (sndDebugFlag & DEBUG_DEBUG) { taosPrintLog("SND ", DEBUG_DEBUG, sndDebugFlag, __VA_ARGS__);}} while (0) // clang-format on -static void restoreStreamTaskId(SStreamTask *pTask, STaskId *pId) { - ASSERT(pTask->info.fillHistory); - pTask->id.taskId = pId->taskId; - pTask->id.streamId = pId->streamId; -} - int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t nextProcessVer) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__AGG && taosArrayGetSize(pTask->upstreamInfo.pList) != 0); int32_t code = streamTaskInit(pTask, pSnode->pMeta, &pSnode->msgCb, nextProcessVer); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 3cc7c6ec66..ac5463e492 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -299,7 +299,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); pStreamTask->status.pSM = streamCreateStateMachine(pStreamTask); pStreamTask->chkInfo.pActiveInfo = streamTaskCreateActiveChkptInfo(); - pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1); + pStreamState = streamStateOpen(taskInfDir, pStreamTask, pStreamTask->id.streamId, pStreamTask->id.taskId, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/tqCommon/tqCommon.c b/source/dnode/vnode/src/tqCommon/tqCommon.c index d9bc8b74d2..963503c135 100644 --- a/source/dnode/vnode/src/tqCommon/tqCommon.c +++ b/source/dnode/vnode/src/tqCommon/tqCommon.c @@ -30,37 +30,26 @@ typedef struct SMStreamCheckpointReadyRspMsg { static int32_t doProcessDummyRspMsg(SStreamMeta* pMeta, SRpcMsg* pMsg); -static STaskId replaceStreamTaskId(SStreamTask* pTask) { - ASSERT(pTask->info.fillHistory); - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - - pTask->id.streamId = pTask->streamTaskId.streamId; - pTask->id.taskId = pTask->streamTaskId.taskId; - - return id; -} - -static void restoreStreamTaskId(SStreamTask* pTask, STaskId* pId) { - ASSERT(pTask->info.fillHistory); - pTask->id.taskId = pId->taskId; - pTask->id.streamId = pId->streamId; -} - int32_t tqExpandStreamTask(SStreamTask* pTask) { SStreamMeta* pMeta = pTask->pMeta; int32_t vgId = pMeta->vgId; - STaskId taskId = {0}; int64_t st = taosGetTimestampMs(); + int64_t streamId = 0; + int32_t taskId = 0; tqDebug("s-task:%s vgId:%d start to expand stream task", pTask->id.idStr, vgId); if (pTask->info.fillHistory) { - taskId = replaceStreamTaskId(pTask); + streamId = pTask->streamTaskId.streamId; + taskId = pTask->streamTaskId.taskId; + } else { + streamId = pTask->id.streamId; + taskId = pTask->id.taskId; } // sink task does not need the pState if (pTask->info.taskLevel != TASK_LEVEL__SINK) { - pTask->pState = streamStateOpen(pMeta->path, pTask, false, -1, -1); + pTask->pState = streamStateOpen(pMeta->path, pTask, false, streamId, taskId, -1, -1); if (pTask->pState == NULL) { tqError("s-task:%s (vgId:%d) failed to open state for task, expand task failed", pTask->id.idStr, vgId); return -1; @@ -69,10 +58,6 @@ int32_t tqExpandStreamTask(SStreamTask* pTask) { } } - if (pTask->info.fillHistory) { - restoreStreamTaskId(pTask, &taskId); - } - SReadHandle handle = { .checkpointId = pTask->chkInfo.checkpointId, .pStateBackend = pTask->pState, diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 05c414078d..e8800c3370 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -608,6 +608,7 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa } taosArrayPush(pMeta->pTaskList, &pTask->id); + taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES); if (streamMetaSaveTask(pMeta, pTask) < 0) { return -1; @@ -617,7 +618,6 @@ int32_t streamMetaRegisterTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTa return -1; } - taosHashPut(pMeta->pTasksMap, &id, sizeof(id), &pTask, POINTER_BYTES); if (pTask->info.fillHistory == 0) { atomic_add_fetch_32(&pMeta->numOfStreamTasks, 1); } @@ -672,21 +672,16 @@ void streamMetaReleaseTask(SStreamMeta* UNUSED_PARAM(pMeta), SStreamTask* pTask) } } -static void doRemoveIdFromList(SStreamMeta* pMeta, int32_t num, SStreamTaskId* id) { +static void doRemoveIdFromList(SArray* pTaskList, int32_t num, SStreamTaskId* id) { bool remove = false; for (int32_t i = 0; i < num; ++i) { - SStreamTaskId* pTaskId = taosArrayGet(pMeta->pTaskList, i); + SStreamTaskId* pTaskId = taosArrayGet(pTaskList, i); if (pTaskId->streamId == id->streamId && pTaskId->taskId == id->taskId) { - stDebug("vgId:%d remove streamId:0x%" PRIx64 " taskId:0x%x succ", pMeta->vgId, id->streamId, id->taskId); - taosArrayRemove(pMeta->pTaskList, i); + taosArrayRemove(pTaskList, i); remove = true; break; - } else { - stDebug("vgId:%d remove streamId:0x%" PRIx64 " taskId:0x%x, entry:0x%" PRIx64 "-0x%x", pMeta->vgId, id->streamId, - id->taskId, pTaskId->streamId, pTaskId->taskId); } } - ASSERT(remove); } @@ -750,26 +745,19 @@ int32_t streamMetaUnregisterTask(SStreamMeta* pMeta, int64_t streamId, int32_t t ppTask = (SStreamTask**)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); if (ppTask) { pTask = *ppTask; - SStreamTaskId pxId = pTask->id; - ASSERT((pxId.taskId == id.taskId) && (pxId.streamId == id.streamId)); - // it is an fill-history task, remove the related stream task's id that points to it if (pTask->info.fillHistory == 0) { atomic_sub_fetch_32(&pMeta->numOfStreamTasks, 1); } - ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); - taosHashRemove(pMeta->pTasksMap, &id, sizeof(id)); - doRemoveIdFromList(pMeta, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); - - ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); + doRemoveIdFromList(pMeta->pTaskList, (int32_t)taosArrayGetSize(pMeta->pTaskList), &pTask->id); streamMetaRemoveTask(pMeta, &id); + ASSERT(taosHashGetSize(pMeta->pTasksMap) == taosArrayGetSize(pMeta->pTaskList)); streamMetaWUnLock(pMeta); ASSERT(pTask->status.timerActive == 0); - if (pTask->info.delaySchedParam != 0 && pTask->info.fillHistory == 0) { stDebug("s-task:%s stop schedTimer, and (before) desc ref:%d", pTask->id.idStr, pTask->refCnt); taosTmrStop(pTask->schedInfo.pDelayTimer); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 47324bd8c9..8919bd48ac 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -98,7 +98,8 @@ int stateKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { return winKeyCmprImpl(&pWin1->key, &pWin2->key); } -SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages) { +SStreamState* streamStateOpen(const char* path, void* pTask, int64_t streamId, int32_t taskId, bool specPath, + int32_t szPage, int32_t pages) { SStreamState* pState = taosMemoryCalloc(1, sizeof(SStreamState)); stDebug("open stream state %p, %s", pState, path); if (pState == NULL) { @@ -114,8 +115,8 @@ SStreamState* streamStateOpen(const char* path, void* pTask, bool specPath, int3 } SStreamTask* pStreamTask = pTask; - pState->taskId = pStreamTask->id.taskId; - pState->streamId = pStreamTask->id.streamId; + pState->streamId = streamId; + pState->taskId = taskId; sprintf(pState->pTdbState->idstr, "0x%" PRIx64 "-0x%x", pState->streamId, pState->taskId); streamTaskSetDb(pStreamTask->pMeta, pTask, pState->pTdbState->idstr); From 985d6195699c581f401f990ae980c088b8df6140 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 23:14:33 +0800 Subject: [PATCH 101/103] fix(stream): fix syntax error. --- include/libs/executor/storageapi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index 330ba31c65..8c67f77adb 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -410,7 +410,7 @@ typedef struct SStateStore { void (*streamFileStateClear)(struct SStreamFileState* pFileState); bool (*needClearDiskBuff)(struct SStreamFileState* pFileState); - SStreamState* (*streamStateOpen)(const char* path, void* pTask, bool specPath, int32_t szPage, int32_t pages); + SStreamState* (*streamStateOpen)(const char* path, void* pTask, int64_t streamId, int32_t taskId, bool specPath, int32_t szPage, int32_t pages); void (*streamStateClose)(SStreamState* pState, bool remove); int32_t (*streamStateBegin)(SStreamState* pState); int32_t (*streamStateCommit)(SStreamState* pState); From bca3cf4e9ee6e698ba8bd99efd15a52daef8882c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 17 Jun 2024 23:51:25 +0800 Subject: [PATCH 102/103] fix(stream): fix error in unit test. --- source/libs/stream/test/backendTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/test/backendTest.cpp b/source/libs/stream/test/backendTest.cpp index dc506cfbc9..e6a508f6f7 100644 --- a/source/libs/stream/test/backendTest.cpp +++ b/source/libs/stream/test/backendTest.cpp @@ -46,7 +46,7 @@ SStreamState *stateCreate(const char *path) { SStreamMeta *pMeta = streamMetaOpen((path), NULL, NULL, NULL, 0, 0, NULL); pTask->pMeta = pMeta; - SStreamState *p = streamStateOpen((char *)path, pTask, true, 32, 32 * 1024); + SStreamState *p = streamStateOpen((char *)path, pTask, 0, 0, true, 32, 32 * 1024); ASSERT(p != NULL); return p; } From 92713ec8c5f847187da2117a347e52ff47b8e47b Mon Sep 17 00:00:00 2001 From: kailixu Date: Tue, 18 Jun 2024 08:44:57 +0800 Subject: [PATCH 103/103] fix: version output of taos/taosd --- source/dnode/mgmt/exe/dmMain.c | 2 +- tools/shell/src/shellArguments.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 786df6b907..65f695cb8b 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -261,7 +261,7 @@ static void dmPrintVersion() { printf("%s\ntaosd version: %s compatible_version: %s\n", TD_PRODUCT_NAME, version, compatible_version); printf("git: %s\n", gitinfo); #ifdef TD_ENTERPRISE - printf("git: %s\n", gitinfoOfInternal); + printf("gitOfInternal: %s\n", gitinfoOfInternal); #endif printf("build: %s\n", buildinfo); } diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index f48b19a287..8368837cc4 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -435,8 +435,8 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shell.info.promptSize = strlen(shell.info.promptHeader); #ifdef TD_ENTERPRISE snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), - "%s\ntaos version: %s compatible_version: %s\ngit: %s\ngit: %s\nbuild: %s", TD_PRODUCT_NAME, version, - compatible_version, gitinfo, gitinfoOfInternal, buildinfo); + "%s\ntaos version: %s compatible_version: %s\ngit: %s\ngitOfInternal: %s\nbuild: %s", TD_PRODUCT_NAME, + version, compatible_version, gitinfo, gitinfoOfInternal, buildinfo); #else snprintf(shell.info.programVersion, sizeof(shell.info.programVersion), "%s\ntaos version: %s compatible_version: %s\ngit: %s\nbuild: %s", TD_PRODUCT_NAME, version,