From fc84b5c95572c2b4ad28c5b879d033c9179b5ba1 Mon Sep 17 00:00:00 2001
From: Shengliang <slguan@taosdata.com>
Date: Wed, 11 May 2022 11:37:13 +0800
Subject: [PATCH 01/35] fix: alter stb

---
 source/dnode/mnode/impl/src/mndStb.c | 13 +++++++++----
 tests/script/tsim/stable/alter1.sim  |  1 +
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c
index ca0ae111a3..12e89277f4 100644
--- a/source/dnode/mnode/impl/src/mndStb.c
+++ b/source/dnode/mnode/impl/src/mndStb.c
@@ -318,6 +318,7 @@ static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew) {
   pOld->updateTime = pNew->updateTime;
   pOld->version = pNew->version;
   pOld->nextColId = pNew->nextColId;
+  pOld->ttl = pNew->ttl;
   pOld->numOfColumns = pNew->numOfColumns;
   pOld->numOfTags = pNew->numOfTags;
   memcpy(pOld->pColumns, pNew->pColumns, pOld->numOfColumns * sizeof(SSchema));
@@ -832,7 +833,7 @@ static int32_t mndProcessVCreateStbRsp(SNodeMsg *pRsp) {
 }
 
 static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) {
-  if (pAlter->commentLen != 0) return 0;
+  if (pAlter->commentLen != 0 || pAlter->ttl != 0) return 0;
 
   if (pAlter->numOfFields < 1 || pAlter->numOfFields != (int32_t)taosArrayGetSize(pAlter->pFields)) {
     terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
@@ -883,7 +884,8 @@ static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) {
   return 0;
 }
 
-static int32_t mndUpdateStbComment(const SStbObj *pOld, SStbObj *pNew, char *pComment, int32_t commentLen) {
+static int32_t mndUpdateStbCommentAndTTL(const SStbObj *pOld, SStbObj *pNew, char *pComment, int32_t commentLen,
+                                         int32_t ttl) {
   if (commentLen > 0) {
     pNew->commentLen = commentLen;
     pNew->comment = taosMemoryCalloc(1, commentLen);
@@ -893,6 +895,9 @@ static int32_t mndUpdateStbComment(const SStbObj *pOld, SStbObj *pNew, char *pCo
     }
     memcpy(pNew->comment, pComment, commentLen);
   }
+  if (ttl >= 0) {
+    pNew->ttl = ttl;
+  }
 
   if (mndAllocStbSchemas(pOld, pNew) != 0) {
     return -1;
@@ -1232,7 +1237,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SNodeMsg *pReq, const SMAlterStbReq *
       code = mndAlterStbColumnBytes(pOld, &stbObj, pField0);
       break;
     case TSDB_ALTER_TABLE_UPDATE_OPTIONS:
-      code = mndUpdateStbComment(pOld, &stbObj, pAlter->comment, pAlter->commentLen);
+      code = mndUpdateStbCommentAndTTL(pOld, &stbObj, pAlter->comment, pAlter->commentLen, pAlter->ttl);
       break;
     default:
       terrno = TSDB_CODE_OPS_NOT_SUPPORT;
@@ -1723,7 +1728,7 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pStb->updateTime, false);  // number of tables
 
-    char *p = taosMemoryMalloc(pStb->commentLen + VARSTR_HEADER_SIZE);  // check malloc failures
+    char *p = taosMemoryCalloc(1, pStb->commentLen + 1 + VARSTR_HEADER_SIZE);  // check malloc failures
     if (p != NULL) {
       if (pStb->commentLen != 0) {
         STR_TO_VARSTR(p, pStb->comment);
diff --git a/tests/script/tsim/stable/alter1.sim b/tests/script/tsim/stable/alter1.sim
index 5cee10756c..1205f50f6e 100644
--- a/tests/script/tsim/stable/alter1.sim
+++ b/tests/script/tsim/stable/alter1.sim
@@ -159,6 +159,7 @@ sql alter table db.stb rename tag t1 tx
 
 print ========== alter common
 sql alter table db.stb comment 'abcde' ;
+sql alter table db.stb ttl 10 ;
 
 sql show db.stables;
 if $data[0][6] != abcde then

From a24318d4a62c890a86bbcfebd52399c786348ad1 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Wed, 11 May 2022 21:56:16 +0800
Subject: [PATCH 02/35] Merge branch '3.0' of github.com:taosdata/TDengine into
 test/chr/TD-14699

---
 tests/system-test/1-insert/insertWithMoreVgroup.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py
index d8050c53c5..c9acc93c59 100644
--- a/tests/system-test/1-insert/insertWithMoreVgroup.py
+++ b/tests/system-test/1-insert/insertWithMoreVgroup.py
@@ -346,8 +346,10 @@ class TDTestCase:
         return
 
     def test_case3(self):
+        self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 8, 1*10000)
+        self.taosBenchCreate("test209","no","db1", "stb1", 1, 8, 1*10000)
+
         # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000)
-        self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*1000)
 
         # self.taosBenchCreate("db1", "stb1", 4, 5, 100*10000)
         # self.taosBenchCreate("db1", "stb1", 1, 5, 100*10000)

From 4765cb9e9a1254366d4bb9afe06703d45475039b Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 12 May 2022 09:42:53 +0800
Subject: [PATCH 03/35] test:modify testcase that using taosBenchmark to test
 multi-process table building and data inserting

---
 tests/system-test/1-insert/insertWithMoreVgroup.py | 2 +-
 tests/system-test/1-insert/manyVgroups.json        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py
index c9acc93c59..d3da4f2c59 100644
--- a/tests/system-test/1-insert/insertWithMoreVgroup.py
+++ b/tests/system-test/1-insert/insertWithMoreVgroup.py
@@ -347,7 +347,7 @@ class TDTestCase:
 
     def test_case3(self):
         self.taosBenchCreate("127.0.0.1","no","db1", "stb1", 1, 8, 1*10000)
-        self.taosBenchCreate("test209","no","db1", "stb1", 1, 8, 1*10000)
+        # self.taosBenchCreate("test209","no","db2", "stb2", 1, 8, 1*10000)
 
         # self.taosBenchCreate("chenhaoran02","no","db1", "stb1", 1, 8, 1*10000)
 
diff --git a/tests/system-test/1-insert/manyVgroups.json b/tests/system-test/1-insert/manyVgroups.json
index 5487dff708..e6719aedc9 100644
--- a/tests/system-test/1-insert/manyVgroups.json
+++ b/tests/system-test/1-insert/manyVgroups.json
@@ -29,8 +29,8 @@
                     "batch_create_tbl_num": 50000,
                     "data_source": "rand",
                     "insert_mode": "taosc",
-                    "insert_rows": 0,
-                    "interlace_rows": 0,
+                    "insert_rows": 10,
+                    "interlace_rows": 100000,
                     "insert_interval": 0,
                     "max_sql_len": 10000000,
                     "disorder_ratio": 0,

From d37d4e5b5eda68ae568c919c57a38dba7c4a2180 Mon Sep 17 00:00:00 2001
From: "slzhou@taodata.com" <shenglian zhou>
Date: Thu, 12 May 2022 09:57:43 +0800
Subject: [PATCH 04/35] error processing of executor when calling aggregate
 function

---
 source/libs/executor/inc/executorimpl.h       |  4 +-
 source/libs/executor/src/executorimpl.c       | 39 +++++++++++++------
 source/libs/executor/src/groupoperator.c      |  8 ++--
 source/libs/executor/src/timewindowoperator.c | 26 ++++++-------
 4 files changed, 47 insertions(+), 30 deletions(-)

diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h
index 93e81aa70e..4881f23134 100644
--- a/source/libs/executor/inc/executorimpl.h
+++ b/source/libs/executor/inc/executorimpl.h
@@ -616,10 +616,10 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t
 int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols,
                     SSDataBlock* pResultBlock, size_t keyBufSize, const char* pkey);
 void    initResultSizeInfo(SOperatorInfo* pOperator, int32_t numOfRows);
-void doBuildResultDatablock(SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf);
+void doBuildResultDatablock(SExecTaskInfo *taskInfo, SOptrBasicInfo *pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf);
 
 void    finalizeMultiTupleQueryResult(int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset);
-void    doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,
+void    doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,
                          int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order);
 int32_t setGroupResultOutputBuf(SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t type, int16_t bytes,
                                 int32_t groupId, SDiskbasedBuf* pBuf, SExecTaskInfo* pTaskInfo, SAggSupporter* pAggSup);
diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 33f0c440ec..9c285856d3 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -155,7 +155,7 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn,
 
 void operatorDummyCloseFn(void* param, int32_t numOfCols) {}
 
-static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf,
+static int32_t doCopyToSDataBlock(SExecTaskInfo *taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf,
                                   SGroupResInfo* pGroupResInfo, int32_t orderType, int32_t* rowCellOffset,
                                   SqlFunctionCtx* pCtx);
 
@@ -579,7 +579,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow
   colDataAppendInt64(pColData, 4, &pQueryWindow->ekey);
 }
 
-void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,
+void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData* pTimeWindowData, int32_t offset,
                       int32_t forwardStep, TSKEY* tsCol, int32_t numOfTotal, int32_t numOfOutput, int32_t order) {
   for (int32_t k = 0; k < numOfOutput; ++k) {
     pCtx[k].startTs = pWin->skey;
@@ -618,9 +618,14 @@ void doApplyFunctions(SqlFunctionCtx* pCtx, STimeWindow* pWin, SColumnInfoData*
       pEntryInfo->numOfRes = 1;
       continue;
     }
-
+    int32_t code = TSDB_CODE_SUCCESS;
     if (functionNeedToExecute(&pCtx[k]) && pCtx[k].fpSet.process != NULL) {
-      pCtx[k].fpSet.process(&pCtx[k]);
+      code = pCtx[k].fpSet.process(&pCtx[k]);
+      if (code != TSDB_CODE_SUCCESS) {
+        qError("%s apply functions error, code: %s", GET_TASKID(taskInfo), tstrerror(code));
+        taskInfo->code = code;
+        longjmp(taskInfo->env, code);
+      }
     }
 
     // restore it
@@ -806,7 +811,13 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SqlFunction
       // this can be set during create the struct
       // todo add a dummy funtion to avoid process check
       if (pCtx[k].fpSet.process != NULL) {
-        pCtx[k].fpSet.process(&pCtx[k]);
+        int32_t code = pCtx[k].fpSet.process(&pCtx[k]);
+        if (code != TSDB_CODE_SUCCESS) {
+          qError("%s call aggregate function error happens, code : %s",
+                 GET_TASKID(pOperator->pTaskInfo), tstrerror(code));
+          pOperator->pTaskInfo->code = code;
+          longjmp(pOperator->pTaskInfo->env, code);
+        }
       }
     }
   }
@@ -2176,7 +2187,7 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p
  * @param pQInfo
  * @param result
  */
-int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
+int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo,
                            int32_t orderType, int32_t* rowCellOffset, SqlFunctionCtx* pCtx) {
   int32_t numOfRows = getNumOfTotalRes(pGroupResInfo);
   int32_t numOfResult = pBlock->info.rows;  // there are already exists result rows
@@ -2215,8 +2226,14 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
       int32_t slotId = pExprInfo[j].base.resSchema.slotId;
 
       pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset);
-      if (pCtx[j].fpSet.process) {
-        pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
+      if (pCtx[j].fpSet.finalize) {
+        int32_t code = TSDB_CODE_SUCCESS;
+        code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
+        if (code != TSDB_CODE_SUCCESS) {
+          qError("%s build result data block error, code %s", GET_TASKID(taskInfo), tstrerror(code));
+          taskInfo->code = code;
+          longjmp(taskInfo->env, code);
+        }
       } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) {
         // do nothing, todo refactor
       } else {
@@ -2243,7 +2260,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
   return 0;
 }
 
-void doBuildResultDatablock(SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo,
+void doBuildResultDatablock(SExecTaskInfo *taskInfo, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SExprInfo* pExprInfo,
                             SDiskbasedBuf* pBuf) {
   assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup);
 
@@ -2257,7 +2274,7 @@ void doBuildResultDatablock(SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo
   }
 
   int32_t orderType = TSDB_ORDER_ASC;
-  doCopyToSDataBlock(pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx);
+  doCopyToSDataBlock(taskInfo, pBlock, pExprInfo, pBuf, pGroupResInfo, orderType, rowCellOffset, pCtx);
 
   // add condition (pBlock->info.rows >= 1) just to runtime happy
   blockDataUpdateTsWindow(pBlock);
@@ -3749,7 +3766,7 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) {
   }
 
   blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf);
+  doBuildResultDatablock(pTaskInfo, pInfo, &pAggInfo->groupResInfo, pOperator->pExpr, pAggInfo->aggSup.pResultBuf);
   if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) {
     doSetOperatorCompleted(pOperator);
   }
diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c
index e3a507bf7c..5d22c13ec6 100644
--- a/source/libs/executor/src/groupoperator.c
+++ b/source/libs/executor/src/groupoperator.c
@@ -234,7 +234,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     }
 
     int32_t rowIndex = j - num;
-    doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC);
+    doApplyFunctions(pTaskInfo, pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC);
 
     // assign the group keys or user input constant values if required
     doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex);
@@ -252,7 +252,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
     }
 
     int32_t rowIndex = pBlock->info.rows - num;
-    doApplyFunctions(pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC);
+    doApplyFunctions(pTaskInfo, pCtx, &w, NULL, rowIndex, num, NULL, pBlock->info.rows, pOperator->numOfExprs, TSDB_ORDER_ASC);
     doAssignGroupKeys(pCtx, pOperator->numOfExprs, pBlock->info.rows, rowIndex);
   }
 }
@@ -268,7 +268,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
   SSDataBlock* pRes = pInfo->binfo.pRes;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+    doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
     if (pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       pOperator->status = OP_EXEC_DONE;
     }
@@ -317,7 +317,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) {
   initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false);
 
   while(1) {
-    doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+    doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
     doFilter(pInfo->pCondition, pRes);
 
     bool hasRemain = hasRemainDataInCurrentGroup(&pInfo->groupResInfo);
diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c
index 738f4821bd..0f3b1bda20 100644
--- a/source/libs/executor/src/timewindowoperator.c
+++ b/source/libs/executor/src/timewindowoperator.c
@@ -703,7 +703,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
                               pInfo->order, false);
 
   updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
-  doApplyFunctions(pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols,
+  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &win, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols,
                    pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
 
   STimeWindow nextWin = win;
@@ -740,7 +740,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
                                 pInfo->order, false);
 
     updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
-    doApplyFunctions(pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols,
+    doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardStep, tsCols,
                      pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
   }
 
@@ -855,7 +855,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
       }
 
       updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
-      doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
+      doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                        pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
 
       // here we start a new session window
@@ -874,7 +874,7 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI
   }
 
   updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
-  doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
+  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                    pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
 }
 
@@ -888,7 +888,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
   SOptrBasicInfo*           pBInfo = &pInfo->binfo;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+    doBuildResultDatablock(pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
       return NULL;
@@ -921,7 +921,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) {
 
   initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+  doBuildResultDatablock(pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
   if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
     doSetOperatorCompleted(pOperator);
   }
@@ -948,7 +948,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator) {
     }
 
     blockDataEnsureCapacity(pBlock, pOperator->resultInfo.capacity);
-    doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+    doBuildResultDatablock(pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
 
     if (pBlock->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
@@ -998,7 +998,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
   }
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+    doBuildResultDatablock(pOperator->pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
     if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       pOperator->status = OP_EXEC_DONE;
     }
@@ -1035,7 +1035,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
 
   initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
   blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(&pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+  doBuildResultDatablock(pOperator->pTaskInfo, &pInfo->binfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
 
   // TODO: remove for stream
   /*ASSERT(pInfo->binfo.pRes->info.rows > 0);*/
@@ -1233,7 +1233,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
 
       // pInfo->numOfRows data belong to the current session window
       updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
-      doApplyFunctions(pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
+      doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &window, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                        pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
 
       // here we start a new session window
@@ -1252,7 +1252,7 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator
   }
 
   updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
-  doApplyFunctions(pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
+  doApplyFunctions(pTaskInfo, pInfo->binfo.pCtx, &pRowSup->win, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
                    pRowSup->numOfRows, NULL, pBlock->info.rows, numOfOutput, TSDB_ORDER_ASC);
 }
 
@@ -1265,7 +1265,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
   SOptrBasicInfo*          pBInfo = &pInfo->binfo;
 
   if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+    doBuildResultDatablock(pOperator->pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
     if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
       doSetOperatorCompleted(pOperator);
       return NULL;
@@ -1298,7 +1298,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
 
   initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
   blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
-  doBuildResultDatablock(pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
+  doBuildResultDatablock(pOperator->pTaskInfo, pBInfo, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf);
   if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) {
     doSetOperatorCompleted(pOperator);
   }

From 85ee4f7df60f90d7154817d253368e22ee88b533 Mon Sep 17 00:00:00 2001
From: "slzhou@taodata.com" <shenglian zhou>
Date: Thu, 12 May 2022 10:43:54 +0800
Subject: [PATCH 05/35] fix finalize function call return value >=0 not error,
 <0 error

---
 source/libs/executor/src/executorimpl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c
index 9c285856d3..a5bc1fdf58 100644
--- a/source/libs/executor/src/executorimpl.c
+++ b/source/libs/executor/src/executorimpl.c
@@ -2229,7 +2229,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* taskInfo, SSDataBlock* pBlock, SExprIn
       if (pCtx[j].fpSet.finalize) {
         int32_t code = TSDB_CODE_SUCCESS;
         code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
-        if (code != TSDB_CODE_SUCCESS) {
+        if (TAOS_FAILED(code)) {
           qError("%s build result data block error, code %s", GET_TASKID(taskInfo), tstrerror(code));
           taskInfo->code = code;
           longjmp(taskInfo->env, code);

From 7ab2e76c3535b12f7f7e75b83f3fe8e4c2ed45dd Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Thu, 12 May 2022 11:31:56 +0800
Subject: [PATCH 06/35] feat(tmq): support get tb name

---
 example/src/tmq.c                      |  9 +++++++--
 include/client/taos.h                  |  5 +----
 include/common/tmsg.h                  | 10 ++++++++++
 source/client/src/tmq.c                | 13 +++++++++++++
 source/dnode/mnode/impl/src/mndTopic.c |  4 ++--
 source/dnode/vnode/src/inc/meta.h      | 10 +++++-----
 source/dnode/vnode/src/tq/tq.c         | 26 ++++++++++++++++++++++++++
 7 files changed, 64 insertions(+), 13 deletions(-)

diff --git a/example/src/tmq.c b/example/src/tmq.c
index 976d658fa6..2ee91c254c 100644
--- a/example/src/tmq.c
+++ b/example/src/tmq.c
@@ -32,6 +32,11 @@ static void msg_process(TAOS_RES* msg) {
     int32_t     numOfFields = taos_field_count(msg);
     taos_print_row(buf, row, fields, numOfFields);
     printf("%s\n", buf);
+
+    const char* tbName = tmq_get_table_name(msg);
+    if (tbName) {
+      printf("from tb: %s\n", tbName);
+    }
   }
 }
 
@@ -101,8 +106,8 @@ int32_t create_topic() {
   }
   taos_free_result(pRes);
 
-  pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");
-  /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from ct1");*/
+  /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
+  pRes = taos_query(pConn, "create topic topic_ctb_column with table as select ts, c1, c2, c3 from st1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
diff --git a/include/client/taos.h b/include/client/taos.h
index 26d4d18234..486d5f5fef 100644
--- a/include/client/taos.h
+++ b/include/client/taos.h
@@ -257,10 +257,7 @@ DLL_EXPORT void           tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_co
 
 DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res);
 DLL_EXPORT int32_t     tmq_get_vgroup_id(TAOS_RES *res);
-// TODO
-#if 0
-DLL_EXPORT char   *tmq_get_table_name(TAOS_RES *res);
-#endif
+DLL_EXPORT const char *tmq_get_table_name(TAOS_RES *res);
 
 #if 0
 DLL_EXPORT int64_t     tmq_get_request_offset(tmq_message_t *message);
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index b50367af03..d34892a278 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2480,6 +2480,10 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp
         SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRsp->blockSchema, i);
         tlen += taosEncodeSSchemaWrapper(buf, pSW);
       }
+      if (pRsp->withTbName) {
+        char* tbName = (char*)taosArrayGetP(pRsp->blockTbName, i);
+        tlen += taosEncodeString(buf, tbName);
+      }
     }
   }
   return tlen;
@@ -2492,6 +2496,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
   buf = taosDecodeFixedI32(buf, &pRsp->blockNum);
   pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void*));
   pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(void*));
+  pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*));
   pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void*));
   if (pRsp->blockNum != 0) {
     buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
@@ -2510,6 +2515,11 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
         buf = taosDecodeSSchemaWrapper(buf, pSW);
         taosArrayPush(pRsp->blockSchema, &pSW);
       }
+      if (pRsp->withTbName) {
+        char* name = NULL;
+        buf = taosDecodeString(buf, &name);
+        taosArrayPush(pRsp->blockTbName, &name);
+      }
     }
   }
   return (void*)buf;
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index 0ce689f19c..b42f072e54 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -1346,3 +1346,16 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) {
     return -1;
   }
 }
+
+const char* tmq_get_table_name(TAOS_RES* res) {
+  if (TD_RES_TMQ(res)) {
+    SMqRspObj* pRspObj = (SMqRspObj*)res;
+    if (!pRspObj->rsp.withTbName || pRspObj->rsp.blockTbName == NULL || pRspObj->resIter < 0 ||
+        pRspObj->resIter >= pRspObj->rsp.blockNum) {
+      return NULL;
+    }
+    const char* name = taosArrayGetP(pRspObj->rsp.blockTbName, pRspObj->resIter);
+    return name;
+  }
+  return NULL;
+}
diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c
index 41d4d5f406..00379ecda1 100644
--- a/source/dnode/mnode/impl/src/mndTopic.c
+++ b/source/dnode/mnode/impl/src/mndTopic.c
@@ -297,8 +297,8 @@ static int32_t mndCreateTopic(SMnode *pMnode, SNodeMsg *pReq, SCMCreateTopicReq
     topicObj.ast = strdup(pCreate->ast);
     topicObj.astLen = strlen(pCreate->ast) + 1;
     topicObj.subType = TOPIC_SUB_TYPE__TABLE;
-    topicObj.withTbName = 0;
-    topicObj.withSchema = 0;
+    topicObj.withTbName = pCreate->withTbName;
+    topicObj.withSchema = pCreate->withSchema;
 
     SNode *pAst = NULL;
     if (nodesStringToNode(pCreate->ast, &pAst) != 0) {
diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h
index 96feee3d7d..6dd9ac0fed 100644
--- a/source/dnode/vnode/src/inc/meta.h
+++ b/source/dnode/vnode/src/inc/meta.h
@@ -111,10 +111,10 @@ int64_t      metaSmaCursorNext(SMSmaCursor* pSmaCur);
 // SMetaDB
 int  metaOpenDB(SMeta* pMeta);
 void metaCloseDB(SMeta* pMeta);
-int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
-int  metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
-int  metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
-int  metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
+// int  metaSaveTableToDB(SMeta* pMeta, STbCfg* pTbCfg, STbDdlH* pHandle);
+int metaRemoveTableFromDb(SMeta* pMeta, tb_uid_t uid);
+int metaSaveSmaToDB(SMeta* pMeta, STSma* pTbCfg);
+int metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
 #endif
 
 #endif
@@ -123,4 +123,4 @@ int  metaRemoveSmaFromDb(SMeta* pMeta, int64_t indexUid);
 }
 #endif
 
-#endif /*_TD_VNODE_META_H_*/
\ No newline at end of file
+#endif /*_TD_VNODE_META_H_*/
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 4b9551b250..29fabc0f9f 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -427,9 +427,12 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqDataBlkRsp rsp = {0};
   rsp.reqOffset = pReq->currentOffset;
   rsp.withSchema = pExec->withSchema;
+  rsp.withTbName = pExec->withTbName;
+
   rsp.blockData = taosArrayInit(0, sizeof(void*));
   rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
   rsp.blockSchema = taosArrayInit(0, sizeof(void*));
+  rsp.blockTbName = taosArrayInit(0, sizeof(void*));
 
   while (1) {
     consumerEpoch = atomic_load_32(&pExec->epoch);
@@ -535,6 +538,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
             taosArrayPush(rsp.blockSchema, &pSW);
           }
 
+          if (pExec->withTbName) {
+            SMetaReader mr = {0};
+            metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
+            int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
+            if (metaGetTableEntryByUid(&mr, uid) < 0) {
+              ASSERT(0);
+            }
+            char* tbName = strdup(mr.me.name);
+            taosArrayPush(rsp.blockTbName, &tbName);
+            metaReaderClear(&mr);
+          }
+
           rsp.blockNum++;
         }
         // db subscribe
@@ -563,6 +578,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           ASSERT(actualLen <= dataStrLen);
           taosArrayPush(rsp.blockDataLen, &actualLen);
           taosArrayPush(rsp.blockData, &buf);
+          if (pExec->withTbName) {
+            SMetaReader mr = {0};
+            metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
+            if (metaGetTableEntryByUid(&mr, block.info.uid) < 0) {
+              ASSERT(0);
+            }
+            char* tbName = strdup(mr.me.name);
+            taosArrayPush(rsp.blockTbName, &tbName);
+            metaReaderClear(&mr);
+          }
 
           SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper);
           taosArrayPush(rsp.blockSchema, &pSW);
@@ -614,6 +639,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   taosArrayDestroy(rsp.blockData);
   taosArrayDestroy(rsp.blockDataLen);
   taosArrayDestroyP(rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper);
+  taosArrayDestroyP(rsp.blockTbName, (FDelete)taosMemoryFree);
 
   return 0;
 }

From 72f9a1c4a4cff5ce41bf43316a2f47bb206cc494 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 03:34:38 +0000
Subject: [PATCH 07/35] fix

---
 source/dnode/vnode/src/meta/metaQuery.c | 10 +++++++++-
 source/libs/tdb/src/db/tdbPCache.c      |  2 +-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 8d2a4ebcf3..7d0a87d79d 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -158,7 +158,9 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
   skmDbKey.sver = sver;
   pKey = &skmDbKey;
   kLen = sizeof(skmDbKey);
+  metaRLock(pMeta);
   ret = tdbDbGet(pMeta->pSkmDb, pKey, kLen, &pVal, &vLen);
+  metaULock(pMeta);
   if (ret < 0) {
     return NULL;
   }
@@ -181,6 +183,7 @@ SSchemaWrapper *metaGetTableSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver, boo
 }
 
 struct SMCtbCursor {
+  SMeta   *pMeta;
   TDBC    *pCur;
   tb_uid_t suid;
   void    *pKey;
@@ -200,9 +203,13 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
     return NULL;
   }
 
+  pCtbCur->pMeta = pMeta;
   pCtbCur->suid = uid;
+  metaRLock(pMeta);
+
   ret = tdbDbcOpen(pMeta->pCtbIdx, &pCtbCur->pCur, NULL);
   if (ret < 0) {
+    metaULock(pMeta);
     taosMemoryFree(pCtbCur);
     return NULL;
   }
@@ -220,6 +227,7 @@ SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) {
 
 void metaCloseCtbCurosr(SMCtbCursor *pCtbCur) {
   if (pCtbCur) {
+    if (pCtbCur->pMeta) metaULock(pCtbCur->pMeta);
     if (pCtbCur->pCur) {
       tdbDbcClose(pCtbCur->pCur);
 
@@ -269,7 +277,7 @@ STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) {
 
   pSW = metaGetTableSchema(pMeta, quid, sver, 0);
   if (!pSW) return NULL;
-  
+
   tdInitTSchemaBuilder(&sb, 0);
   for (int i = 0; i < pSW->nCols; i++) {
     pSchema = pSW->pSchema + i;
diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c
index aa05687426..8574e071f2 100644
--- a/source/libs/tdb/src/db/tdbPCache.c
+++ b/source/libs/tdb/src/db/tdbPCache.c
@@ -242,7 +242,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) {
   int     h;
 
   h = tdbPCachePageHash(&(pPage->pgid));
-  for (ppPage = &(pCache->pgHash[h % pCache->nHash]); *ppPage != pPage; ppPage = &((*ppPage)->pHashNext))
+  for (ppPage = &(pCache->pgHash[h % pCache->nHash]); (*ppPage) && *ppPage != pPage; ppPage = &((*ppPage)->pHashNext))
     ;
   ASSERT(*ppPage == pPage);
   *ppPage = pPage->pHashNext;

From 51094f3434549cdebf0fbfec3a1c2e9c0a7a814f Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 12 May 2022 11:53:47 +0800
Subject: [PATCH 08/35] fix: table would be null when destory commit handle

---
 source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 031d200a66..07a68d780a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -485,8 +485,10 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) {
 
   for (int i = 1; i < pCommith->niters; i++) {
     tSkipListDestroyIter(pCommith->iters[i].pIter);
-    tdFreeSchema(pCommith->iters[i].pTable->pSchema);
-    taosMemoryFree(pCommith->iters[i].pTable);
+    if (pCommith->iters[i].pTable) {
+      tdFreeSchema(pCommith->iters[i].pTable->pSchema);
+      taosMemoryFreeClear(pCommith->iters[i].pTable);
+    }
   }
 
   taosMemoryFree(pCommith->iters);

From 67ae3c2efa8c254b8877a1b12e57a824072c0b9c Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 12 May 2022 11:55:58 +0800
Subject: [PATCH 09/35] fix: table would be null when destroy commit handle

---
 source/dnode/vnode/src/tsdb/tsdbCommit.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 031d200a66..07a68d780a 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -485,8 +485,10 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith) {
 
   for (int i = 1; i < pCommith->niters; i++) {
     tSkipListDestroyIter(pCommith->iters[i].pIter);
-    tdFreeSchema(pCommith->iters[i].pTable->pSchema);
-    taosMemoryFree(pCommith->iters[i].pTable);
+    if (pCommith->iters[i].pTable) {
+      tdFreeSchema(pCommith->iters[i].pTable->pSchema);
+      taosMemoryFreeClear(pCommith->iters[i].pTable);
+    }
   }
 
   taosMemoryFree(pCommith->iters);

From 8b9a2c20f31224c7b9602e7910ebdafd18056a1b Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Thu, 12 May 2022 12:05:58 +0800
Subject: [PATCH 10/35] enh(sync): add error log, linux api error, %X

---
 source/dnode/vnode/src/vnd/vnodeSync.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c
index 1260f9a3e7..f0f5338c4d 100644
--- a/source/dnode/vnode/src/vnd/vnodeSync.c
+++ b/source/dnode/vnode/src/vnd/vnodeSync.c
@@ -72,6 +72,7 @@ int32_t vnodeSendMsg(void *rpcHandle, const SEpSet *pEpSet, SRpcMsg *pMsg) {
   int32_t ret = 0;
   SMsgCb *pMsgCb = rpcHandle;
   if (pMsgCb->queueFps[SYNC_QUEUE] != NULL) {
+    pMsg->noResp = 1;
     tmsgSendReq(rpcHandle, pEpSet, pMsg);
   } else {
     vError("vnodeSendMsg queue is NULL, SYNC_QUEUE:%d", SYNC_QUEUE);

From 77f4c3b5ab6ca568fc716bf3cb4fb9af048feebc Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 12 May 2022 13:30:05 +0800
Subject: [PATCH 11/35] fix: set commit table when move blk idx between
 different last file

---
 source/dnode/vnode/src/tsdb/tsdbCommit.c | 42 ++++++++++++++++++++----
 1 file changed, 36 insertions(+), 6 deletions(-)

diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index 031d200a66..3521e9b1f5 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -70,6 +70,7 @@ static int  tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
 static void tsdbResetCommitFile(SCommitH *pCommith);
 static int  tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
 static int  tsdbCommitToTable(SCommitH *pCommith, int tid);
+static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
 static int  tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx);
 static int  tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
 static int  tsdbComparKeyBlock(const void *arg1, const void *arg2);
@@ -889,9 +890,11 @@ static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
 }
 
 static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
-  SReadH *pReadh = &pCommith->readh;
-  int     nBlocks = pIdx->numOfBlocks;
-  int     bidx = 0;
+  SReadH   *pReadh = &pCommith->readh;
+  STsdb    *pTsdb = TSDB_READ_REPO(pReadh);
+  STSchema *pTSchema = NULL;
+  int       nBlocks = pIdx->numOfBlocks;
+  int       bidx = 0;
 
   tsdbResetCommitTable(pCommith);
 
@@ -901,30 +904,49 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
     return -1;
   }
 
+  STable table = {.tid = pIdx->uid, .uid = pIdx->uid, .pSchema = NULL};
+  pCommith->pTable = &table;
+
   while (bidx < nBlocks) {
+    if (!pTSchema && !tsdbCommitIsSameFile(pCommith, bidx)) {
+      // Set commit table
+      pTSchema = metaGetTbTSchema(REPO_META(pTsdb), pIdx->uid, 0);  // TODO: schema version
+      if (!pTSchema) {
+        terrno = TSDB_CODE_OUT_OF_MEMORY;
+        return -1;
+      }
+      table.pSchema = pTSchema;
+      if (tsdbSetCommitTable(pCommith, &table) < 0) {
+        taosMemoryFreeClear(pTSchema);
+        return -1;
+      }
+    }
+
     if (tsdbMoveBlock(pCommith, bidx) < 0) {
       tsdbError("vgId:%d failed to move block into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith),
                 TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno));
+      taosMemoryFreeClear(pTSchema);
       return -1;
     }
+
     ++bidx;
   }
 
-  STable table = {.tid = pIdx->uid, .uid = pIdx->uid, .pSchema = NULL};
-  TSDB_COMMIT_TABLE(pCommith) = &table;
-
   if (tsdbWriteBlockInfo(pCommith) < 0) {
     tsdbError("vgId:%d failed to write SBlockInfo part into file %s since %s", TSDB_COMMIT_REPO_ID(pCommith),
               TSDB_FILE_FULL_NAME(TSDB_COMMIT_HEAD_FILE(pCommith)), tstrerror(terrno));
+    taosMemoryFreeClear(pTSchema);
     return -1;
   }
 
+  taosMemoryFreeClear(pTSchema);
   return 0;
 }
 
 static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) {
   STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
 
+
   pCommith->pTable = pTable;
 
   if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) {
@@ -1321,6 +1343,14 @@ static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) {
   return 0;
 }
 
+static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx) {
+  SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
+  if (pBlock->last) {
+    return pCommith->isLFileSame;
+  }
+  return pCommith->isDFileSame;
+}
+
 static int tsdbMoveBlock(SCommitH *pCommith, int bidx) {
   SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
   SDFile *pDFile;

From c32a3400553cdfe65937be86e67c9173cbf404de Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 06:13:59 +0000
Subject: [PATCH 12/35] fix memory leak

---
 source/dnode/vnode/src/tsdb/tsdbMemTable.c | 10 ++++++++++
 source/libs/tdb/src/db/tdbBtree.c          |  1 +
 2 files changed, 11 insertions(+)

diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index d40a73eb67..1e8fbb48c7 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -62,6 +62,16 @@ int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable) {
 void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable) {
   if (pMemTable) {
     taosHashCleanup(pMemTable->pHashIdx);
+    SSkipListIterator *pIter = tSkipListCreateIter(pMemTable->pSlIdx);
+    SSkipListNode     *pNode = NULL;
+    STbData           *pTbData = NULL;
+    for (;;) {
+      if (!tSkipListIterNext(pIter)) break;
+      pNode = tSkipListIterGet(pIter);
+      pTbData = (STbData *)pNode->pData;
+      tsdbFreeTbData(pTbData);
+    }
+    tSkipListDestroyIter(pIter);
     tSkipListDestroy(pMemTable->pSlIdx);
     taosMemoryFree(pMemTable);
   }
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index cf7dd50103..fffda68731 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -114,6 +114,7 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, tdb_cmpr_fn_t kcmpr, SB
 
 int tdbBtreeClose(SBTree *pBt) {
   if (pBt) {
+    tdbFree(pBt->pBuf);
     tdbOsFree(pBt);
   }
   return 0;

From 1dbe0650e03d8316c185fea2fb2ef3cd531c9b2e Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Thu, 12 May 2022 14:20:50 +0800
Subject: [PATCH 13/35] enh(wal): set errno code

---
 source/libs/wal/src/walRead.c  | 13 ++++++++++---
 source/libs/wal/src/walWrite.c |  1 +
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 0cfe75bf33..7dfe1b8989 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -130,6 +130,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
     }
   }
 
+  // code set inner
   if (walReadSeekFilePos(pRead, pRet->firstVer, ver) < 0) {
     return -1;
   }
@@ -249,16 +250,22 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
   // TODO: check wal life
   if (pRead->curVersion != ver) {
     if (walReadSeekVer(pRead, ver) < 0) {
+      terrno = TSDB_CODE_WAL_INVALID_VER;
+      wError("unexpected wal log version: % " PRId64 ", since seek error", ver);
       return -1;
     }
   }
 
-  if (!taosValidFile(pRead->pReadLogTFile)) {
-    return -1;
-  }
+  /*if (!taosValidFile(pRead->pReadLogTFile)) {*/
+  /*return -1;*/
+  /*}*/
 
   code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
   if (code != sizeof(SWalHead)) {
+    if (code < 0)
+      terrno = TAOS_SYSTEM_ERROR(errno);
+    else
+      terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
     return -1;
   }
 
diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c
index da1c36dcc4..2e43997584 100644
--- a/source/libs/wal/src/walWrite.c
+++ b/source/libs/wal/src/walWrite.c
@@ -225,6 +225,7 @@ int walRoll(SWal *pWal) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
+  // terrno set inner
   code = walRollFileInfo(pWal);
   if (code != 0) {
     return -1;

From 71e43677f74b7c6c217e979934b8c324985f661b Mon Sep 17 00:00:00 2001
From: Ganlin Zhao <ganlinzhao@hotmail.com>
Date: Thu, 12 May 2022 14:38:56 +0800
Subject: [PATCH 14/35] feat(query): add csum function

---
 source/libs/function/inc/builtinsimpl.h |  3 ++
 source/libs/function/src/builtins.c     | 41 ++++++++++++++++++
 source/libs/function/src/builtinsimpl.c | 57 ++++++++++++++++++++++++-
 3 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h
index e3b7127efe..e1e30d37ea 100644
--- a/source/libs/function/inc/builtinsimpl.h
+++ b/source/libs/function/inc/builtinsimpl.h
@@ -94,6 +94,9 @@ bool stateFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
 int32_t stateCountFunction(SqlFunctionCtx* pCtx);
 int32_t stateDurationFunction(SqlFunctionCtx* pCtx);
 
+bool getCsumFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
+int32_t csumFunction(SqlFunctionCtx* pCtx);
+
 bool getSelectivityFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv);
 
 #ifdef __cplusplus
diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c
index 07ad0f7d1f..c9136433e0 100644
--- a/source/libs/function/src/builtins.c
+++ b/source/libs/function/src/builtins.c
@@ -308,6 +308,37 @@ static int32_t translateStateDuration(SFunctionNode* pFunc, char* pErrBuf, int32
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t translateCsum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
+  if (1 != LIST_LENGTH(pFunc->pParameterList)) {
+    return TSDB_CODE_SUCCESS;
+  }
+
+  SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
+  if (QUERY_NODE_COLUMN != nodeType(pPara)) {
+    return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR,
+                           "The input parameter of CSUM function can only be column");
+  }
+
+  uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type;
+  uint8_t resType;
+  if (!IS_NUMERIC_TYPE(colType)) {
+    return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
+  } else {
+    if (IS_SIGNED_NUMERIC_TYPE(colType)) {
+      resType = TSDB_DATA_TYPE_BIGINT;
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) {
+      resType = TSDB_DATA_TYPE_UBIGINT;
+    } else if (IS_FLOAT_TYPE(colType)) {
+      resType = TSDB_DATA_TYPE_DOUBLE;
+    } else {
+      ASSERT(0);
+    }
+  }
+
+  pFunc->node.resType = (SDataType) { .bytes = tDataTypes[resType].bytes, .type = resType};
+  return TSDB_CODE_SUCCESS;
+}
+
 static int32_t translateLastRow(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
   // todo
   return TSDB_CODE_SUCCESS;
@@ -742,6 +773,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .processFunc  = stateDurationFunction,
     .finalizeFunc = NULL
   },
+  {
+    .name = "csum",
+    .type = FUNCTION_TYPE_CSUM,
+    .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC,
+    .translateFunc = translateCsum,
+    .getEnvFunc   = getCsumFuncEnv,
+    .initFunc     = functionSetup,
+    .processFunc  = csumFunction,
+    .finalizeFunc = NULL
+  },
   {
     .name = "abs",
     .type = FUNCTION_TYPE_ABS,
diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c
index efc2992075..173dc9fd57 100644
--- a/source/libs/function/src/builtinsimpl.c
+++ b/source/libs/function/src/builtinsimpl.c
@@ -2815,7 +2815,6 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) {
   SInputColumnInfoData* pInput = &pCtx->input;
 
   SColumnInfoData* pInputCol = pInput->pData[0];
-  SColumnInfoData* pTsOutput = pCtx->pTsOutput;
 
   int32_t numOfElems = 0;
   SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
@@ -2853,7 +2852,6 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
   TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
 
   SColumnInfoData* pInputCol = pInput->pData[0];
-  SColumnInfoData* pTsOutput = pCtx->pTsOutput;
 
   int32_t numOfElems = 0;
   SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
@@ -2893,3 +2891,58 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) {
 
   return numOfElems;
 }
+
+bool getCsumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
+  pEnv->calcMemSize = sizeof(SSumRes);
+  return true;
+}
+
+int32_t csumFunction(SqlFunctionCtx* pCtx) {
+  SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
+  SSumRes*             pSumRes = GET_ROWCELL_INTERBUF(pResInfo);
+
+  SInputColumnInfoData* pInput = &pCtx->input;
+  TSKEY* tsList = (int64_t*)pInput->pPTS->pData;
+
+  SColumnInfoData* pInputCol = pInput->pData[0];
+  SColumnInfoData* pTsOutput = pCtx->pTsOutput;
+  SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput;
+
+  int32_t numOfElems = 0;
+  int32_t type = pInputCol->info.type;
+  int32_t startOffset = pCtx->offset;
+  for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) {
+    int32_t pos = startOffset + numOfElems;
+    if (colDataIsNull_f(pInputCol->nullbitmap, i)) {
+      //colDataAppendNULL(pOutput, i);
+      continue;
+    }
+
+    char* data = colDataGetData(pInputCol, i);
+    if (IS_SIGNED_NUMERIC_TYPE(type)) {
+      int64_t v;
+      GET_TYPED_DATA(v, int64_t, type, data);
+      pSumRes->isum += v;
+      colDataAppend(pOutput, pos, (char *)&pSumRes->isum, false);
+    } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
+      uint64_t v;
+      GET_TYPED_DATA(v, uint64_t, type, data);
+      pSumRes->usum += v;
+      colDataAppend(pOutput, pos, (char *)&pSumRes->usum, false);
+    } else if (IS_FLOAT_TYPE(type)) {
+      double v;
+      GET_TYPED_DATA(v, double, type, data);
+      pSumRes->dsum += v;
+      colDataAppend(pOutput, pos, (char *)&pSumRes->dsum, false);
+    }
+
+    //TODO: remove this after pTsOutput is handled
+    if (pTsOutput != NULL) {
+      colDataAppendInt64(pTsOutput, pos, &tsList[i]);
+    }
+
+    numOfElems++;
+  }
+
+  return numOfElems;
+}

From d7bd682237e710f1a6d319150d113d82dc6d53dd Mon Sep 17 00:00:00 2001
From: "slzhou@taodata.com" <shenglian zhou>
Date: Thu, 12 May 2022 14:42:57 +0800
Subject: [PATCH 15/35] column has member hasNull

---
 include/libs/function/tudf.h    |  2 ++
 include/util/tcoding.h          | 15 +++++++++++++++
 source/common/src/tdatablock.c  |  2 ++
 source/libs/function/src/tudf.c |  2 ++
 4 files changed, 21 insertions(+)

diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h
index bdccd29acf..8e156edcd2 100644
--- a/include/libs/function/tudf.h
+++ b/include/libs/function/tudf.h
@@ -89,6 +89,7 @@ typedef struct SUdfColumnData {
 
 typedef struct SUdfColumn {
   SUdfColumnMeta colMeta;
+  bool           hasNull;
   SUdfColumnData colData;
 } SUdfColumn;
 
@@ -232,6 +233,7 @@ static FORCE_INLINE void udfColDataSetNull(SUdfColumn* pColumn, int32_t row) {
   } else {
     udfColDataSetNull_f(pColumn, row);
   }
+  pColumn->hasNull = true;
 }
 
 static FORCE_INLINE int32_t udfColDataSet(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) {
diff --git a/include/util/tcoding.h b/include/util/tcoding.h
index 3f00c79f46..74e64d5292 100644
--- a/include/util/tcoding.h
+++ b/include/util/tcoding.h
@@ -59,6 +59,21 @@ static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) {
 
 static FORCE_INLINE void *taosSkipFixedLen(const void *buf, size_t len) { return POINTER_SHIFT(buf, len); }
 
+// --- Bool
+
+static FORCE_INLINE int32_t taosEncodeFixedBool(void **buf, bool value) {
+  if (buf != NULL) {
+    ((int8_t *)(*buf))[0] = value ? 1 : 0;
+    *buf = POINTER_SHIFT(*buf, sizeof(int8_t));
+  }
+  return (int32_t)sizeof(int8_t);
+}
+
+static FORCE_INLINE void *taosDecodeFixedBool(const void *buf, bool *value) {
+  *value = ((int8_t *)buf)[0] == 0 ? false : true;
+  return POINTER_SHIFT(buf, sizeof(int8_t));
+}
+
 // ---- Fixed U16
 static FORCE_INLINE int32_t taosEncodeFixedU16(void **buf, uint16_t value) {
   if (buf != NULL) {
diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c
index b58e4bd1dd..9053101938 100644
--- a/source/common/src/tdatablock.c
+++ b/source/common/src/tdatablock.c
@@ -1311,6 +1311,7 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
     tlen += taosEncodeFixedI16(buf, pColData->info.colId);
     tlen += taosEncodeFixedI16(buf, pColData->info.type);
     tlen += taosEncodeFixedI32(buf, pColData->info.bytes);
+    tlen += taosEncodeFixedBool(buf, pColData->hasNull);
 
     if (IS_VAR_DATA_TYPE(pColData->info.type)) {
       tlen += taosEncodeBinary(buf, pColData->varmeta.offset, sizeof(int32_t) * rows);
@@ -1340,6 +1341,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
     buf = taosDecodeFixedI16(buf, &data.info.colId);
     buf = taosDecodeFixedI16(buf, &data.info.type);
     buf = taosDecodeFixedI32(buf, &data.info.bytes);
+    buf = taosDecodeFixedBool(buf, &data.hasNull);
 
     if (IS_VAR_DATA_TYPE(data.info.type)) {
       buf = taosDecodeBinary(buf, (void**)&data.varmeta.offset, pBlock->info.rows * sizeof(int32_t));
diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index a577ea200f..9d90dca63e 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -695,6 +695,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
     udfCol->colMeta.scale = col->info.scale;
     udfCol->colMeta.precision = col->info.precision;
     udfCol->colData.numOfRows = udfBlock->numOfRows;
+    udfCol->hasNull = col->hasNull;
     if (IS_VAR_DATA_TYPE(udfCol->colMeta.type)) {
       udfCol->colData.varLenCol.varOffsetsLen = sizeof(int32_t) * udfBlock->numOfRows;
       udfCol->colData.varLenCol.varOffsets = taosMemoryMalloc(udfCol->colData.varLenCol.varOffsetsLen);
@@ -731,6 +732,7 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
   col->info.bytes = meta->bytes;
   col->info.scale = meta->scale;
   col->info.type = meta->type;
+  col->hasNull = udfCol->hasNull;
   SUdfColumnData *data = &udfCol->colData;
 
   if (!IS_VAR_DATA_TYPE(meta->type)) {

From 06853043bd05ddccbd32b5fec71553aa73023fdf Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Thu, 12 May 2022 14:57:58 +0800
Subject: [PATCH 16/35] feat(tmq): add config msg.with.table.name

---
 example/src/tmq.c              |  3 ++-
 include/common/tmsg.h          |  1 +
 source/client/src/tmq.c        | 39 ++++++++++++++++++++++++----------
 source/dnode/vnode/src/tq/tq.c | 11 +++++++---
 source/libs/wal/src/walRead.c  |  8 ++-----
 5 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/example/src/tmq.c b/example/src/tmq.c
index 2ee91c254c..0b5f3be1b0 100644
--- a/example/src/tmq.c
+++ b/example/src/tmq.c
@@ -107,7 +107,7 @@ int32_t create_topic() {
   taos_free_result(pRes);
 
   /*pRes = taos_query(pConn, "create topic topic_ctb_column as abc1");*/
-  pRes = taos_query(pConn, "create topic topic_ctb_column with table as select ts, c1, c2, c3 from st1");
+  pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");
   if (taos_errno(pRes) != 0) {
     printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes));
     return -1;
@@ -166,6 +166,7 @@ tmq_t* build_consumer() {
   tmq_conf_set(conf, "td.connect.user", "root");
   tmq_conf_set(conf, "td.connect.pass", "taosdata");
   /*tmq_conf_set(conf, "td.connect.db", "abc1");*/
+  tmq_conf_set(conf, "msg.with.table.name", "true");
   tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
   tmq_t* tmq = tmq_consumer_new(conf, NULL, 0);
   assert(tmq);
diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index d34892a278..7bb1d70421 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -2371,6 +2371,7 @@ typedef struct {
 typedef struct {
   SMsgHead head;
   char     subKey[TSDB_SUBSCRIBE_KEY_LEN];
+  int8_t   withTbName;
   int32_t  epoch;
   uint64_t reqId;
   int64_t  consumerId;
diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c
index b42f072e54..a6b8b842f9 100644
--- a/source/client/src/tmq.c
+++ b/source/client/src/tmq.c
@@ -57,16 +57,17 @@ struct tmq_topic_vgroup_list_t {
 };
 
 struct tmq_conf_t {
-  char           clientId[256];
-  char           groupId[TSDB_CGROUP_LEN];
-  int8_t         autoCommit;
-  int8_t         resetOffset;
-  uint16_t       port;
-  int32_t        autoCommitInterval;
-  char*          ip;
-  char*          user;
-  char*          pass;
-  char*          db;
+  char     clientId[256];
+  char     groupId[TSDB_CGROUP_LEN];
+  int8_t   autoCommit;
+  int8_t   resetOffset;
+  int8_t   withTbName;
+  uint16_t port;
+  int32_t  autoCommitInterval;
+  char*    ip;
+  char*    user;
+  char*    pass;
+  /*char*          db;*/
   tmq_commit_cb* commitCb;
   void*          commitCbUserParam;
 };
@@ -75,6 +76,7 @@ struct tmq_t {
   // conf
   char           groupId[TSDB_CGROUP_LEN];
   char           clientId[256];
+  int8_t         withTbName;
   int8_t         autoCommit;
   int32_t        autoCommitInterval;
   int32_t        resetOffsetCfg;
@@ -187,6 +189,7 @@ typedef struct {
 
 tmq_conf_t* tmq_conf_new() {
   tmq_conf_t* conf = taosMemoryCalloc(1, sizeof(tmq_conf_t));
+  conf->withTbName = -1;
   conf->autoCommit = true;
   conf->autoCommitInterval = 5000;
   conf->resetOffset = TMQ_CONF__RESET_OFFSET__EARLIEAST;
@@ -240,6 +243,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     }
   }
 
+  if (strcmp(key, "msg.with.table.name") == 0) {
+    if (strcmp(value, "true") == 0) {
+      conf->withTbName = 1;
+    } else if (strcmp(value, "false") == 0) {
+      conf->withTbName = 0;
+    } else if (strcmp(value, "none") == 0) {
+      conf->withTbName = -1;
+    } else {
+      return TMQ_CONF_INVALID;
+    }
+  }
+
   if (strcmp(key, "td.connect.ip") == 0) {
     conf->ip = strdup(value);
     return TMQ_CONF_OK;
@@ -257,7 +272,7 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value
     return TMQ_CONF_OK;
   }
   if (strcmp(key, "td.connect.db") == 0) {
-    conf->db = strdup(value);
+    /*conf->db = strdup(value);*/
     return TMQ_CONF_OK;
   }
 
@@ -485,6 +500,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
   // set conf
   strcpy(pTmq->clientId, conf->clientId);
   strcpy(pTmq->groupId, conf->groupId);
+  pTmq->withTbName = conf->withTbName;
   pTmq->autoCommit = conf->autoCommit;
   pTmq->autoCommitInterval = conf->autoCommitInterval;
   pTmq->commitCb = conf->commitCb;
@@ -1104,6 +1120,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t waitTime, SMqClientTopic*
   pReq->subKey[tlen] = TMQ_SEPARATOR;
   strcpy(pReq->subKey + tlen + 1, pTopic->topicName);
 
+  pReq->withTbName = tmq->withTbName;
   pReq->waitTime = waitTime;
   pReq->consumerId = tmq->consumerId;
   pReq->epoch = tmq->epoch;
diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c
index 29fabc0f9f..6f0c25fb91 100644
--- a/source/dnode/vnode/src/tq/tq.c
+++ b/source/dnode/vnode/src/tq/tq.c
@@ -427,13 +427,18 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
   SMqDataBlkRsp rsp = {0};
   rsp.reqOffset = pReq->currentOffset;
   rsp.withSchema = pExec->withSchema;
-  rsp.withTbName = pExec->withTbName;
 
   rsp.blockData = taosArrayInit(0, sizeof(void*));
   rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t));
   rsp.blockSchema = taosArrayInit(0, sizeof(void*));
   rsp.blockTbName = taosArrayInit(0, sizeof(void*));
 
+  int8_t withTbName = pExec->withTbName;
+  if (pReq->withTbName != -1) {
+    withTbName = pReq->withTbName;
+  }
+  rsp.withTbName = withTbName;
+
   while (1) {
     consumerEpoch = atomic_load_32(&pExec->epoch);
     if (consumerEpoch > reqEpoch) {
@@ -538,7 +543,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
             taosArrayPush(rsp.blockSchema, &pSW);
           }
 
-          if (pExec->withTbName) {
+          if (withTbName) {
             SMetaReader mr = {0};
             metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
             int64_t uid = pExec->pExecReader[workerId]->msgIter.uid;
@@ -578,7 +583,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
           ASSERT(actualLen <= dataStrLen);
           taosArrayPush(rsp.blockDataLen, &actualLen);
           taosArrayPush(rsp.blockData, &buf);
-          if (pExec->withTbName) {
+          if (withTbName) {
             SMetaReader mr = {0};
             metaReaderInit(&mr, pTq->pVnode->pMeta, 0);
             if (metaGetTableEntryByUid(&mr, block.info.uid) < 0) {
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 7dfe1b8989..64e6881cd0 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -155,9 +155,7 @@ int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) {
     if (code < 0) return -1;
   }
 
-  if (!taosValidFile(pRead->pReadLogTFile)) {
-    return -1;
-  }
+  ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
 
   code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalHead));
   if (code != sizeof(SWalHead)) {
@@ -256,9 +254,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
     }
   }
 
-  /*if (!taosValidFile(pRead->pReadLogTFile)) {*/
-  /*return -1;*/
-  /*}*/
+  ASSERT(taosValidFile(pRead->pReadLogTFile) == true);
 
   code = taosReadFile(pRead->pReadLogTFile, pRead->pHead, sizeof(SWalHead));
   if (code != sizeof(SWalHead)) {

From 62b2ccedf13d245b1b1e5e44a0f7fce53d4fb6b8 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 07:09:27 +0000
Subject: [PATCH 17/35] return errno when table not exits

---
 include/common/tmsg.h                      |  3 ++-
 source/common/src/tmsg.c                   |  7 ++++---
 source/dnode/vnode/src/tsdb/tsdbMemTable.c | 11 +++++++++++
 source/dnode/vnode/src/vnd/vnodeSvr.c      |  9 ++++-----
 4 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/include/common/tmsg.h b/include/common/tmsg.h
index d655b82a08..08d5765246 100644
--- a/include/common/tmsg.h
+++ b/include/common/tmsg.h
@@ -252,6 +252,7 @@ STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
 int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema);
 
 typedef struct {
+  int32_t code;
   int8_t  hashMeta;
   int64_t uid;
   char*   tblFName;
@@ -271,7 +272,7 @@ typedef struct {
 
 int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
 int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void tFreeSSubmitRsp(SSubmitRsp *pRsp);
+void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
 #define COL_SMA_ON  ((int8_t)0x1)
 #define COL_IDX_ON  ((int8_t)0x2)
diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c
index ebd81c7da3..021ee8455e 100644
--- a/source/common/src/tmsg.c
+++ b/source/common/src/tmsg.c
@@ -4032,6 +4032,7 @@ int32_t tDecodeSVSubmitReq(SDecoder *pCoder, SVSubmitReq *pReq) {
 static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBlock) {
   if (tStartEncode(pEncoder) < 0) return -1;
 
+  if (tEncodeI32(pEncoder, pBlock->code) < 0) return -1;
   if (tEncodeI8(pEncoder, pBlock->hashMeta) < 0) return -1;
   if (pBlock->hashMeta) {
     if (tEncodeI64(pEncoder, pBlock->uid) < 0) return -1;
@@ -4047,10 +4048,11 @@ static int32_t tEncodeSSubmitBlkRsp(SEncoder *pEncoder, const SSubmitBlkRsp *pBl
 static int32_t tDecodeSSubmitBlkRsp(SDecoder *pDecoder, SSubmitBlkRsp *pBlock) {
   if (tStartDecode(pDecoder) < 0) return -1;
 
+  if (tDecodeI32(pDecoder, &pBlock->code) < 0) return -1;
   if (tDecodeI8(pDecoder, &pBlock->hashMeta) < 0) return -1;
   if (pBlock->hashMeta) {
     if (tDecodeI64(pDecoder, &pBlock->uid) < 0) return -1;
-    pBlock->tblFName= taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
+    pBlock->tblFName = taosMemoryCalloc(TSDB_TABLE_FNAME_LEN, 1);
     if (NULL == pBlock->tblFName) return -1;
     if (tDecodeCStrTo(pDecoder, pBlock->tblFName) < 0) return -1;
   }
@@ -4089,7 +4091,7 @@ int32_t tDecodeSSubmitRsp(SDecoder *pDecoder, SSubmitRsp *pRsp) {
     if (tDecodeSSubmitBlkRsp(pDecoder, pRsp->pBlocks + iBlock) < 0) return -1;
   }
 
-  tEndDecode(pDecoder);  
+  tEndDecode(pDecoder);
   tDecoderClear(pDecoder);
   return 0;
 }
@@ -4108,4 +4110,3 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) {
 
   taosMemoryFree(pRsp);
 }
-
diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
index 1e8fbb48c7..037b099345 100644
--- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c
+++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c
@@ -310,6 +310,17 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo
   TSKEY          keyMax;
   SSubmitBlk    *pBlkCopy;
 
+  // check if table exists
+  SMetaReader mr = {0};
+  SMetaEntry  me = {0};
+  metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
+  if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) {
+    metaReaderClear(&mr);
+    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+    return -1;
+  }
+  metaReaderClear(&mr);
+
   // create container is nedd
   tptr = taosHashGet(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid));
   if (tptr == NULL) {
diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index fc2b6fe676..158e14f2ad 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -502,7 +502,7 @@ _exit:
   return 0;
 }
 
-static int vnodeDebugPrintSingleSubmitMsg(SMeta         *pMeta, SSubmitBlk  *pBlock, SSubmitMsgIter *msgIter, const char *tags) {
+static int vnodeDebugPrintSingleSubmitMsg(SMeta *pMeta, SSubmitBlk *pBlock, SSubmitMsgIter *msgIter, const char *tags) {
   SSubmitBlkIter blkIter = {0};
   STSchema      *pSchema = NULL;
   tb_uid_t       suid = 0;
@@ -544,7 +544,7 @@ static int vnodeDebugPrintSubmitMsg(SVnode *pVnode, SSubmitReq *pMsg, const char
   while (true) {
     if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1;
     if (pBlock == NULL) break;
-    
+
     vnodeDebugPrintSingleSubmitMsg(pMeta, pBlock, &msgIter, tags);
   }
 
@@ -595,7 +595,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
 
       if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) {
         if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) {
-          pRsp->code = terrno;
+          submitBlkRsp.code = terrno;
           tDecoderClear(&decoder);
           goto _exit;
         }
@@ -617,8 +617,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
     }
 
     if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) {
-      pRsp->code = terrno;
-      goto _exit;
+      submitBlkRsp.code = terrno;
     }
 
     submitRsp.numOfRows += submitBlkRsp.numOfRows;

From b69a26678c6fa8fee3d104da2fafd857fde3b506 Mon Sep 17 00:00:00 2001
From: Minghao Li <castermode@gmail.com>
Date: Thu, 12 May 2022 15:23:41 +0800
Subject: [PATCH 18/35] enh(sync): raft config change

---
 include/common/tmsgdef.h                 |  1 +
 source/libs/sync/inc/syncInt.h           |  1 +
 source/libs/sync/inc/syncUtil.h          |  5 +++
 source/libs/sync/src/syncAppendEntries.c | 22 +++++++++---
 source/libs/sync/src/syncCommit.c        | 13 ++++++-
 source/libs/sync/src/syncMain.c          | 44 ++++++++++++++++++++++--
 source/libs/sync/src/syncUtil.c          | 28 +++++++++++++++
 7 files changed, 107 insertions(+), 7 deletions(-)

diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h
index 8e918c40f9..c7deaa7845 100644
--- a/include/common/tmsgdef.h
+++ b/include/common/tmsgdef.h
@@ -217,6 +217,7 @@ enum {
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_UNKNOWN, "vnode-sync-unknown", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_COMMON_RESPONSE, "vnode-sync-common-response", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_APPLY_MSG, "vnode-sync-apply-msg", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_SYNC_CONFIG_CHANGE, "vnode-sync-config-change", NULL, NULL)
   
   TD_DEF_MSG_TYPE(TDMT_VND_SYNC_VNODE, "vnode-sync-vnode", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_ALTER_VNODE, "vnode-alter-vnode", NULL, NULL)
diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h
index cb539e8379..8a21eea7b7 100644
--- a/source/libs/sync/inc/syncInt.h
+++ b/source/libs/sync/inc/syncInt.h
@@ -271,6 +271,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S
 cJSON*  syncNode2Json(const SSyncNode* pSyncNode);
 char*   syncNode2Str(const SSyncNode* pSyncNode);
 char*   syncNode2SimpleStr(const SSyncNode* pSyncNode);
+void    syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig);
 
 SSyncNode* syncNodeAcquire(int64_t rid);
 void       syncNodeRelease(SSyncNode* pNode);
diff --git a/source/libs/sync/inc/syncUtil.h b/source/libs/sync/inc/syncUtil.h
index 83f31c5dac..159af1610e 100644
--- a/source/libs/sync/inc/syncUtil.h
+++ b/source/libs/sync/inc/syncUtil.h
@@ -57,6 +57,11 @@ SyncIndex   syncUtilMinIndex(SyncIndex a, SyncIndex b);
 SyncIndex   syncUtilMaxIndex(SyncIndex a, SyncIndex b);
 void        syncUtilMsgHtoN(void* msg);
 void        syncUtilMsgNtoH(void* msg);
+bool        syncUtilIsData(tmsg_t msgType);
+bool        syncUtilUserPreCommit(tmsg_t msgType);
+bool        syncUtilUserCommit(tmsg_t msgType);
+bool        syncUtilUserRollback(tmsg_t msgType);
+
 
 #ifdef __cplusplus
 }
diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c
index 6623ed1caa..aed19d042e 100644
--- a/source/libs/sync/src/syncAppendEntries.c
+++ b/source/libs/sync/src/syncAppendEntries.c
@@ -19,6 +19,7 @@
 #include "syncRaftStore.h"
 #include "syncUtil.h"
 #include "syncVoteMgr.h"
+#include "syncRaftCfg.h"
 
 // TLA+ Spec
 // HandleAppendEntriesRequest(i, j, m) ==
@@ -199,7 +200,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
             SSyncRaftEntry* pRollBackEntry = logStoreGetEntry(ths->pLogStore, index);
             assert(pRollBackEntry != NULL);
 
-            if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
+            //if (pRollBackEntry->msgType != TDMT_VND_SYNC_NOOP) {
+            if (syncUtilUserRollback(pRollBackEntry->msgType)) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg);
 
@@ -227,7 +229,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
         SRpcMsg rpcMsg;
         syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
         if (ths->pFsm != NULL) {
-          if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
             SFsmCbMeta cbMeta;
             cbMeta.index = pAppendEntry->index;
             cbMeta.isWeak = pAppendEntry->isWeak;
@@ -258,7 +261,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
       SRpcMsg rpcMsg;
       syncEntry2OriginalRpc(pAppendEntry, &rpcMsg);
       if (ths->pFsm != NULL) {
-        if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+        //if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+        if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) {
           SFsmCbMeta cbMeta;
           cbMeta.index = pAppendEntry->index;
           cbMeta.isWeak = pAppendEntry->isWeak;
@@ -320,7 +324,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
               SRpcMsg rpcMsg;
               syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-              if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+              //if (ths->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+              if (ths->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
                 SFsmCbMeta cbMeta;
                 cbMeta.index = pEntry->index;
                 cbMeta.isWeak = pEntry->isWeak;
@@ -330,6 +335,15 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) {
                 ths->pFsm->FpCommitCb(ths->pFsm, &rpcMsg, cbMeta);
               }
 
+              // config change
+              if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
+                SSyncCfg newSyncCfg;
+                int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
+                ASSERT(ret == 0);
+
+                syncNodeUpdateConfig(ths, &newSyncCfg);
+              }
+
               rpcFreeCont(rpcMsg.pCont);
               syncEntryDestory(pEntry);
             }
diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c
index 5d6cbd2a58..620f0e9cd2 100644
--- a/source/libs/sync/src/syncCommit.c
+++ b/source/libs/sync/src/syncCommit.c
@@ -19,6 +19,7 @@
 #include "syncRaftLog.h"
 #include "syncRaftStore.h"
 #include "syncUtil.h"
+#include "syncRaftCfg.h"
 
 // \* Leader i advances its commitIndex.
 // \* This is done as a separate step from handling AppendEntries responses,
@@ -101,7 +102,8 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
           SRpcMsg rpcMsg;
           syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
-          if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          //if (pSyncNode->pFsm->FpCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+          if (pSyncNode->pFsm->FpCommitCb != NULL && syncUtilUserCommit(pEntry->originalRpcType)) {
             SFsmCbMeta cbMeta;
             cbMeta.index = pEntry->index;
             cbMeta.isWeak = pEntry->isWeak;
@@ -111,6 +113,15 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) {
             pSyncNode->pFsm->FpCommitCb(pSyncNode->pFsm, &rpcMsg, cbMeta);
           }
 
+          // config change
+            if (pEntry->originalRpcType == TDMT_VND_SYNC_CONFIG_CHANGE) {
+              SSyncCfg newSyncCfg;
+              int32_t ret = syncCfgFromStr(rpcMsg.pCont, &newSyncCfg);
+              ASSERT(ret == 0);
+
+              syncNodeUpdateConfig(pSyncNode, &newSyncCfg);
+          }
+
           rpcFreeCont(rpcMsg.pCont);
           syncEntryDestory(pEntry);
         }
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 911d8384f0..da23d8415b 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -116,6 +116,15 @@ void syncStop(int64_t rid) {
 
 int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) {
   int32_t ret = 0;
+  char *configChange = syncCfg2Str((SSyncCfg*)pSyncCfg);
+  SRpcMsg rpcMsg = {0};
+  rpcMsg.msgType = TDMT_VND_SYNC_CONFIG_CHANGE;
+  rpcMsg.noResp = 1;
+  rpcMsg.contLen = strlen(configChange) + 1;
+  rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen);
+  snprintf(rpcMsg.pCont, rpcMsg.contLen, "%s", configChange);
+  taosMemoryFree(configChange);
+  ret = syncPropose(rid, &rpcMsg, false);
   return ret;
 }
 
@@ -849,6 +858,35 @@ char* syncNode2SimpleStr(const SSyncNode* pSyncNode) {
   return s;
 }
 
+void syncNodeUpdateConfig(SSyncNode* pSyncNode, SSyncCfg *newConfig) {
+  pSyncNode->pRaftCfg->cfg = *newConfig;
+  int32_t ret = raftCfgPersist(pSyncNode->pRaftCfg);
+  ASSERT(ret == 0);
+
+  // init internal
+  pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex];
+  syncUtilnodeInfo2raftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId);
+
+  // init peersNum, peers, peersId
+  pSyncNode->peersNum = pSyncNode->pRaftCfg->cfg.replicaNum - 1;
+  int j = 0;
+  for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
+    if (i != pSyncNode->pRaftCfg->cfg.myIndex) {
+      pSyncNode->peersNodeInfo[j] = pSyncNode->pRaftCfg->cfg.nodeInfo[i];
+      j++;
+    }
+  }
+  for (int i = 0; i < pSyncNode->peersNum; ++i) {
+    syncUtilnodeInfo2raftId(&pSyncNode->peersNodeInfo[i], pSyncNode->vgId, &pSyncNode->peersId[i]);
+  }
+
+  // init replicaNum, replicasId
+  pSyncNode->replicaNum = pSyncNode->pRaftCfg->cfg.replicaNum;
+  for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) {
+    syncUtilnodeInfo2raftId(&pSyncNode->pRaftCfg->cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]);
+  }
+}
+
 SSyncNode* syncNodeAcquire(int64_t rid) {
   SSyncNode* pNode = taosAcquireRef(tsNodeRefId, rid);
   if (pNode == NULL) {
@@ -1207,7 +1245,8 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta;
         cbMeta.index = pEntry->index;
         cbMeta.isWeak = pEntry->isWeak;
@@ -1228,7 +1267,8 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) {
     syncEntry2OriginalRpc(pEntry, &rpcMsg);
 
     if (ths->pFsm != NULL) {
-      if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      //if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_VND_SYNC_NOOP) {
+      if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) {
         SFsmCbMeta cbMeta;
         cbMeta.index = pEntry->index;
         cbMeta.isWeak = pEntry->isWeak;
diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c
index 3110c0b2a3..cf045a6926 100644
--- a/source/libs/sync/src/syncUtil.c
+++ b/source/libs/sync/src/syncUtil.c
@@ -212,4 +212,32 @@ void syncUtilMsgNtoH(void* msg) {
   SMsgHead* pHead = msg;
   pHead->contLen = ntohl(pHead->contLen);
   pHead->vgId = ntohl(pHead->vgId);
+}
+
+bool syncUtilIsData(tmsg_t msgType) {
+  if (msgType == TDMT_VND_SYNC_NOOP || msgType == TDMT_VND_SYNC_CONFIG_CHANGE) {
+    return false;
+  }
+  return true;
+}
+
+bool syncUtilUserPreCommit(tmsg_t msgType) {
+  if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) {
+    return true;
+  }
+  return false;
+}
+
+bool syncUtilUserCommit(tmsg_t msgType) {
+  if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) {
+    return true;
+  }
+  return false;
+}
+
+bool syncUtilUserRollback(tmsg_t msgType) {
+  if (msgType != TDMT_VND_SYNC_NOOP && msgType != TDMT_VND_SYNC_CONFIG_CHANGE) {
+    return true;
+  }
+  return false;
 }
\ No newline at end of file

From 4511f5828fb3f5e650acc5405366df9611722cfc Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 12 May 2022 15:34:08 +0800
Subject: [PATCH 19/35] test: add test case for tmq

---
 tests/system-test/7-tmq/basic5.py |   85 ++-
 tests/system-test/fulltest.sh     |    4 +
 tests/test/c/tmqSim.c             | 1026 +++++++++++++++--------------
 3 files changed, 608 insertions(+), 507 deletions(-)

diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py
index 99aa4e72aa..7ed9f7ebe7 100644
--- a/tests/system-test/7-tmq/basic5.py
+++ b/tests/system-test/7-tmq/basic5.py
@@ -61,7 +61,7 @@ class TDTestCase:
         tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum))
         return
 
-    def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,startTs):
+    def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
         tdLog.debug("start to insert data ............")
         tdSql.execute("use %s" %dbName)
         pre_insert = "insert into "
@@ -72,13 +72,15 @@ class TDTestCase:
             sql += " %s_%d values "%(stbName,i)
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
-                if (j > 0) and (j%2000 == 0):
+                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
                     tdSql.execute(sql)
-                    sql = "insert into %s_%d values " %(stbName,i)
+                    if j < rowsPerTbl - 1:
+                        sql = "insert into %s_%d values " %(stbName,i)
+                    else:
+                        sql = "insert into "
         #end sql
         if sql != pre_insert:
-            # print(sql)
-            print("sql:%s"%sql)
+            #print("insert sql:%s"%sql)
             tdSql.execute(sql)
         tdLog.debug("insert data ............ [OK]")
         return
@@ -96,6 +98,7 @@ class TDTestCase:
                            parameterDict["stbName"],\
                            parameterDict["ctbNum"],\
                            parameterDict["rowsPerTbl"],\
+                           parameterDict["batchNum"],\
                            parameterDict["startTs"])                           
         return                   
         
@@ -117,13 +120,81 @@ class TDTestCase:
                          'vgroups':    1,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 10,    \
+                         'rowsPerTbl': 10000,    \
+                         'batchNum':   10,       \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
         prepareEnvThread.start()
+        time.sleep(1)
         
+        # wait stb ready
+        while 1:
+            tdSql.query("show %s.stables"%parameterDict['dbName'])
+            if tdSql.getRows() == 1:
+            #if (self.queryRows == 1):
+                time.sleep(1)
+                break
+
+        tdLog.info("create topics from super table")
+        topicFromStb = 'topic_stb_column'
+        topicFromCtb = 'topic_ctb_column'        
+        
+        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
+        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName']))
+
+        tdSql.query("show topics")
+        tdSql.checkRows(2)
+        topic1 = tdSql.getData(0 , 0)
+        topic2 = tdSql.getData(1 , 0)
+        if topic1 != topicFromStb or topic1 != topicFromCtb:
+            tdLog.exit("topic error") 
+        if topic2 != topicFromStb or topic2 != topicFromCtb:
+            tdLog.exit("topic error") 
+         
+        tdLog.info("create consume info table and consume result table")
+        cdbName = 'cdb'
+        tdSql.query("create database %s"%cdbName)
+        tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)")
+        tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)")
+
+        consumerId   = 0
+        expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] + 1) * parameterDict["ctbNum"]
+        topicList    = topicFromStb
+        ifcheckdata  = 0
+        keyList      = 'group.id:cgrp1,               \
+                        enable.auto.commit:false,     \
+                        auto.commit.interval.ms:6000, \
+                        auto.offset.reset:none'
+        sql = "insert into consumeinfo values "
+        sql += "(now, %d, '%s', '%s', %l64d, %d)"%(consumerId, topicList, keyList, expectmsgcnt, ifcheckdata)
+        tdSql.query(sql)
+        
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        
+        shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
+        shellCmd += " -y %d -d %s, -g %d, -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) 
+        shellCmd += "> /dev/null 2>&1 &"        
+        tdLog.info(shellCmd)
+        os.system(taosCmd)        
+
         # wait for data ready
-        prepareEnvThread.join()        
+        prepareEnvThread.join()
+        
+        tdLog.info("check consume result")
+        while 1:
+            tdSql.query("select * from consumeresult")
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            if tdSql.getRows() == 1:
+            #if (self.queryRows == 1):
+                time.sleep(1)
+                break
+            
+        tdSql.checkData(0 , 1, consumerId)
+        tdSql.checkData(0 , 2, expectmsgcnt)
+        tdSql.checkData(0 , 3, expectrowcnt)
         
         tdLog.printNoPrefix("======== test scenario 2: ")
 
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index a6b4408cdc..817f814873 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -51,3 +51,7 @@ python3 ./test.py -f 2-query/arcsin.py
 python3 ./test.py -f 2-query/arccos.py
 python3 ./test.py -f 2-query/arctan.py
 # python3 ./test.py -f 2-query/query_cols_tags_and_or.py
+
+python3 ./test.py -f 7-tmq/basic5.py
+
+
diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c
index 4a59d18d87..bc3aa091c3 100644
--- a/tests/test/c/tmqSim.c
+++ b/tests/test/c/tmqSim.c
@@ -1,500 +1,526 @@
-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <time.h>
-
-#include "taos.h"
-#include "taoserror.h"
-#include "tlog.h"
-
-#define GREEN     "\033[1;32m"
-#define NC        "\033[0m"
-#define min(a, b) (((a) < (b)) ? (a) : (b))
-
-#define MAX_SQL_STR_LEN         (1024 * 1024)
-#define MAX_ROW_STR_LEN         (16 * 1024)
-#define MAX_CONSUMER_THREAD_CNT (16)
-
-typedef struct {
-  TdThread thread;
-  int32_t  consumerId;
-
-  int32_t ifCheckData;
-  int64_t expectMsgCnt;
-
-  int64_t consumeMsgCnt;
-  int64_t consumeRowCnt;
-  int32_t checkresult;
-
-  char topicString[1024];
-  char keyString[1024];
-
-  int32_t numOfTopic;
-  char    topics[32][64];
-
-  int32_t numOfKey;
-  char    key[32][64];
-  char    value[32][64];
-
-  tmq_t*      tmq;
-  tmq_list_t* topicList;
-
-} SThreadInfo;
-
-typedef struct {
-  // input from argvs
-  char        cdbName[32];
-  char        dbName[32];
-  int32_t     showMsgFlag;
-  int32_t     showRowFlag;
-  int32_t     consumeDelay;  // unit s
-  int32_t     numOfThread;
-  SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT];
-} SConfInfo;
-
-static SConfInfo g_stConfInfo;
-TdFilePtr        g_fp = NULL;
-
-// char* g_pRowValue = NULL;
-// TdFilePtr g_fp = NULL;
-
-static void printHelp() {
-  char indent[10] = "        ";
-  printf("Used to test the tmq feature with sim cases\n");
-
-  printf("%s%s\n", indent, "-c");
-  printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
-  printf("%s%s\n", indent, "-d");
-  printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default ");
-  printf("%s%s\n", indent, "-g");
-  printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag);
-  printf("%s%s\n", indent, "-r");
-  printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag);
-  printf("%s%s\n", indent, "-y");
-  printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay);
-  exit(EXIT_SUCCESS);
-}
-
-void initLogFile() {
-  // FILE *fp = fopen(g_stConfInfo.resultFileName, "a");
-  char file[256];
-  sprintf(file, "%s/../log/tmqlog.txt", configDir);
-  TdFilePtr pFile = taosOpenFile(file, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM);
-  if (NULL == pFile) {
-    fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt");
-    exit(-1);
-  }
-  g_fp = pFile;
-}
-
-void saveConfigToLogFile() {
-  time_t    tTime = taosGetTimestampSec();
-  struct tm tm = *taosLocalTime(&tTime, NULL);
-
-  taosFprintfFile(g_fp, "###################################################################\n");
-  taosFprintfFile(g_fp, "# configDir:           %s\n", configDir);
-  taosFprintfFile(g_fp, "# dbName:              %s\n", g_stConfInfo.dbName);
-  taosFprintfFile(g_fp, "# cdbName:             %s\n", g_stConfInfo.cdbName);
-  taosFprintfFile(g_fp, "# showMsgFlag:         %d\n", g_stConfInfo.showMsgFlag);
-  taosFprintfFile(g_fp, "# showRowFlag:         %d\n", g_stConfInfo.showRowFlag);
-  taosFprintfFile(g_fp, "# consumeDelay:        %d\n", g_stConfInfo.consumeDelay);
-  taosFprintfFile(g_fp, "# numOfThread:         %d\n", g_stConfInfo.numOfThread);
-
-  for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
-    taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
-    taosFprintfFile(g_fp, "  Topics: ");
-    for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) {
-      taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]);
-    }
-    taosFprintfFile(g_fp, "\n");
-    taosFprintfFile(g_fp, "  Key: ");
-    for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) {
-      taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]);
-    }
-    taosFprintfFile(g_fp, "\n");
-  }
-
-  taosFprintfFile(g_fp, "# Test time:                %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
-                  tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
-  taosFprintfFile(g_fp, "###################################################################\n");
-}
-
-void parseArgument(int32_t argc, char* argv[]) {
-  memset(&g_stConfInfo, 0, sizeof(SConfInfo));
-  g_stConfInfo.showMsgFlag = 0;
-  g_stConfInfo.showRowFlag = 0;
-  g_stConfInfo.consumeDelay = 5;
-
-  for (int32_t i = 1; i < argc; i++) {
-    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
-      printHelp();
-      exit(0);
-    } else if (strcmp(argv[i], "-d") == 0) {
-      strcpy(g_stConfInfo.dbName, argv[++i]);
-    } else if (strcmp(argv[i], "-w") == 0) {
-      strcpy(g_stConfInfo.cdbName, argv[++i]);
-    } else if (strcmp(argv[i], "-c") == 0) {
-      strcpy(configDir, argv[++i]);
-    } else if (strcmp(argv[i], "-g") == 0) {
-      g_stConfInfo.showMsgFlag = atol(argv[++i]);
-    } else if (strcmp(argv[i], "-r") == 0) {
-      g_stConfInfo.showRowFlag = atol(argv[++i]);
-    } else if (strcmp(argv[i], "-y") == 0) {
-      g_stConfInfo.consumeDelay = atol(argv[++i]);
-    } else {
-      printf("%s unknow para: %s %s", GREEN, argv[++i], NC);
-      exit(-1);
-    }
-  }
-
-  initLogFile();
-
-  taosFprintfFile(g_fp, "====parseArgument() success\n");
-
-#if 1
-  pPrint("%s configDir:%s %s", GREEN, configDir, NC);
-  pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC);
-  pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC);
-  pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC);
-  pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC);
-  pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC);
-#endif
-}
-
-void splitStr(char** arr, char* str, const char* del) {
-  char* s = strtok(str, del);
-  while (s != NULL) {
-    *arr++ = s;
-    s = strtok(NULL, del);
-  }
-}
-
-void ltrim(char* str) {
-  if (str == NULL || *str == '\0') {
-    return;
-  }
-  int   len = 0;
-  char* p = str;
-  while (*p != '\0' && isspace(*p)) {
-    ++p;
-    ++len;
-  }
-  memmove(str, p, strlen(str) - len + 1);
-  // return str;
-}
-
-static int     running = 1;
-static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) {
-  char    buf[1024];
-  int32_t totalRows = 0;
-
-  // printf("topic: %s\n", tmq_get_topic_name(msg));
-  // printf("vg:%d\n", tmq_get_vgroup_id(msg));
-  taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable);
-  taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg));
-
-  while (1) {
-    TAOS_ROW row = taos_fetch_row(msg);
-    if (row == NULL) break;
-    if (0 != g_stConfInfo.showRowFlag) {
-      TAOS_FIELD* fields = taos_fetch_fields(msg);
-      int32_t     numOfFields = taos_field_count(msg);
-      taos_print_row(buf, row, fields, numOfFields);
-      taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf);
-    }
-    totalRows++;
-  }
-
-  return totalRows;
-}
-
-int queryDB(TAOS* taos, char* command) {
-  TAOS_RES* pRes = taos_query(taos, command);
-  int       code = taos_errno(pRes);
-  // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
-  if (code != 0) {
-    pError("failed to reason:%s, sql: %s", tstrerror(code), command);
-    taos_free_result(pRes);
-    return -1;
-  }
-  taos_free_result(pRes);
-  return 0;
-}
-
-static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
-  printf("tmq_commit_cb_print() commit %d\n", resp);
-}
-
-void build_consumer(SThreadInfo* pInfo) {
-  tmq_conf_t* conf = tmq_conf_new();
-
-  // tmq_conf_set(conf, "td.connect.ip", "localhost");
-  // tmq_conf_set(conf, "td.connect.port", "6030");
-  tmq_conf_set(conf, "td.connect.user", "root");
-  tmq_conf_set(conf, "td.connect.pass", "taosdata");
-
-  tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
-
-  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
-
-  // tmq_conf_set(conf, "group.id", "cgrp1");
-  for (int32_t i = 0; i < pInfo->numOfKey; i++) {
-    tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]);
-  }
-
-  // tmq_conf_set(conf, "client.id", "c-001");
-
-  // tmq_conf_set(conf, "enable.auto.commit", "true");
-  // tmq_conf_set(conf, "enable.auto.commit", "false");
-
-  // tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
-
-  // tmq_conf_set(conf, "auto.offset.reset", "none");
-  // tmq_conf_set(conf, "auto.offset.reset", "earliest");
-  // tmq_conf_set(conf, "auto.offset.reset", "latest");
-
-  pInfo->tmq = tmq_consumer_new(conf, NULL, 0);
-  return;
-}
-
-void build_topic_list(SThreadInfo* pInfo) {
-  pInfo->topicList = tmq_list_new();
-  // tmq_list_append(topic_list, "test_stb_topic_1");
-  for (int32_t i = 0; i < pInfo->numOfTopic; i++) {
-    tmq_list_append(pInfo->topicList, pInfo->topics[i]);
-  }
-  return;
-}
-
-int32_t saveConsumeResult(SThreadInfo* pInfo) {
-  char sqlStr[1024] = {0};
-
-  TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
-  assert(pConn != NULL);
-
-  // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
-  sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName,
-          pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult);
-
-  TAOS_RES* pRes = taos_query(pConn, sqlStr);
-  if (taos_errno(pRes) != 0) {
-    printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes));
-    taos_free_result(pRes);
-    exit(-1);
-  }
-
-  taos_free_result(pRes);
-
-  return 0;
-}
-
-void loop_consume(SThreadInfo* pInfo) {
-  tmq_resp_err_t err;
-
-  int64_t totalMsgs = 0;
-  int64_t totalRows = 0;
-
-  while (running) {
-    TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000);
-    if (tmqMsg) {
-      if (0 != g_stConfInfo.showMsgFlag) {
-        totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId);
-      }
-
-      taos_free_result(tmqMsg);
-
-      totalMsgs++;
-
-      if (totalMsgs >= pInfo->expectMsgCnt) {
-        break;
-      }
-    } else {
-      break;
-    }
-  }
-
-  pInfo->consumeMsgCnt = totalMsgs;
-  pInfo->consumeRowCnt = totalRows;
-
-  taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n",
-                  pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt);
-}
-
-void* consumeThreadFunc(void* param) {
-  int32_t totalMsgs = 0;
-
-  SThreadInfo* pInfo = (SThreadInfo*)param;
-
-  build_consumer(pInfo);
-  build_topic_list(pInfo);
-  if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) {
-    return NULL;
-  }
-
-  tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList);
-  if (err) {
-    printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
-    exit(-1);
-  }
-
-  loop_consume(pInfo);
-
-  tmq_commit(pInfo->tmq, NULL, 0);
-
-  err = tmq_unsubscribe(pInfo->tmq);
-  if (err) {
-    printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
-    pInfo->consumeMsgCnt = -1;
-    return NULL;
-  }
-
-  err = tmq_consumer_close(pInfo->tmq);
-  if (err) {
-    printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err));
-    exit(-1);
-  }
-  pInfo->tmq = NULL;
-
-  // save consume result into consumeresult table
-  saveConsumeResult(pInfo);
-
-  return NULL;
-}
-
-void parseConsumeInfo() {
-  char*      token;
-  const char delim[2] = ",";
-  const char ch = ':';
-
-  for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
-    token = strtok(g_stConfInfo.stThreads[i].topicString, delim);
-    while (token != NULL) {
-      // printf("%s\n", token );
-      strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token);
-      ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]);
-      // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
-      g_stConfInfo.stThreads[i].numOfTopic++;
-
-      token = strtok(NULL, delim);
-    }
-
-    token = strtok(g_stConfInfo.stThreads[i].keyString, delim);
-    while (token != NULL) {
-      // printf("%s\n", token );
-      {
-        char* pstr = token;
-        ltrim(pstr);
-        char* ret = strchr(pstr, ch);
-        memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr);
-        strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1);
-        // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey],
-        // g_stConfInfo.value[g_stConfInfo.numOfKey]);
-        g_stConfInfo.stThreads[i].numOfKey++;
-      }
-
-      token = strtok(NULL, delim);
-    }
-  }
-}
-
-int32_t getConsumeInfo() {
-  char sqlStr[1024] = {0};
-
-  TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
-  assert(pConn != NULL);
-
-  sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName);
-  TAOS_RES* pRes = taos_query(pConn, sqlStr);
-  if (taos_errno(pRes) != 0) {
-    printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes));
-    taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes));
-    taosCloseFile(&g_fp);
-    taos_free_result(pRes);
-    exit(-1);
-  }
-
-  TAOS_ROW    row = NULL;
-  int         num_fields = taos_num_fields(pRes);
-  TAOS_FIELD* fields = taos_fetch_fields(pRes);
-
-  // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint,
-  // ifcheckdata int
-
-  int32_t numOfThread = 0;
-  while ((row = taos_fetch_row(pRes))) {
-    int32_t* lengths = taos_fetch_lengths(pRes);
-
-    for (int i = 0; i < num_fields; ++i) {
-      if (row[i] == NULL || 0 == i) {
-        continue;
-      }
-
-      if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
-        g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]);
-      } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
-        memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]);
-      } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
-        memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]);
-      } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) {
-        g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]);
-      } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
-        g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]);
-      }
-    }
-    numOfThread++;
-  }
-  g_stConfInfo.numOfThread = numOfThread;
-
-  taos_free_result(pRes);
-
-  parseConsumeInfo();
-
-  return 0;
-}
-
-int main(int32_t argc, char* argv[]) {
-  parseArgument(argc, argv);
-  getConsumeInfo();
-  saveConfigToLogFile();
-
-  TdThreadAttr thattr;
-  taosThreadAttrInit(&thattr);
-  taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
-
-  // pthread_create one thread to consume
-  taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread);
-  for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) {
-    taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc,
-                     (void*)(&(g_stConfInfo.stThreads[i])));
-  }
-
-  for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
-    taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL);
-  }
-
-  // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt);
-
-  taosFprintfFile(g_fp, "==== close tmqlog ====\n");
-  taosCloseFile(&g_fp);
-
-  return 0;
-}
-
+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+
+#include "taos.h"
+#include "taoserror.h"
+#include "tlog.h"
+
+#define GREEN     "\033[1;32m"
+#define NC        "\033[0m"
+#define min(a, b) (((a) < (b)) ? (a) : (b))
+
+#define MAX_SQL_STR_LEN         (1024 * 1024)
+#define MAX_ROW_STR_LEN         (16 * 1024)
+#define MAX_CONSUMER_THREAD_CNT (16)
+
+typedef struct {
+  TdThread thread;
+  int32_t  consumerId;
+
+  int32_t  autoCommitIntervalMs;  // 1000 ms  
+  char     autoCommit[8];         // true, false
+  char     autoOffsetRest[16];    // none, earliest, latest
+
+  int32_t ifCheckData;
+  int64_t expectMsgCnt;
+
+  int64_t consumeMsgCnt;
+  int64_t consumeRowCnt;
+  int32_t checkresult;
+
+  char topicString[1024];
+  char keyString[1024];
+
+  int32_t numOfTopic;
+  char    topics[32][64];
+
+  int32_t numOfKey;
+  char    key[32][64];
+  char    value[32][64];
+
+  tmq_t*      tmq;
+  tmq_list_t* topicList;
+
+} SThreadInfo;
+
+typedef struct {
+  // input from argvs
+  char        cdbName[32];
+  char        dbName[32];
+  int32_t     showMsgFlag;
+  int32_t     showRowFlag;
+  int32_t     consumeDelay;  // unit s
+  int32_t     numOfThread;
+  SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT];
+} SConfInfo;
+
+static SConfInfo g_stConfInfo;
+TdFilePtr        g_fp = NULL;
+
+// char* g_pRowValue = NULL;
+// TdFilePtr g_fp = NULL;
+
+static void printHelp() {
+  char indent[10] = "        ";
+  printf("Used to test the tmq feature with sim cases\n");
+
+  printf("%s%s\n", indent, "-c");
+  printf("%s%s%s%s\n", indent, indent, "Configuration directory, default is ", configDir);
+  printf("%s%s\n", indent, "-d");
+  printf("%s%s%s\n", indent, indent, "The name of the database for cosumer, no default ");
+  printf("%s%s\n", indent, "-g");
+  printf("%s%s%s%d\n", indent, indent, "showMsgFlag, default is ", g_stConfInfo.showMsgFlag);
+  printf("%s%s\n", indent, "-r");
+  printf("%s%s%s%d\n", indent, indent, "showRowFlag, default is ", g_stConfInfo.showRowFlag);
+  printf("%s%s\n", indent, "-y");
+  printf("%s%s%s%d\n", indent, indent, "consume delay, default is s", g_stConfInfo.consumeDelay);
+  exit(EXIT_SUCCESS);
+}
+
+void initLogFile() {
+  // FILE *fp = fopen(g_stConfInfo.resultFileName, "a");
+  char file[256];
+  sprintf(file, "%s/../log/tmqlog.txt", configDir);
+  TdFilePtr pFile = taosOpenFile(file, TD_FILE_TEXT | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM);
+  if (NULL == pFile) {
+    fprintf(stderr, "Failed to open %s for save result\n", "./tmqlog.txt");
+    exit(-1);
+  }
+  g_fp = pFile;
+}
+
+void saveConfigToLogFile() {
+  time_t    tTime = taosGetTimestampSec();
+  struct tm tm = *taosLocalTime(&tTime, NULL);
+
+  taosFprintfFile(g_fp, "###################################################################\n");
+  taosFprintfFile(g_fp, "# configDir:           %s\n", configDir);
+  taosFprintfFile(g_fp, "# dbName:              %s\n", g_stConfInfo.dbName);
+  taosFprintfFile(g_fp, "# cdbName:             %s\n", g_stConfInfo.cdbName);
+  taosFprintfFile(g_fp, "# showMsgFlag:         %d\n", g_stConfInfo.showMsgFlag);
+  taosFprintfFile(g_fp, "# showRowFlag:         %d\n", g_stConfInfo.showRowFlag);
+  taosFprintfFile(g_fp, "# consumeDelay:        %d\n", g_stConfInfo.consumeDelay);
+  taosFprintfFile(g_fp, "# numOfThread:         %d\n", g_stConfInfo.numOfThread);
+
+  for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
+    taosFprintfFile(g_fp, "# consumer %d info:\n", g_stConfInfo.stThreads[i].consumerId);
+	taosFprintfFile(g_fp, "  auto commit:              %s\n", g_stConfInfo.stThreads[i].autoCommit);
+	taosFprintfFile(g_fp, "  auto commit interval ms:  %d\n", g_stConfInfo.stThreads[i].autoCommitIntervalMs);
+	taosFprintfFile(g_fp, "  auto offset rest:         %s\n", g_stConfInfo.stThreads[i].autoOffsetRest);
+    taosFprintfFile(g_fp, "  Topics: ");
+    for (int j = 0; j < g_stConfInfo.stThreads[i].numOfTopic; j++) {
+      taosFprintfFile(g_fp, "%s, ", g_stConfInfo.stThreads[i].topics[j]);
+    }
+    taosFprintfFile(g_fp, "\n");
+    taosFprintfFile(g_fp, "  Key: ");
+    for (int k = 0; k < g_stConfInfo.stThreads[i].numOfKey; k++) {
+      taosFprintfFile(g_fp, "%s:%s, ", g_stConfInfo.stThreads[i].key[k], g_stConfInfo.stThreads[i].value[k]);
+    }
+    taosFprintfFile(g_fp, "\n");
+  }
+
+  taosFprintfFile(g_fp, "# Test time:                %d-%02d-%02d %02d:%02d:%02d\n", tm.tm_year + 1900, tm.tm_mon + 1,
+                  tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
+  taosFprintfFile(g_fp, "###################################################################\n");
+}
+
+void parseArgument(int32_t argc, char* argv[]) {
+  memset(&g_stConfInfo, 0, sizeof(SConfInfo));
+  g_stConfInfo.showMsgFlag = 0;
+  g_stConfInfo.showRowFlag = 0;
+  g_stConfInfo.consumeDelay = 5;
+
+  for (int32_t i = 1; i < argc; i++) {
+    if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+      printHelp();
+      exit(0);
+    } else if (strcmp(argv[i], "-d") == 0) {
+      strcpy(g_stConfInfo.dbName, argv[++i]);
+    } else if (strcmp(argv[i], "-w") == 0) {
+      strcpy(g_stConfInfo.cdbName, argv[++i]);
+    } else if (strcmp(argv[i], "-c") == 0) {
+      strcpy(configDir, argv[++i]);
+    } else if (strcmp(argv[i], "-g") == 0) {
+      g_stConfInfo.showMsgFlag = atol(argv[++i]);
+    } else if (strcmp(argv[i], "-r") == 0) {
+      g_stConfInfo.showRowFlag = atol(argv[++i]);
+    } else if (strcmp(argv[i], "-y") == 0) {
+      g_stConfInfo.consumeDelay = atol(argv[++i]);
+    } else {
+      printf("%s unknow para: %s %s", GREEN, argv[++i], NC);
+      exit(-1);
+    }
+  }
+
+  initLogFile();
+
+  taosFprintfFile(g_fp, "====parseArgument() success\n");
+
+#if 1
+  pPrint("%s configDir:%s %s", GREEN, configDir, NC);
+  pPrint("%s dbName:%s %s", GREEN, g_stConfInfo.dbName, NC);
+  pPrint("%s cdbName:%s %s", GREEN, g_stConfInfo.cdbName, NC);
+  pPrint("%s consumeDelay:%d %s", GREEN, g_stConfInfo.consumeDelay, NC);
+  pPrint("%s showMsgFlag:%d %s", GREEN, g_stConfInfo.showMsgFlag, NC);
+  pPrint("%s showRowFlag:%d %s", GREEN, g_stConfInfo.showRowFlag, NC);
+#endif
+}
+
+void splitStr(char** arr, char* str, const char* del) {
+  char* s = strtok(str, del);
+  while (s != NULL) {
+    *arr++ = s;
+    s = strtok(NULL, del);
+  }
+}
+
+void ltrim(char* str) {
+  if (str == NULL || *str == '\0') {
+    return;
+  }
+  int   len = 0;
+  char* p = str;
+  while (*p != '\0' && isspace(*p)) {
+    ++p;
+    ++len;
+  }
+  memmove(str, p, strlen(str) - len + 1);
+  // return str;
+}
+
+static int     running = 1;
+static int32_t msg_process(TAOS_RES* msg, int64_t msgIndex, int32_t threadLable) {
+  char    buf[1024];
+  int32_t totalRows = 0;
+
+  // printf("topic: %s\n", tmq_get_topic_name(msg));
+  // printf("vg:%d\n", tmq_get_vgroup_id(msg));
+  taosFprintfFile(g_fp, "msg index:%" PRId64 ", threadLable: %d\n", msgIndex, threadLable);
+  taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), tmq_get_vgroup_id(msg));
+
+  while (1) {
+    TAOS_ROW row = taos_fetch_row(msg);
+    if (row == NULL) break;
+    if (0 != g_stConfInfo.showRowFlag) {
+      TAOS_FIELD* fields = taos_fetch_fields(msg);
+      int32_t     numOfFields = taos_field_count(msg);
+      taos_print_row(buf, row, fields, numOfFields);
+      taosFprintfFile(g_fp, "rows[%d]: %s\n", totalRows, buf);
+    }
+    totalRows++;
+  }
+
+  return totalRows;
+}
+
+int queryDB(TAOS* taos, char* command) {
+  TAOS_RES* pRes = taos_query(taos, command);
+  int       code = taos_errno(pRes);
+  // if ((code != 0) && (code != TSDB_CODE_RPC_AUTH_REQUIRED)) {
+  if (code != 0) {
+    pError("failed to reason:%s, sql: %s", tstrerror(code), command);
+    taos_free_result(pRes);
+    return -1;
+  }
+  taos_free_result(pRes);
+  return 0;
+}
+
+static void tmq_commit_cb_print(tmq_t* tmq, tmq_resp_err_t resp, tmq_topic_vgroup_list_t* offsets, void* param) {
+  printf("tmq_commit_cb_print() commit %d\n", resp);
+}
+
+void build_consumer(SThreadInfo* pInfo) {
+  tmq_conf_t* conf = tmq_conf_new();
+
+  // tmq_conf_set(conf, "td.connect.ip", "localhost");
+  // tmq_conf_set(conf, "td.connect.port", "6030");
+  tmq_conf_set(conf, "td.connect.user", "root");
+  tmq_conf_set(conf, "td.connect.pass", "taosdata");
+
+  //tmq_conf_set(conf, "td.connect.db", g_stConfInfo.dbName);
+
+  tmq_conf_set_offset_commit_cb(conf, tmq_commit_cb_print, NULL);
+
+  // tmq_conf_set(conf, "group.id", "cgrp1");
+  for (int32_t i = 0; i < pInfo->numOfKey; i++) {
+    tmq_conf_set(conf, pInfo->key[i], pInfo->value[i]);
+  }
+
+  // tmq_conf_set(conf, "client.id", "c-001");
+
+  // tmq_conf_set(conf, "enable.auto.commit", "true");
+  // tmq_conf_set(conf, "enable.auto.commit", "false");
+
+  // tmq_conf_set(conf, "auto.commit.interval.ms", "1000");
+
+  // tmq_conf_set(conf, "auto.offset.reset", "none");
+  // tmq_conf_set(conf, "auto.offset.reset", "earliest");
+  // tmq_conf_set(conf, "auto.offset.reset", "latest");
+
+  pInfo->tmq = tmq_consumer_new(conf, NULL, 0);
+
+  tmq_conf_destroy(conf);
+  
+  return;
+}
+
+void build_topic_list(SThreadInfo* pInfo) {
+  pInfo->topicList = tmq_list_new();
+  // tmq_list_append(topic_list, "test_stb_topic_1");
+  for (int32_t i = 0; i < pInfo->numOfTopic; i++) {
+    tmq_list_append(pInfo->topicList, pInfo->topics[i]);
+  }
+  return;
+}
+
+int32_t saveConsumeResult(SThreadInfo* pInfo) {
+  char sqlStr[1024] = {0};
+
+  TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  assert(pConn != NULL);
+
+  // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int
+  sprintf(sqlStr, "insert into %s.consumeresult values (now, %d, %" PRId64 ", %" PRId64 ", %d)", g_stConfInfo.cdbName,
+          pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt, pInfo->checkresult);
+
+  TAOS_RES* pRes = taos_query(pConn, sqlStr);
+  if (taos_errno(pRes) != 0) {
+    printf("error in save consumeinfo, reason:%s\n", taos_errstr(pRes));
+    taos_free_result(pRes);
+    exit(-1);
+  }
+
+  taos_free_result(pRes);
+
+  return 0;
+}
+
+void loop_consume(SThreadInfo* pInfo) {
+  tmq_resp_err_t err;
+
+  int64_t totalMsgs = 0;
+  int64_t totalRows = 0;
+
+  while (running) {
+    TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000);
+    if (tmqMsg) {
+      if (0 != g_stConfInfo.showMsgFlag) {
+        totalRows += msg_process(tmqMsg, totalMsgs, pInfo->consumerId);
+      }
+
+      taos_free_result(tmqMsg);
+
+      totalMsgs++;
+
+      if (totalMsgs >= pInfo->expectMsgCnt) {
+	  	taosFprintfFile(g_fp, "==== totalMsgs >= pInfo->expectMsgCnt, so break\n");
+        break;
+      }
+    } else {    
+	  taosFprintfFile(g_fp, "==== delay over time, so break\n");
+      break;
+    }
+  }
+
+  pInfo->consumeMsgCnt = totalMsgs;
+  pInfo->consumeRowCnt = totalRows;
+
+  taosFprintfFile(g_fp, "==== consumerId: %d, consumeMsgCnt: %" PRId64 ", consumeRowCnt: %" PRId64 "\n",
+                  pInfo->consumerId, pInfo->consumeMsgCnt, pInfo->consumeRowCnt);
+}
+
+void* consumeThreadFunc(void* param) {
+  int32_t totalMsgs = 0;
+
+  SThreadInfo* pInfo = (SThreadInfo*)param;
+
+  build_consumer(pInfo);
+  build_topic_list(pInfo);
+  if ((NULL == pInfo->tmq) || (NULL == pInfo->topicList)) {
+    return NULL;
+  }
+
+  tmq_resp_err_t err = tmq_subscribe(pInfo->tmq, pInfo->topicList);
+  if (err) {
+    printf("tmq_subscribe() fail, reason: %s\n", tmq_err2str(err));
+    exit(-1);
+  }
+  
+  tmq_list_destroy(pInfo->topicList);
+  pInfo->topicList = NULL;
+
+  loop_consume(pInfo);
+
+  tmq_commit(pInfo->tmq, NULL, 0);
+
+  err = tmq_unsubscribe(pInfo->tmq);
+  if (err) {
+    printf("tmq_unsubscribe() fail, reason: %s\n", tmq_err2str(err));
+    pInfo->consumeMsgCnt = -1;
+    return NULL;
+  }
+  
+  err = tmq_consumer_close(pInfo->tmq);
+  if (err) {
+    printf("tmq_consumer_close() fail, reason: %s\n", tmq_err2str(err));
+    exit(-1);
+  }
+  pInfo->tmq = NULL;
+
+  // save consume result into consumeresult table
+  saveConsumeResult(pInfo);
+
+  return NULL;
+}
+
+void parseConsumeInfo() {
+  char*      token;
+  const char delim[2] = ",";
+  const char ch = ':';
+
+  for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
+    token = strtok(g_stConfInfo.stThreads[i].topicString, delim);
+    while (token != NULL) {
+      // printf("%s\n", token );
+      strcpy(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic], token);
+      ltrim(g_stConfInfo.stThreads[i].topics[g_stConfInfo.stThreads[i].numOfTopic]);
+      // printf("%s\n", g_stConfInfo.topics[g_stConfInfo.numOfTopic]);
+      g_stConfInfo.stThreads[i].numOfTopic++;
+
+      token = strtok(NULL, delim);
+    }
+
+    token = strtok(g_stConfInfo.stThreads[i].keyString, delim);
+    while (token != NULL) {
+      // printf("%s\n", token );
+      {
+        char* pstr = token;
+        ltrim(pstr);
+        char* ret = strchr(pstr, ch);
+        memcpy(g_stConfInfo.stThreads[i].key[g_stConfInfo.stThreads[i].numOfKey], pstr, ret - pstr);
+        strcpy(g_stConfInfo.stThreads[i].value[g_stConfInfo.stThreads[i].numOfKey], ret + 1);
+        // printf("key: %s, value: %s\n", g_stConfInfo.key[g_stConfInfo.numOfKey],
+        // g_stConfInfo.value[g_stConfInfo.numOfKey]);
+        g_stConfInfo.stThreads[i].numOfKey++;
+      }
+
+      token = strtok(NULL, delim);
+    }
+  }
+}
+
+int32_t getConsumeInfo() {
+  char sqlStr[1024] = {0};
+
+  TAOS* pConn = taos_connect(NULL, "root", "taosdata", NULL, 0);
+  assert(pConn != NULL);
+
+  sprintf(sqlStr, "select * from %s.consumeinfo", g_stConfInfo.cdbName);
+  TAOS_RES* pRes = taos_query(pConn, sqlStr);
+  if (taos_errno(pRes) != 0) {
+    printf("error in get consumeinfo, reason:%s\n", taos_errstr(pRes));
+    taosFprintfFile(g_fp, "error in get consumeinfo, reason:%s\n", taos_errstr(pRes));
+    taosCloseFile(&g_fp);
+    taos_free_result(pRes);
+    exit(-1);
+  }
+
+  TAOS_ROW    row = NULL;
+  int         num_fields = taos_num_fields(pRes);
+  TAOS_FIELD* fields = taos_fetch_fields(pRes);
+
+  // schema: ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint,
+  // ifcheckdata int
+
+  int32_t numOfThread = 0;
+  while ((row = taos_fetch_row(pRes))) {
+    int32_t* lengths = taos_fetch_lengths(pRes);
+
+    // set default value
+    g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = 5000;
+    memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, "true", strlen("true"));
+    memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, "earlieast", strlen("earlieast"));  
+
+    for (int i = 0; i < num_fields; ++i) {
+      if (row[i] == NULL || 0 == i) {
+        continue;
+      }
+
+      if ((1 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
+        g_stConfInfo.stThreads[numOfThread].consumerId = *((int32_t*)row[i]);
+      } else if ((2 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
+        memcpy(g_stConfInfo.stThreads[numOfThread].topicString, row[i], lengths[i]);
+      } else if ((3 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
+        memcpy(g_stConfInfo.stThreads[numOfThread].keyString, row[i], lengths[i]);
+      } else if ((4 == i) && (fields[i].type == TSDB_DATA_TYPE_BIGINT)) {
+        g_stConfInfo.stThreads[numOfThread].expectMsgCnt = *((int64_t*)row[i]);
+      } else if ((5 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
+        g_stConfInfo.stThreads[numOfThread].ifCheckData = *((int32_t*)row[i]);
+      } else if ((6 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {		
+        memcpy(g_stConfInfo.stThreads[numOfThread].autoCommit, row[i], lengths[i]);
+      } else if ((7 == i) && (fields[i].type == TSDB_DATA_TYPE_INT)) {
+        g_stConfInfo.stThreads[numOfThread].autoCommitIntervalMs = *((int32_t*)row[i]);
+      } else if ((8 == i) && (fields[i].type == TSDB_DATA_TYPE_BINARY)) {
+        memcpy(g_stConfInfo.stThreads[numOfThread].autoOffsetRest, row[i], lengths[i]);
+      }
+    }
+    numOfThread++;
+  }
+  g_stConfInfo.numOfThread = numOfThread;
+
+  taos_free_result(pRes);
+
+  parseConsumeInfo();
+
+  return 0;
+}
+
+int main(int32_t argc, char* argv[]) {
+  parseArgument(argc, argv);
+  getConsumeInfo();
+  saveConfigToLogFile();
+
+  TdThreadAttr thattr;
+  taosThreadAttrInit(&thattr);
+  taosThreadAttrSetDetachState(&thattr, PTHREAD_CREATE_JOINABLE);
+
+  // pthread_create one thread to consume
+  taosFprintfFile(g_fp, "==== create %d consume thread ====\n", g_stConfInfo.numOfThread);
+  for (int32_t i = 0; i < g_stConfInfo.numOfThread; ++i) {
+    taosThreadCreate(&(g_stConfInfo.stThreads[i].thread), &thattr, consumeThreadFunc,
+                     (void*)(&(g_stConfInfo.stThreads[i])));
+  }
+
+  for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) {
+    taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL);
+  }
+
+  // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt);
+
+  taosFprintfFile(g_fp, "==== close tmqlog ====\n");
+  taosCloseFile(&g_fp);
+
+  return 0;
+}
+

From 3309dd71db1f3152e64eb83095b1a6a737484147 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 12 May 2022 15:36:30 +0800
Subject: [PATCH 20/35] test: add test case for tmq

---
 tests/system-test/7-tmq/basic5.py | 143 +++++++++++++++++++-----------
 1 file changed, 91 insertions(+), 52 deletions(-)

diff --git a/tests/system-test/7-tmq/basic5.py b/tests/system-test/7-tmq/basic5.py
index 7ed9f7ebe7..267b6ff5d8 100644
--- a/tests/system-test/7-tmq/basic5.py
+++ b/tests/system-test/7-tmq/basic5.py
@@ -13,14 +13,12 @@ from util.dnodes import *
 
 class TDTestCase:
     hostname = socket.gethostname()
-    rpcDebugFlagVal = '143'
-    clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
-    clientCfgDict["rpcDebugFlag"]  = rpcDebugFlagVal
-
-    updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
-    updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
-
-    print ("===================: ", updatecfgDict)
+    #rpcDebugFlagVal = '143'
+    #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
+    #clientCfgDict["rpcDebugFlag"]  = rpcDebugFlagVal
+    #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
+    #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
+    #print ("===================: ", updatecfgDict)
 
     def init(self, conn, logSql):
         tdLog.debug(f"start to excute {__file__}")
@@ -43,27 +41,35 @@ class TDTestCase:
                     break
         return buildPath
 
-    def create_tables(self,dbName,vgroups,stbName,ctbNum,rowsPerTbl):
-        tdSql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))        
-        tdSql.execute("use %s" %dbName)
-        tdSql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
+    def newcur(self,cfg,host,port):
+        user = "root"
+        password = "taosdata"
+        con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
+        cur=con.cursor()
+        print(cur)
+        return cur
+
+    def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl):
+        tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))        
+        tsql.execute("use %s" %dbName)
+        tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
         pre_create = "create table"
         sql = pre_create
         #tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
         for i in range(ctbNum):
             sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1)
             if (i > 0) and (i%100 == 0):
-                tdSql.execute(sql)
+                tsql.execute(sql)
                 sql = pre_create
         if sql != pre_create:
-            tdSql.execute(sql)
+            tsql.execute(sql)
             
         tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum))
         return
 
-    def insert_data(self,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
+    def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
         tdLog.debug("start to insert data ............")
-        tdSql.execute("use %s" %dbName)
+        tsql.execute("use %s" %dbName)
         pre_insert = "insert into "
         sql = pre_insert
 
@@ -73,7 +79,7 @@ class TDTestCase:
             for j in range(rowsPerTbl):
                 sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
                 if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
-                    tdSql.execute(sql)
+                    tsql.execute(sql)
                     if j < rowsPerTbl - 1:
                         sql = "insert into %s_%d values " %(stbName,i)
                     else:
@@ -81,25 +87,29 @@ class TDTestCase:
         #end sql
         if sql != pre_insert:
             #print("insert sql:%s"%sql)
-            tdSql.execute(sql)
+            tsql.execute(sql)
         tdLog.debug("insert data ............ [OK]")
         return
         
     def prepareEnv(self, **parameterDict):
         print ("input parameters:")
         print (parameterDict)
-        self.create_tables(parameterDict["dbName"],\
+        # create new connector for my thread
+        tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030)
+        self.create_tables(tsql,\
+                           parameterDict["dbName"],\
                            parameterDict["vgroups"],\
                            parameterDict["stbName"],\
                            parameterDict["ctbNum"],\
                            parameterDict["rowsPerTbl"])
 
-        self.insert_data(parameterDict["dbName"],\
-                           parameterDict["stbName"],\
-                           parameterDict["ctbNum"],\
-                           parameterDict["rowsPerTbl"],\
-                           parameterDict["batchNum"],\
-                           parameterDict["startTs"])                           
+        self.insert_data(tsql,\
+                         parameterDict["dbName"],\
+                         parameterDict["stbName"],\
+                         parameterDict["ctbNum"],\
+                         parameterDict["rowsPerTbl"],\
+                         parameterDict["batchNum"],\
+                         parameterDict["startTs"])                           
         return                   
         
     def run(self):
@@ -116,24 +126,29 @@ class TDTestCase:
         tdLog.printNoPrefix("======== test scenario 1: ")
         tdLog.info("step 1: create database, stb, ctb and insert data")
         # create and start thread
-        parameterDict = {'dbName':     'db',     \
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db',     \
                          'vgroups':    1,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 10000,    \
+                         'rowsPerTbl': 10000,   \
                          'batchNum':   10,       \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
         prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
         prepareEnvThread.start()
-        time.sleep(1)
+        time.sleep(2)
         
         # wait stb ready
         while 1:
-            tdSql.query("show %s.stables"%parameterDict['dbName'])
-            if tdSql.getRows() == 1:
-            #if (self.queryRows == 1):
-                time.sleep(1)
+            #tdSql.query("show %s.stables"%parameterDict['dbName'])
+            tdSql.query("show db.stables")
+            #print (self.queryResult)
+            #print (tdSql.getRows())
+            if tdSql.getRows() == 1:            
                 break
+            else:
+                time.sleep(1)
 
         tdLog.info("create topics from super table")
         topicFromStb = 'topic_stb_column'
@@ -141,57 +156,81 @@ class TDTestCase:
         
         tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
         tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName']))
-
+        
+        time.sleep(1)
         tdSql.query("show topics")
-        tdSql.checkRows(2)
+        print ("======================================")
+        #print (self.queryResult)
+        #tdSql.checkRows(2)
         topic1 = tdSql.getData(0 , 0)
         topic2 = tdSql.getData(1 , 0)
-        if topic1 != topicFromStb or topic1 != topicFromCtb:
-            tdLog.exit("topic error") 
-        if topic2 != topicFromStb or topic2 != topicFromCtb:
-            tdLog.exit("topic error") 
+        print (topic1)
+        print (topic2)
+        
+        print (topicFromStb)
+        print (topicFromCtb)
+        #tdLog.info("show topics: %s, %s"%topic1, topic2)
+        #if topic1 != topicFromStb or topic1 != topicFromCtb:
+        #    tdLog.exit("topic error1") 
+        #if topic2 != topicFromStb or topic2 != topicFromCtb:
+        #    tdLog.exit("topic error2") 
          
         tdLog.info("create consume info table and consume result table")
-        cdbName = 'cdb'
-        tdSql.query("create database %s"%cdbName)
+        cdbName = parameterDict["dbName"]
+        #tdSql.query("create database %s"%cdbName)
+        #tdSql.query("use %s"%cdbName)
         tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)")
         tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)")
 
         consumerId   = 0
-        expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] + 1) * parameterDict["ctbNum"]
+        expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"]
+        expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"]
         topicList    = topicFromStb
         ifcheckdata  = 0
-        keyList      = 'group.id:cgrp1,               \
-                        enable.auto.commit:false,     \
-                        auto.commit.interval.ms:6000, \
-                        auto.offset.reset:none'
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
         sql = "insert into consumeinfo values "
-        sql += "(now, %d, '%s', '%s', %l64d, %d)"%(consumerId, topicList, keyList, expectmsgcnt, ifcheckdata)
+        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata)
         tdSql.query(sql)
         
+        tdLog.info("check stb if there are data")
+        while 1:
+            tdSql.query("select count(*) from %s"%parameterDict["stbName"])
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            countOfStb = tdSql.getData(0, 0)
+            if countOfStb != 0:
+                tdLog.info("count from stb: %d"%countOfStb)
+                break
+            else:
+                time.sleep(1)
+        
         tdLog.info("start consume processor")
         pollDelay = 5
         showMsg   = 1
         showRow   = 1
         
         shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
-        shellCmd += " -y %d -d %s, -g %d, -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) 
+        shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) 
         shellCmd += "> /dev/null 2>&1 &"        
         tdLog.info(shellCmd)
-        os.system(taosCmd)        
+        os.system(shellCmd)        
 
         # wait for data ready
         prepareEnvThread.join()
         
-        tdLog.info("check consume result")
+        tdLog.info("insert process end, and start to check consume result")
         while 1:
             tdSql.query("select * from consumeresult")
             #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
             if tdSql.getRows() == 1:
-            #if (self.queryRows == 1):
-                time.sleep(1)
                 break
-            
+            else:
+                time.sleep(5)
+
+        expectrowcnt = parameterDict["rowsPerTbl"]  * parameterDict["ctbNum"]
+                    
         tdSql.checkData(0 , 1, consumerId)
         tdSql.checkData(0 , 2, expectmsgcnt)
         tdSql.checkData(0 , 3, expectrowcnt)

From c95d3cdb36e18e5a33cb18f8e0bce460793b5a33 Mon Sep 17 00:00:00 2001
From: "slzhou@taodata.com" <shenglian zhou>
Date: Thu, 12 May 2022 15:43:32 +0800
Subject: [PATCH 21/35] feature(udf):error follows tsdb_code standard

---
 include/libs/function/tudf.h    |  9 ---------
 include/util/taoserror.h        | 10 +++++++++-
 source/libs/function/src/tudf.c | 14 +++++++-------
 source/libs/function/src/udfd.c |  2 +-
 source/util/src/terror.c        |  8 ++++++++
 5 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h
index 8e156edcd2..b5c38e14f4 100644
--- a/include/libs/function/tudf.h
+++ b/include/libs/function/tudf.h
@@ -39,15 +39,6 @@ extern "C" {
 //======================================================================================
 //begin API to taosd and qworker
 
-enum {
-  UDFC_CODE_STOPPING = -1,
-  UDFC_CODE_PIPE_READ_ERR = -2,
-  UDFC_CODE_CONNECT_PIPE_ERR = -3,
-  UDFC_CODE_LOAD_UDF_FAILURE = -4,
-  UDFC_CODE_INVALID_STATE = -5,
-  UDFC_CODE_NO_PIPE = -6,
-};
-
 typedef void *UdfcFuncHandle;
 
 /**
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 52b2f0c670..58cdd2cab4 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -645,7 +645,15 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_FUNC_FUNTION_PARA_NUM         TAOS_DEF_ERROR_CODE(0, 0x2801)
 #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE        TAOS_DEF_ERROR_CODE(0, 0x2802)
 #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE       TAOS_DEF_ERROR_CODE(0, 0x2803)
-#define TSDB_CODE_FUNC_INVALID_FUNTION          TAOS_DEF_ERROR_CODE(0, 0x2604)
+#define TSDB_CODE_FUNC_INVALID_FUNTION          TAOS_DEF_ERROR_CODE(0, 0x2804)
+
+//udf
+#define TSDB_CODE_UDF_STOPPING                  TAOS_DEF_ERROR_CODE(0, 0x2901)
+#define TSDB_CODE_UDF_PIPE_READ_ERR             TAOS_DEF_ERROR_CODE(0, 0x2902)
+#define TSDB_CODE_UDF_PIPE_CONNECT_ERR          TAOS_DEF_ERROR_CODE(0, 0x2903)
+#define TSDB_CODE_UDF_PIPE_NO_PIPE              TAOS_DEF_ERROR_CODE(0, 0x2904)
+#define TSDB_CODE_UDF_LOAD_UDF_FAILURE          TAOS_DEF_ERROR_CODE(0, 0x2905)
+#define TSDB_CODE_UDF_INVALID_STATE             TAOS_DEF_ERROR_CODE(0, 0x2906)
 
 #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE     TAOS_DEF_ERROR_CODE(0, 0x3000)
 #define TSDB_CODE_SML_INVALID_PRECISION_TYPE    TAOS_DEF_ERROR_CODE(0, 0x3001)
diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index 9d90dca63e..dfa7fac15a 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -931,7 +931,7 @@ void udfcUvHandleError(SClientUvConn *conn) {
   while (!QUEUE_EMPTY(&conn->taskQueue)) {
     QUEUE* h = QUEUE_HEAD(&conn->taskQueue);
     SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, connTaskQueue);
-    task->errCode = UDFC_CODE_PIPE_READ_ERR;
+    task->errCode = TSDB_CODE_UDF_PIPE_READ_ERR;
     QUEUE_REMOVE(&task->connTaskQueue);
     QUEUE_REMOVE(&task->procTaskQueue);
     uv_sem_post(&task->taskSem);
@@ -1119,7 +1119,7 @@ void cleanUpUvTasks(SUdfdProxy *udfc) {
     QUEUE_REMOVE(h);
     SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, recvTaskQueue);
     if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
-      task->errCode = UDFC_CODE_STOPPING;
+      task->errCode = TSDB_CODE_UDF_STOPPING;
     }
     uv_sem_post(&task->taskSem);
   }
@@ -1129,7 +1129,7 @@ void cleanUpUvTasks(SUdfdProxy *udfc) {
     QUEUE_REMOVE(h);
     SClientUvTaskNode *task = QUEUE_DATA(h, SClientUvTaskNode, procTaskQueue);
     if (udfc->gUdfcState == UDFC_STATE_STOPPING) {
-      task->errCode = UDFC_CODE_STOPPING;
+      task->errCode = TSDB_CODE_UDF_STOPPING;
     }
     uv_sem_post(&task->taskSem);
   }
@@ -1213,7 +1213,7 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
 int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
   fnInfo("udfc setup udf. udfName: %s", udfName);
   if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) {
-    return UDFC_CODE_INVALID_STATE;
+    return TSDB_CODE_UDF_INVALID_STATE;
   }
   SClientUdfTask *task = taosMemoryCalloc(1,sizeof(SClientUdfTask));
   task->errCode = 0;
@@ -1227,7 +1227,7 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
   int32_t errCode = udfcRunUdfUvTask(task, UV_TASK_CONNECT);
   if (errCode != 0) {
     fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName);
-    return UDFC_CODE_CONNECT_PIPE_ERR;
+    return TSDB_CODE_UDF_PIPE_CONNECT_ERR;
   }
 
   udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
@@ -1254,7 +1254,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
   SClientUdfUvSession *session = (SClientUdfUvSession *) handle;
   if (session->udfUvPipe == NULL) {
     fnError("No pipe to udfd");
-    return UDFC_CODE_NO_PIPE;
+    return TSDB_CODE_UDF_PIPE_NO_PIPE;
   }
   SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
   task->errCode = 0;
@@ -1374,7 +1374,7 @@ int32_t teardownUdf(UdfcFuncHandle handle) {
   SClientUdfUvSession *session = (SClientUdfUvSession *) handle;
   if (session->udfUvPipe == NULL) {
     fnError("pipe to udfd does not exist");
-    return UDFC_CODE_NO_PIPE;
+    return TSDB_CODE_UDF_PIPE_NO_PIPE;
   }
 
   SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index 0ad4674cfa..f006695f14 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -102,7 +102,7 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
   int err = uv_dlopen(udf->path, &udf->lib);
   if (err != 0) {
     fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
-    return UDFC_CODE_LOAD_UDF_FAILURE;
+    return TSDB_CODE_UDF_LOAD_UDF_FAILURE;
   }
 
   char initFuncName[TSDB_FUNC_NAME_LEN+5] = {0};
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index e32ecfc695..9676e4e1f9 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -454,6 +454,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_PAR_PERMISSION_DENIED,        "Permission denied")
 //planner
 TAOS_DEFINE_ERROR(TSDB_CODE_PLAN_INTERNAL_ERROR,          "planner internal error")
 
+//udf
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING,                 "udf is stopping")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR,            "udf pipe read error")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR,         "udf pipe connect error")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE,             "udf no pipe")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE,         "udf load failure")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE,            "udf invalid state")
+
 //schemaless
 TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE,    "Invalid line protocol type")
 TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PRECISION_TYPE,   "Invalid timestamp precision type")

From bc4cd5ef3da28effd8ceec8b025292a8ba2970d1 Mon Sep 17 00:00:00 2001
From: jiajingbin <jiajaybin@126.com>
Date: Thu, 12 May 2022 16:21:31 +0800
Subject: [PATCH 22/35] test: add ts filter testcases

---
 .../2-query/query_cols_tags_and_or.py         | 118 ++++++++++--------
 1 file changed, 69 insertions(+), 49 deletions(-)

diff --git a/tests/system-test/2-query/query_cols_tags_and_or.py b/tests/system-test/2-query/query_cols_tags_and_or.py
index a62960cf43..77e91aa983 100644
--- a/tests/system-test/2-query/query_cols_tags_and_or.py
+++ b/tests/system-test/2-query/query_cols_tags_and_or.py
@@ -93,104 +93,124 @@ class TDTestCase:
             res = tdSql.query(query_sql.replace('*', 'last(*)'), True)
         return int(res[0][-4])
 
-    def queryTsCol(self, tb_name):
+    def queryTsCol(self, tb_name, check_elm=None):
+        select_elm = "*" if check_elm is None else check_elm
         # ts and ts
-        query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"'
+        query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"'
         tdSql.query(query_sql)
         tdSql.checkRows(11)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" and ts <= "2021-01-13 12:00:00"'
+        query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" and ts <= "2021-01-13 12:00:00"'
         tdSql.query(query_sql)
-        # tdSql.checkRows(2)
-        # tdSql.checkEqual(self.queryLastC10(query_sql), 6)
+        tdSql.checkRows(2)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 6) if select_elm == "*" else False
 
         ## ts or and tinyint col
-        query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or c1 = 2'
-        tdSql.error(query_sql)
+        query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-11 12:00:00" or c1 = 2'
+        tdSql.query(query_sql)
+        tdSql.checkRows(7)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts <= "2021-01-11 12:00:00" and c1 != 2'
+        query_sql = f'select {select_elm} from {tb_name} where ts <= "2021-01-11 12:00:00" and c1 != 2'
         tdSql.query(query_sql)
         tdSql.checkRows(4)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 5)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 5) if select_elm == "*" else False
 
         ## ts or and smallint col
-        query_sql = f'select * from {tb_name} where ts <> "2021-01-11 12:00:00" or c2 = 10'
-        tdSql.error(query_sql)
+        query_sql = f'select {select_elm} from {tb_name} where ts <> "2021-01-11 12:00:00" or c2 = 10'
+        tdSql.query(query_sql)
+        tdSql.checkRows(10)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts <= "2021-01-11 12:00:00" and c2 <= 1'
+
+        query_sql = f'select {select_elm} from {tb_name} where ts <= "2021-01-11 12:00:00" and c2 <= 1'
         tdSql.query(query_sql)
         tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 1)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 1) if select_elm == "*" else False
 
         ## ts or and int col
-        query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" or c3 = 4'
-        tdSql.error(query_sql)
+        query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" or c3 = 4'
+        tdSql.query(query_sql)
+        tdSql.checkRows(8)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" and c3 = 4'
+        query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" and c3 = 4'
         tdSql.query(query_sql)
         tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 4)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 4) if select_elm == "*" else False
 
         ## ts or and big col
-        query_sql = f'select * from {tb_name} where ts is Null or c4 = 5'
-        tdSql.error(query_sql)
-
-        query_sql = f'select * from {tb_name} where ts is not Null and c4 = 2'
+        query_sql = f'select {select_elm} from {tb_name} where ts is Null or c4 = 5'
         tdSql.query(query_sql)
         tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 3)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 5) if select_elm == "*" else False
+
+        query_sql = f'select {select_elm} from {tb_name} where ts is not Null and c4 = 2'
+        tdSql.query(query_sql)
+        tdSql.checkRows(1)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 3) if select_elm == "*" else False
 
         ## ts or and float col
-        query_sql = f'select * from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c5 = 6.6'
-        tdSql.error(query_sql)
+        query_sql = f'select {select_elm} from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c5 = 6.6'
+        tdSql.query(query_sql)
+        tdSql.checkRows(5)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" and c5 = 1.1'
+        query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" and c5 = 1.1'
         tdSql.query(query_sql)
         tdSql.checkRows(4)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 4)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 4) if select_elm == "*" else False
 
         ## ts or and double col
-        query_sql = f'select * from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c6 = 7.7'
-        tdSql.error(query_sql) 
+        query_sql = f'select {select_elm} from {tb_name} where ts between "2021-01-17 12:00:00" and "2021-01-23 12:00:00" or c6 = 7.7'
+        tdSql.query(query_sql)
+        tdSql.checkRows(5)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" and c6 = 1.1'
+        query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" and c6 = 1.1'
         tdSql.query(query_sql)
         tdSql.checkRows(4)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 4)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 4) if select_elm == "*" else False
 
         ## ts or and binary col
-        query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" or c7 like "binary_"'
-        tdSql.error(query_sql)
-
-        query_sql = f'select * from {tb_name} where ts <= "2021-01-11 12:00:00" and c7 in ("binary")'
+        query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" or c7 like "binary_"'
         tdSql.query(query_sql)
         tdSql.checkRows(5)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 5)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 8) if select_elm == "*" else False
+
+        query_sql = f'select {select_elm} from {tb_name} where ts <= "2021-01-11 12:00:00" and c7 in ("binary")'
+        tdSql.query(query_sql)
+        tdSql.checkRows(5)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 5) if select_elm == "*" else False
 
         ## ts or and nchar col
-        query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" or c8 like "nchar%"'
-        tdSql.error(query_sql)
+        query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" or c8 like "nchar%"'
+        tdSql.query(query_sql)
+        tdSql.checkRows(10)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 10) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" and c8 is Null'
+        query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" and c8 is Null'
         tdSql.query(query_sql)
         tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 11)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
         ## ts or and bool col
-        query_sql = f'select * from {tb_name} where ts < "2021-01-11 12:00:00" or c9=false'
-        tdSql.error(query_sql)
+        query_sql = f'select {select_elm} from {tb_name} where ts < "2021-01-11 12:00:00" or c9=false'
+        tdSql.query(query_sql)
+        tdSql.checkRows(6)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 11) if select_elm == "*" else False
 
-        query_sql = f'select * from {tb_name} where ts >= "2021-01-11 12:00:00" and c9=true'
+        query_sql = f'select {select_elm} from {tb_name} where ts >= "2021-01-11 12:00:00" and c9=true'
         tdSql.query(query_sql)
         tdSql.checkRows(5)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 9)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 9) if select_elm == "*" else False
 
         ## multi cols
-        query_sql = f'select * from {tb_name} where ts > "2021-01-03 12:00:00" and c1 != 2 and c2 >= 2 and c3 <> 4 and c4 < 4 and c5 > 1 and c6 >= 1.1 and c7 is not Null and c8 = "nchar" and c9=false'
+        query_sql = f'select {select_elm} from {tb_name} where ts > "2021-01-03 12:00:00" and c1 != 2 and c2 >= 2 and c3 <> 4 and c4 < 4 and c5 > 1 and c6 >= 1.1 and c7 is not Null and c8 = "nchar" and c9=false'
         tdSql.query(query_sql)
         tdSql.checkRows(1)
-        tdSql.checkEqual(self.queryLastC10(query_sql), 10)
+        tdSql.checkEqual(self.queryLastC10(query_sql), 10) if select_elm == "*" else False
 
     def queryTsTag(self, tb_name):
         ## ts and tinyint col
@@ -2029,12 +2049,12 @@ class TDTestCase:
         tb_name = self.initStb()
         self.queryFullTagType(tb_name)
 
-    def checkTbTsCol(self):
+    def checkTbTsCol(self, check_elm):
         '''
             Ordinary table ts and col check
         '''
         tb_name = self.initTb()
-        self.queryTsCol(tb_name)
+        self.queryTsCol(tb_name, check_elm)
 
     def checkStbTsTol(self):
         tb_name = self.initStb()
@@ -2112,8 +2132,8 @@ class TDTestCase:
         for check_elm in [None, column_name]:
             self.checkTbColTypeOperator(check_elm)
             self.checkStbColTypeOperator(check_elm)
+            self.checkTbTsCol(check_elm)
         # self.checkStbTagTypeOperator()
-        # self.checkTbTsCol()
         # self.checkStbTsTol()
         # self.checkStbTsTag()
         # self.checkStbTsColTag()

From 265f2a858252c69f7a258e7ec018c25c767347b1 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 12 May 2022 16:36:08 +0800
Subject: [PATCH 23/35] test: add get rows api in test frame

---
 tests/pytest/util/sql.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py
index 19a5d3ecc1..adbab04e07 100644
--- a/tests/pytest/util/sql.py
+++ b/tests/pytest/util/sql.py
@@ -150,6 +150,9 @@ class TDSql:
             raise Exception(repr(e))
         return (self.queryRows, timeout)
 
+    def getRows(self):
+        return self.queryRows
+
     def checkRows(self, expectRows):
         if self.queryRows == expectRows:
             tdLog.info("sql:%s, queryRows:%d == expect:%d" % (self.sql, self.queryRows, expectRows))

From 89271550133c6e0ae50feade6c686ffa200b8033 Mon Sep 17 00:00:00 2001
From: "slzhou@taodata.com" <shenglian zhou>
Date: Thu, 12 May 2022 16:49:31 +0800
Subject: [PATCH 24/35] feat:add udf dedicated errors

---
 include/util/taoserror.h         |   1 +
 source/common/src/tglobal.c      |   4 +-
 source/libs/function/src/tudf.c  |   2 +-
 source/libs/function/src/udfd.c  | 338 ++++++++++++++++---------------
 source/libs/function/test/udf2.c |   6 +
 source/util/src/terror.c         |   1 +
 6 files changed, 190 insertions(+), 162 deletions(-)

diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 58cdd2cab4..238600160a 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -654,6 +654,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_UDF_PIPE_NO_PIPE              TAOS_DEF_ERROR_CODE(0, 0x2904)
 #define TSDB_CODE_UDF_LOAD_UDF_FAILURE          TAOS_DEF_ERROR_CODE(0, 0x2905)
 #define TSDB_CODE_UDF_INVALID_STATE             TAOS_DEF_ERROR_CODE(0, 0x2906)
+#define TSDB_CODE_UDF_INVALID_INPUT             TAOS_DEF_ERROR_CODE(0, 0x2907)
 
 #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE     TAOS_DEF_ERROR_CODE(0, 0x3000)
 #define TSDB_CODE_SML_INVALID_PRECISION_TYPE    TAOS_DEF_ERROR_CODE(0, 0x3001)
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index f73a982110..42c8e18f3e 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -444,7 +444,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
   if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1;
   if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1;
 
-  if (cfgAddBool(pCfg, "startUdfd", tsStartUdfd, 0) != 0) return -1;
+  if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1;
   return 0;
 }
 
@@ -585,7 +585,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
   tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
   tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32;
 
-  tsStartUdfd = cfgGetItem(pCfg, "startUdfd")->bval;
+  tsStartUdfd = cfgGetItem(pCfg, "udf")->bval;
 
   if (tsQueryBufferSize >= 0) {
     tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL;
diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c
index dfa7fac15a..8e96a2a063 100644
--- a/source/libs/function/src/tudf.c
+++ b/source/libs/function/src/tudf.c
@@ -1497,7 +1497,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
   taosArrayDestroy(tempBlock.pDataBlock);
 
   taosMemoryFree(newState.buf);
-  return TSDB_CODE_SUCCESS;
+  return udfCode;
 }
 
 int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock) {
diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c
index f006695f14..34681dc6cd 100644
--- a/source/libs/function/src/udfd.c
+++ b/source/libs/function/src/udfd.c
@@ -140,6 +140,182 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
   return 0;
 }
 
+void udfdProcessSetupRequest(SUvUdfWork* uvUdf, SUdfRequest* request) {
+    // TODO: tracable id from client. connect, setup, call, teardown
+    fnInfo("%" PRId64 " setup request. udf name: %s", request->seqNum, request->setup.udfName);
+    SUdfSetupRequest *setup = &request->setup;
+    int32_t code = TSDB_CODE_SUCCESS;
+    SUdf *udf = NULL;
+    uv_mutex_lock(&global.udfsMutex);
+    SUdf **udfInHash = taosHashGet(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName));
+    if (udfInHash) {
+      ++(*udfInHash)->refCount;
+      udf = *udfInHash;
+      uv_mutex_unlock(&global.udfsMutex);
+    } else {
+      SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf));
+      udfNew->refCount = 1;
+      udfNew->state = UDF_STATE_INIT;
+
+      uv_mutex_init(&udfNew->lock);
+      uv_cond_init(&udfNew->condReady);
+      udf = udfNew;
+      taosHashPut(global.udfsHash, request->setup.udfName, strlen(request->setup.udfName), &udfNew, sizeof(&udfNew));
+      uv_mutex_unlock(&global.udfsMutex);
+    }
+
+    uv_mutex_lock(&udf->lock);
+    if (udf->state == UDF_STATE_INIT) {
+      udf->state = UDF_STATE_LOADING;
+      code = udfdLoadUdf(setup->udfName, udf);
+      if (udf->initFunc) {
+        udf->initFunc();
+      }
+      udf->state = UDF_STATE_READY;
+      uv_cond_broadcast(&udf->condReady);
+      uv_mutex_unlock(&udf->lock);
+    } else {
+      while (udf->state != UDF_STATE_READY) {
+        uv_cond_wait(&udf->condReady, &udf->lock);
+      }
+      uv_mutex_unlock(&udf->lock);
+    }
+    SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle));
+    handle->udf = udf;
+
+    SUdfResponse rsp;
+    rsp.seqNum = request->seqNum;
+    rsp.type = request->type;
+    rsp.code = code;
+    rsp.setupRsp.udfHandle = (int64_t)(handle);
+    rsp.setupRsp.outputType = udf->outputType;
+    rsp.setupRsp.outputLen = udf->outputLen;
+    rsp.setupRsp.bufSize = udf->bufSize;
+
+    int32_t len = encodeUdfResponse(NULL, &rsp);
+    rsp.msgLen = len;
+    void *bufBegin = taosMemoryMalloc(len);
+    void *buf = bufBegin;
+    encodeUdfResponse(&buf, &rsp);
+
+    uvUdf->output = uv_buf_init(bufBegin, len);
+
+    taosMemoryFree(uvUdf->input.base);
+    return;
+}
+
+void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
+  SUdfCallRequest *call = &request->call;
+  fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request->seqNum, call->callType,
+          call->udfHandle);
+  SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle);
+  SUdf            *udf = handle->udf;
+  SUdfResponse  response = {0};
+  SUdfResponse *rsp = &response;
+  SUdfCallResponse *subRsp = &rsp->callRsp;
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  switch(call->callType) {
+    case TSDB_UDF_CALL_SCALA_PROC: {
+      SUdfColumn output = {0};
+
+      SUdfDataBlock input = {0};
+      convertDataBlockToUdfDataBlock(&call->block, &input);
+      code = udf->scalarProcFunc(&input, &output);
+
+      convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
+      freeUdfColumn(&output);
+      break;
+    }
+    case TSDB_UDF_CALL_AGG_INIT: {
+      SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
+                             .bufLen= udf->bufSize,
+                             .numOfResult = 0};
+      udf->aggStartFunc(&outBuf);
+      subRsp->resultBuf = outBuf;
+      break;
+    }
+    case TSDB_UDF_CALL_AGG_PROC: {
+      SUdfDataBlock input = {0};
+      convertDataBlockToUdfDataBlock(&call->block, &input);
+      SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
+                             .bufLen= udf->bufSize,
+                             .numOfResult = 0};
+      code = udf->aggProcFunc(&input, &call->interBuf, &outBuf);
+      subRsp->resultBuf = outBuf;
+
+      break;
+    }
+    case TSDB_UDF_CALL_AGG_FIN: {
+      SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
+                             .bufLen= udf->bufSize,
+                             .numOfResult = 0};
+      code = udf->aggFinishFunc(&call->interBuf, &outBuf);
+      subRsp->resultBuf = outBuf;
+      break;
+    }
+    default:
+      break;
+  }
+
+  rsp->seqNum = request->seqNum;
+  rsp->type = request->type;
+  rsp->code = code;
+  subRsp->callType = call->callType;
+
+  int32_t len = encodeUdfResponse(NULL, rsp);
+  rsp->msgLen = len;
+  void *bufBegin = taosMemoryMalloc(len);
+  void *buf = bufBegin;
+  encodeUdfResponse(&buf, rsp);
+  uvUdf->output = uv_buf_init(bufBegin, len);
+
+  taosMemoryFree(uvUdf->input.base);
+  return;
+}
+
+void udfdProcessTeardownRequest(SUvUdfWork* uvUdf, SUdfRequest* request) {
+  SUdfTeardownRequest *teardown = &request->teardown;
+  fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request->seqNum, teardown->udfHandle);
+  SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(teardown->udfHandle);
+  SUdf *udf = handle->udf;
+  bool  unloadUdf = false;
+  int32_t code = TSDB_CODE_SUCCESS;
+
+  uv_mutex_lock(&global.udfsMutex);
+  udf->refCount--;
+  if (udf->refCount == 0) {
+    unloadUdf = true;
+    taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
+  }
+  uv_mutex_unlock(&global.udfsMutex);
+  if (unloadUdf) {
+    uv_cond_destroy(&udf->condReady);
+    uv_mutex_destroy(&udf->lock);
+    if (udf->destroyFunc) {
+      (udf->destroyFunc)();
+    }
+    uv_dlclose(&udf->lib);
+    taosMemoryFree(udf);
+  }
+  taosMemoryFree(handle);
+
+  SUdfResponse  response;
+  SUdfResponse *rsp = &response;
+  rsp->seqNum = request->seqNum;
+  rsp->type = request->type;
+  rsp->code = code;
+  int32_t len = encodeUdfResponse(NULL, rsp);
+  rsp->msgLen = len;
+  void *bufBegin = taosMemoryMalloc(len);
+  void *buf = bufBegin;
+  encodeUdfResponse(&buf, rsp);
+  uvUdf->output = uv_buf_init(bufBegin, len);
+
+  taosMemoryFree(uvUdf->input.base);
+  return;
+}
+
 void udfdProcessRequest(uv_work_t *req) {
   SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
   SUdfRequest request = {0};
@@ -147,172 +323,16 @@ void udfdProcessRequest(uv_work_t *req) {
 
   switch (request.type) {
     case UDF_TASK_SETUP: {
-      // TODO: tracable id from client. connect, setup, call, teardown
-      fnInfo("%" PRId64 " setup request. udf name: %s", request.seqNum, request.setup.udfName);
-      SUdfSetupRequest *setup = &request.setup;
-
-      SUdf *udf = NULL;
-      uv_mutex_lock(&global.udfsMutex);
-      SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName));
-      if (udfInHash) {
-        ++(*udfInHash)->refCount;
-        udf = *udfInHash;
-        uv_mutex_unlock(&global.udfsMutex);
-      } else {
-        SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf));
-        udfNew->refCount = 1;
-        udfNew->state = UDF_STATE_INIT;
-
-        uv_mutex_init(&udfNew->lock);
-        uv_cond_init(&udfNew->condReady);
-        udf = udfNew;
-        taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew));
-        uv_mutex_unlock(&global.udfsMutex);
-      }
-
-      uv_mutex_lock(&udf->lock);
-      if (udf->state == UDF_STATE_INIT) {
-        udf->state = UDF_STATE_LOADING;
-        udfdLoadUdf(setup->udfName, udf);
-        if (udf->initFunc) {
-          udf->initFunc();
-        }
-        udf->state = UDF_STATE_READY;
-        uv_cond_broadcast(&udf->condReady);
-        uv_mutex_unlock(&udf->lock);
-      } else {
-        while (udf->state != UDF_STATE_READY) {
-          uv_cond_wait(&udf->condReady, &udf->lock);
-        }
-        uv_mutex_unlock(&udf->lock);
-      }
-      SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle));
-      handle->udf = udf;
-      SUdfResponse rsp;
-      rsp.seqNum = request.seqNum;
-      rsp.type = request.type;
-      rsp.code = 0;
-      rsp.setupRsp.udfHandle = (int64_t)(handle);
-      rsp.setupRsp.outputType = udf->outputType;
-      rsp.setupRsp.outputLen = udf->outputLen;
-      rsp.setupRsp.bufSize = udf->bufSize;
-      int32_t len = encodeUdfResponse(NULL, &rsp);
-      rsp.msgLen = len;
-      void *bufBegin = taosMemoryMalloc(len);
-      void *buf = bufBegin;
-      encodeUdfResponse(&buf, &rsp);
-
-      uvUdf->output = uv_buf_init(bufBegin, len);
-
-      taosMemoryFree(uvUdf->input.base);
+      udfdProcessSetupRequest(uvUdf, &request);
       break;
     }
 
     case UDF_TASK_CALL: {
-      SUdfCallRequest *call = &request.call;
-      fnDebug("%" PRId64 "call request. call type %d, handle: %" PRIx64, request.seqNum, call->callType,
-              call->udfHandle);
-      SUdfcFuncHandle *handle = (SUdfcFuncHandle *)(call->udfHandle);
-      SUdf            *udf = handle->udf;
-      SUdfResponse  response = {0};
-      SUdfResponse *rsp = &response;
-      SUdfCallResponse *subRsp = &rsp->callRsp;
-
-      switch(call->callType) {
-        case TSDB_UDF_CALL_SCALA_PROC: {
-          SUdfColumn output = {0};
-
-          SUdfDataBlock input = {0};
-          convertDataBlockToUdfDataBlock(&call->block, &input);
-          udf->scalarProcFunc(&input, &output);
-
-          convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
-          freeUdfColumn(&output);
-          break;
-        }
-        case TSDB_UDF_CALL_AGG_INIT: {
-          SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
-                                 .bufLen= udf->bufSize,
-                                 .numOfResult = 0};
-          udf->aggStartFunc(&outBuf);
-          subRsp->resultBuf = outBuf;
-          break;
-        }
-        case TSDB_UDF_CALL_AGG_PROC: {
-          SUdfDataBlock input = {0};
-          convertDataBlockToUdfDataBlock(&call->block, &input);
-          SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
-                                 .bufLen= udf->bufSize,
-                                 .numOfResult = 0};
-          udf->aggProcFunc(&input, &call->interBuf, &outBuf);
-          subRsp->resultBuf = outBuf;
-
-          break;
-        }
-        case TSDB_UDF_CALL_AGG_FIN: {
-          SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize),
-                                 .bufLen= udf->bufSize,
-                                 .numOfResult = 0};
-          udf->aggFinishFunc(&call->interBuf, &outBuf);
-          subRsp->resultBuf = outBuf;
-          break;
-        }
-        default:
-          break;
-      }
-
-      rsp->seqNum = request.seqNum;
-      rsp->type = request.type;
-      rsp->code = 0;
-      subRsp->callType = call->callType;
-
-      int32_t len = encodeUdfResponse(NULL, rsp);
-      rsp->msgLen = len;
-      void *bufBegin = taosMemoryMalloc(len);
-      void *buf = bufBegin;
-      encodeUdfResponse(&buf, rsp);
-      uvUdf->output = uv_buf_init(bufBegin, len);
-
-      taosMemoryFree(uvUdf->input.base);
+      udfdProcessCallRequest(uvUdf, &request);
       break;
     }
     case UDF_TASK_TEARDOWN: {
-      SUdfTeardownRequest *teardown = &request.teardown;
-      fnInfo("teardown. %" PRId64 "handle:%" PRIx64, request.seqNum, teardown->udfHandle) SUdfcFuncHandle *handle =
-          (SUdfcFuncHandle *)(teardown->udfHandle);
-      SUdf *udf = handle->udf;
-      bool  unloadUdf = false;
-      uv_mutex_lock(&global.udfsMutex);
-      udf->refCount--;
-      if (udf->refCount == 0) {
-        unloadUdf = true;
-        taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
-      }
-      uv_mutex_unlock(&global.udfsMutex);
-      if (unloadUdf) {
-        uv_cond_destroy(&udf->condReady);
-        uv_mutex_destroy(&udf->lock);
-        if (udf->destroyFunc) {
-          (udf->destroyFunc)();
-        }
-        uv_dlclose(&udf->lib);
-        taosMemoryFree(udf);
-      }
-      taosMemoryFree(handle);
-
-      SUdfResponse  response;
-      SUdfResponse *rsp = &response;
-      rsp->seqNum = request.seqNum;
-      rsp->type = request.type;
-      rsp->code = 0;
-      int32_t len = encodeUdfResponse(NULL, rsp);
-      rsp->msgLen = len;
-      void *bufBegin = taosMemoryMalloc(len);
-      void *buf = bufBegin;
-      encodeUdfResponse(&buf, rsp);
-      uvUdf->output = uv_buf_init(bufBegin, len);
-
-      taosMemoryFree(uvUdf->input.base);
+      udfdProcessTeardownRequest(uvUdf, &request);
       break;
     }
     default: {
diff --git a/source/libs/function/test/udf2.c b/source/libs/function/test/udf2.c
index be485bc905..b3b60f93a4 100644
--- a/source/libs/function/test/udf2.c
+++ b/source/libs/function/test/udf2.c
@@ -27,6 +27,12 @@ int32_t udf2_start(SUdfInterBuf *buf) {
 int32_t udf2(SUdfDataBlock* block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) {
   int64_t sumSquares = *(int64_t*)interBuf->buf;
   int8_t numOutput = 0;
+  for (int32_t i = 0; i < block->numOfCols; ++i) {
+    SUdfColumn* col = block->udfCols[i];
+    if (col->colMeta.type != TSDB_DATA_TYPE_INT) {
+      return TSDB_CODE_UDF_INVALID_INPUT;
+    }
+  }
   for (int32_t i = 0; i < block->numOfCols; ++i) {
     for (int32_t j = 0; j < block->numOfRows; ++j) {
       SUdfColumn* col = block->udfCols[i];
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 9676e4e1f9..a1bc37e6cd 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -461,6 +461,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR,         "udf pipe connect erro
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE,             "udf no pipe")
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE,         "udf load failure")
 TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE,            "udf invalid state")
+TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT,            "udf invalid function input")
 
 //schemaless
 TAOS_DEFINE_ERROR(TSDB_CODE_SML_INVALID_PROTOCOL_TYPE,    "Invalid line protocol type")

From 2218213134003ff021647c5cb664f3a004fe4acc Mon Sep 17 00:00:00 2001
From: Liu Jicong <liujicong@qq.com>
Date: Thu, 12 May 2022 17:01:33 +0800
Subject: [PATCH 25/35] enh(wal): add log

---
 source/libs/wal/src/walRead.c | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 64e6881cd0..b04dea0213 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -65,11 +65,18 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
   ret = taosLSeekFile(pIdxTFile, offset, SEEK_SET);
   if (ret < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
+    wError("failed to seek idx file, ver %ld, pos: %ld, since %s", ver, offset, terrstr());
     return -1;
   }
   SWalIdxEntry entry;
-  if (taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
-    terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+  if ((ret = taosReadFile(pIdxTFile, &entry, sizeof(SWalIdxEntry))) != sizeof(SWalIdxEntry)) {
+    if (ret < 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      wError("failed to read idx file, since %s", terrstr());
+    } else {
+      terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
+      wError("read idx file incompletely, read bytes %d, bytes should be %lu", ret, sizeof(SWalIdxEntry));
+    }
     return -1;
   }
 
@@ -77,6 +84,7 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
   ret = taosLSeekFile(pLogTFile, entry.offset, SEEK_SET);
   if (ret < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
+    wError("failed to seek log file, ver %ld, pos: %ld, since %s", ver, entry.offset, terrstr());
     return -1;
   }
   return ret;
@@ -92,6 +100,8 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
   TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ);
   if (pLogTFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
+    terrno = TSDB_CODE_WAL_INVALID_VER;
+    wError("cannot open file %s, since %s", fnameStr, terrstr());
     return -1;
   }
 
@@ -99,6 +109,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
   TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ);
   if (pIdxTFile == NULL) {
     terrno = TAOS_SYSTEM_ERROR(errno);
+    wError("cannot open file %s, since %s", fnameStr, terrstr());
     return -1;
   }
 
@@ -113,6 +124,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
     return 0;
   }
   if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
+    wError("invalid version: % " PRId64 ", first ver %ld, last ver %ld", ver, pWal->vers.firstVer, pWal->vers.lastVer);
     terrno = TSDB_CODE_WAL_INVALID_VER;
     return -1;
   }
@@ -125,12 +137,13 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
   SWalFileInfo *pRet = taosArraySearch(pWal->fileInfoSet, &tmpInfo, compareWalFileInfo, TD_LE);
   ASSERT(pRet != NULL);
   if (pRead->curFileFirstVer != pRet->firstVer) {
+    // error code set inner
     if (walReadChangeFile(pRead, pRet->firstVer) < 0) {
       return -1;
     }
   }
 
-  // code set inner
+  // error code set inner
   if (walReadSeekFilePos(pRead, pRet->firstVer, ver) < 0) {
     return -1;
   }
@@ -248,8 +261,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
   // TODO: check wal life
   if (pRead->curVersion != ver) {
     if (walReadSeekVer(pRead, ver) < 0) {
-      terrno = TSDB_CODE_WAL_INVALID_VER;
-      wError("unexpected wal log version: % " PRId64 ", since seek error", ver);
+      wError("unexpected wal log version: % " PRId64 ", since %s", ver, terrstr());
       return -1;
     }
   }

From f19272909c5af46e16675cc719bbd8a85a775f4e Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Thu, 12 May 2022 17:17:02 +0800
Subject: [PATCH 26/35] fix: some problems of parser

---
 include/libs/nodes/querynodes.h               |  2 +-
 include/util/taoserror.h                      |  1 +
 source/libs/parser/inc/parUtil.h              |  1 +
 source/libs/parser/src/parAstCreater.c        |  2 +-
 source/libs/parser/src/parInsert.c            | 29 ++++++++-------
 source/libs/parser/src/parInsertData.c        |  4 +--
 source/libs/parser/src/parTranslater.c        | 35 +++++++++++++------
 source/libs/parser/src/parUtil.c              | 21 ++++++++++-
 source/libs/planner/src/planOptimizer.c       | 20 ++++++++---
 source/libs/planner/src/planUtil.c            |  2 +-
 source/libs/planner/test/planOptimizeTest.cpp |  2 ++
 11 files changed, 87 insertions(+), 32 deletions(-)

diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h
index c0e2ec91e0..d07f29c487 100644
--- a/include/libs/nodes/querynodes.h
+++ b/include/libs/nodes/querynodes.h
@@ -232,9 +232,9 @@ typedef struct SSelectStmt {
   char        stmtName[TSDB_TABLE_NAME_LEN];
   uint8_t     precision;
   bool        isEmptyResult;
+  bool        isTimeOrderQuery;
   bool        hasAggFuncs;
   bool        hasRepeatScanFuncs;
-  bool        isTimeOrderQuery;
 } SSelectStmt;
 
 typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
diff --git a/include/util/taoserror.h b/include/util/taoserror.h
index 52b2f0c670..51dc14111c 100644
--- a/include/util/taoserror.h
+++ b/include/util/taoserror.h
@@ -634,6 +634,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_PAR_INVALID_TAGS_NUM          TAOS_DEF_ERROR_CODE(0, 0x2643)
 #define TSDB_CODE_PAR_PERMISSION_DENIED         TAOS_DEF_ERROR_CODE(0, 0x2644)
 #define TSDB_CODE_PAR_INVALID_STREAM_QUERY      TAOS_DEF_ERROR_CODE(0, 0x2645)
+#define TSDB_CODE_PAR_INVALID_INTERNAL_PK       TAOS_DEF_ERROR_CODE(0, 0x2646)
 
 //planner
 #define TSDB_CODE_PLAN_INTERNAL_ERROR           TAOS_DEF_ERROR_CODE(0, 0x2700)
diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h
index c146d42e05..f82d29d27e 100644
--- a/source/libs/parser/inc/parUtil.h
+++ b/source/libs/parser/inc/parUtil.h
@@ -46,6 +46,7 @@ SSchema*      getTableTagSchema(const STableMeta* pTableMeta);
 int32_t       getNumOfColumns(const STableMeta* pTableMeta);
 int32_t       getNumOfTags(const STableMeta* pTableMeta);
 STableComInfo getTableInfo(const STableMeta* pTableMeta);
+STableMeta*   tableMetaDup(const STableMeta* pTableMeta);
 int           parseJsontoTagData(const char* json, SKVRowBuilder* kvRowBuilder, SMsgBuf* errMsg, int16_t startColId);
 
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen);
diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c
index b7a14a81c6..74a023f0cd 100644
--- a/source/libs/parser/src/parAstCreater.c
+++ b/source/libs/parser/src/parAstCreater.c
@@ -78,7 +78,7 @@ static bool checkUserName(SAstCreateContext* pCxt, SToken* pUserName) {
 static bool checkPassword(SAstCreateContext* pCxt, const SToken* pPasswordToken, char* pPassword) {
   if (NULL == pPasswordToken) {
     pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
-  } else if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN - 2)) {
+  } else if (pPasswordToken->n >= (TSDB_USET_PASSWORD_LEN + 2)) {
     pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NAME_OR_PASSWD_TOO_LONG);
   } else {
     strncpy(pPassword, pPasswordToken->z, pPasswordToken->n);
diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index 3d069257c9..dfc56eff39 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -237,7 +237,7 @@ static int32_t createSName(SName* pName, SToken* pTableName, int32_t acctId, con
   return code;
 }
 
-static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char *dbFname, bool isStb) {
+static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char* dbFname, bool isStb) {
   SParseContext* pBasicCtx = pCxt->pComCxt;
 
   bool pass = false;
@@ -252,6 +252,7 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char *db
   } else {
     CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, name,
                                    &pCxt->pTableMeta));
+    ASSERT(pCxt->pTableMeta->tableInfo.rowSize > 0);
     SVgroupInfo vg;
     CHECK_CODE(
         catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, name, &vg));
@@ -260,9 +261,13 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SName* name, char *db
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t getTableMeta(SInsertParseContext* pCxt, SName* name, char *dbFname) { return getTableMetaImpl(pCxt, name, dbFname, false); }
+static int32_t getTableMeta(SInsertParseContext* pCxt, SName* name, char* dbFname) {
+  return getTableMetaImpl(pCxt, name, dbFname, false);
+}
 
-static int32_t getSTableMeta(SInsertParseContext* pCxt, SName* name, char *dbFname) { return getTableMetaImpl(pCxt, name, dbFname, true); }
+static int32_t getSTableMeta(SInsertParseContext* pCxt, SName* name, char* dbFname) {
+  return getTableMetaImpl(pCxt, name, dbFname, true);
+}
 
 static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pSchema) {
   while (start < end) {
@@ -863,7 +868,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SName* name, char* tb
   createSName(&sname, &sToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
   char stbFName[TSDB_TABLE_FNAME_LEN];
   tNameExtractFullName(&sname, stbFName);
-  
+
   CHECK_CODE(getSTableMeta(pCxt, &sname, stbFName));
   if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) {
     return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed");
@@ -1063,8 +1068,8 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) {
 //   [...];
 static int32_t parseInsertBody(SInsertParseContext* pCxt) {
   int32_t tbNum = 0;
-  char tbFName[TSDB_TABLE_FNAME_LEN];
-  bool autoCreateTbl = false;
+  char    tbFName[TSDB_TABLE_FNAME_LEN];
+  bool    autoCreateTbl = false;
 
   // for each table
   while (1) {
@@ -1158,7 +1163,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
       return TSDB_CODE_TSC_OUT_OF_MEMORY;
     }
     memcpy(tags, &pCxt->tags, sizeof(pCxt->tags));
-    (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl, pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj);
+    (*pCxt->pStmtCb->setInfoFn)(pCxt->pStmtCb->pStmt, pCxt->pTableMeta, tags, tbFName, autoCreateTbl,
+                                pCxt->pVgroupsHashObj, pCxt->pTableBlockHashObj);
 
     memset(&pCxt->tags, 0, sizeof(pCxt->tags));
     pCxt->pVgroupsHashObj = NULL;
@@ -1479,7 +1485,6 @@ int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBu
       taosMemoryFree(pSTSchema);
     }
 #endif
-
   }
 
   if (rowEnd) {
@@ -1677,10 +1682,10 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsFormat, SArray* cols
   buildCreateTbReq(&smlHandle->createTblReq, tableName, row, pTableMeta->suid);
 
   STableDataBlocks* pDataBlock = NULL;
-  ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid), TSDB_DEFAULT_PAYLOAD_SIZE,
-                                  sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize, pTableMeta,
-                                  &pDataBlock, NULL, &smlHandle->createTblReq);
-  if(ret != TSDB_CODE_SUCCESS){
+  ret = getDataBlockFromList(smlHandle->pBlockHash, &pTableMeta->uid, sizeof(pTableMeta->uid),
+                             TSDB_DEFAULT_PAYLOAD_SIZE, sizeof(SSubmitBlk), getTableInfo(pTableMeta).rowSize,
+                             pTableMeta, &pDataBlock, NULL, &smlHandle->createTblReq);
+  if (ret != TSDB_CODE_SUCCESS) {
     buildInvalidOperationMsg(&pBuf, "create data block error");
     return ret;
   }
diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c
index 8deaad6091..677dbca0e9 100644
--- a/source/libs/parser/src/parInsertData.c
+++ b/source/libs/parser/src/parInsertData.c
@@ -137,7 +137,7 @@ static int32_t createDataBlock(size_t defaultSize, int32_t rowSize, int32_t star
   }
   memset(dataBuf->pData, 0, sizeof(SSubmitBlk));
 
-  dataBuf->pTableMeta = pTableMeta;
+  dataBuf->pTableMeta = tableMetaDup(pTableMeta);
 
   SParsedDataColInfo* pColInfo = &dataBuf->boundColumnInfo;
   SSchema*            pSchema = getTableColumnSchema(dataBuf->pTableMeta);
@@ -465,7 +465,7 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p
         taosMemoryFreeClear(blkKeyInfo.pKeyTuple);
         return ret;
       }
-
+      ASSERT(pOneTableBlock->pTableMeta->tableInfo.rowSize > 0);
       // the maximum expanded size in byte when a row-wise data is converted to SDataRow format
       int32_t expandSize = isRawPayload ? getRowExpandSize(pOneTableBlock->pTableMeta) : 0;
       int64_t destSize = dataBuf->size + pOneTableBlock->size + pBlocks->numOfRows * expandSize +
diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c
index 3fed68188d..c86c1ac2e9 100644
--- a/source/libs/parser/src/parTranslater.c
+++ b/source/libs/parser/src/parTranslater.c
@@ -368,11 +368,15 @@ static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* p
   return TSDB_CODE_SUCCESS;
 }
 
+static bool isInternalPrimaryKey(const SColumnNode* pCol) {
+  return PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && 0 == strcmp(pCol->colName, PK_TS_COL_INTERNAL_NAME);
+}
+
 static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) {
   bool found = false;
   if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) {
     const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta;
-    if (PRIMARYKEY_TIMESTAMP_COL_ID == pCol->colId && 0 == strcmp(pCol->colName, PK_TS_COL_INTERNAL_NAME)) {
+    if (isInternalPrimaryKey(pCol)) {
       setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema, false, pCol);
       return true;
     }
@@ -389,7 +393,9 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) {
     SNode*     pNode;
     FOREACH(pNode, pProjectList) {
       SExprNode* pExpr = (SExprNode*)pNode;
-      if (0 == strcmp(pCol->colName, pExpr->aliasName)) {
+      if (0 == strcmp(pCol->colName, pExpr->aliasName) ||
+          ((QUERY_NODE_COLUMN == nodeType(pExpr) && PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pExpr)->colId) &&
+           isInternalPrimaryKey(pCol))) {
         setColumnInfoByExpr(pTable, pExpr, pCol);
         found = true;
         break;
@@ -433,7 +439,11 @@ static EDealRes translateColumnWithoutPrefix(STranslateContext* pCxt, SColumnNod
     }
   }
   if (!found) {
-    return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName);
+    if (isInternalPrimaryKey(pCol)) {
+      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_INTERNAL_PK);
+    } else {
+      return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, pCol->colName);
+    }
   }
   return DEAL_RES_CONTINUE;
 }
@@ -3655,6 +3665,9 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt*
     destroyCreateTbReq(&req);
     return TSDB_CODE_OUT_OF_MEMORY;
   }
+  if (pStmt->ignoreExists) {
+    req.flags |= TD_CREATE_IF_NOT_EXISTS;
+  }
   SNode*   pCol;
   col_id_t index = 0;
   FOREACH(pCol, pStmt->pCols) {
@@ -3785,24 +3798,27 @@ static int32_t rewriteCreateTable(STranslateContext* pCxt, SQuery* pQuery) {
   return code;
 }
 
-static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, const char* pDbName,
-                                     const char* pTableName, SKVRow row, uint64_t suid, SVgroupInfo* pVgInfo) {
+static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, SCreateSubTableClause* pStmt, SKVRow row,
+                                     uint64_t suid, SVgroupInfo* pVgInfo) {
   char  dbFName[TSDB_DB_FNAME_LEN] = {0};
   SName name = {.type = TSDB_DB_NAME_T, .acctId = acctId};
-  strcpy(name.dbname, pDbName);
+  strcpy(name.dbname, pStmt->dbName);
   tNameGetFullDbName(&name, dbFName);
 
   struct SVCreateTbReq req = {0};
   req.type = TD_CHILD_TABLE;
-  req.name = strdup(pTableName);
+  req.name = strdup(pStmt->tableName);
   req.ctb.suid = suid;
   req.ctb.pTag = row;
+  if (pStmt->ignoreExists) {
+    req.flags |= TD_CREATE_IF_NOT_EXISTS;
+  }
 
   SVgroupCreateTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
   if (pTableBatch == NULL) {
     SVgroupCreateTableBatch tBatch = {0};
     tBatch.info = *pVgInfo;
-    strcpy(tBatch.dbName, pDbName);
+    strcpy(tBatch.dbName, pStmt->dbName);
 
     tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
     taosArrayPush(tBatch.req.pArray, &req);
@@ -3964,8 +3980,7 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla
     code = getTableHashVgroup(pCxt, pStmt->dbName, pStmt->tableName, &info);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt->dbName, pStmt->tableName, row,
-                             pSuperTableMeta->uid, &info);
+    addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, row, pSuperTableMeta->uid, &info);
   }
 
   taosMemoryFreeClear(pSuperTableMeta);
diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c
index 715b9b97e6..43aea8de7c 100644
--- a/source/libs/parser/src/parUtil.c
+++ b/source/libs/parser/src/parUtil.c
@@ -146,6 +146,8 @@ static char* getSyntaxErrFormat(int32_t errCode) {
       return "Invalid binary/nchar column length";
     case TSDB_CODE_PAR_INVALID_TAGS_NUM:
       return "Invalid number of tag columns";
+    case TSDB_CODE_PAR_INVALID_INTERNAL_PK:
+      return "Invalid _c0 or _rowts expression";
     case TSDB_CODE_OUT_OF_MEMORY:
       return "Out of memory";
     default:
@@ -191,7 +193,7 @@ int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char*
   return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
 }
 
-SSchema *getTableColumnSchema(const STableMeta *pTableMeta) {
+SSchema* getTableColumnSchema(const STableMeta* pTableMeta) {
   assert(pTableMeta != NULL);
   return (SSchema*)pTableMeta->schema;
 }
@@ -226,6 +228,23 @@ STableComInfo getTableInfo(const STableMeta* pTableMeta) {
   return pTableMeta->tableInfo;
 }
 
+static uint32_t getTableMetaSize(const STableMeta* pTableMeta) {
+  int32_t totalCols = 0;
+  if (pTableMeta->tableInfo.numOfColumns >= 0) {
+    totalCols = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags;
+  }
+
+  return sizeof(STableMeta) + totalCols * sizeof(SSchema);
+}
+
+STableMeta* tableMetaDup(const STableMeta* pTableMeta) {
+  size_t size = getTableMetaSize(pTableMeta);
+
+  STableMeta* p = taosMemoryMalloc(size);
+  memcpy(p, pTableMeta, size);
+  return p;
+}
+
 int32_t trimString(const char* src, int32_t len, char* dst, int32_t dlen) {
   if (len <= 0 || dlen <= 0) return 0;
 
diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c
index 30f6f03a6c..5ed7d9c1b5 100644
--- a/source/libs/planner/src/planOptimizer.c
+++ b/source/libs/planner/src/planOptimizer.c
@@ -151,20 +151,32 @@ static bool needOptimizeDynamicScan(const SFunctionNode* pFunc) {
 
 static int32_t osdGetRelatedFuncs(SScanLogicNode* pScan, SNodeList** pSdrFuncs, SNodeList** pDsoFuncs) {
   SNodeList* pAllFuncs = osdGetAllFuncs(pScan->node.pParent);
+  SNodeList* pTmpSdrFuncs = NULL;
+  SNodeList* pTmpDsoFuncs = NULL;
   SNode*     pFunc = NULL;
+  bool       otherFunc = false;
   FOREACH(pFunc, pAllFuncs) {
     int32_t code = TSDB_CODE_SUCCESS;
     if (needOptimizeDataRequire((SFunctionNode*)pFunc)) {
-      code = nodesListMakeStrictAppend(pSdrFuncs, nodesCloneNode(pFunc));
+      code = nodesListMakeStrictAppend(&pTmpSdrFuncs, nodesCloneNode(pFunc));
     } else if (needOptimizeDynamicScan((SFunctionNode*)pFunc)) {
-      code = nodesListMakeStrictAppend(pDsoFuncs, nodesCloneNode(pFunc));
+      code = nodesListMakeStrictAppend(&pTmpDsoFuncs, nodesCloneNode(pFunc));
+    } else {
+      otherFunc = true;
     }
     if (TSDB_CODE_SUCCESS != code) {
-      nodesDestroyList(*pSdrFuncs);
-      nodesDestroyList(*pDsoFuncs);
+      nodesDestroyList(pTmpSdrFuncs);
+      nodesDestroyList(pTmpDsoFuncs);
       return code;
     }
   }
+  if (otherFunc) {
+    nodesDestroyList(pTmpSdrFuncs);
+    nodesDestroyList(pTmpDsoFuncs);
+  } else {
+    *pSdrFuncs = pTmpSdrFuncs;
+    *pDsoFuncs = pTmpDsoFuncs;
+  }
   return TSDB_CODE_SUCCESS;
 }
 
diff --git a/source/libs/planner/src/planUtil.c b/source/libs/planner/src/planUtil.c
index 36625b28fb..3c83d9f53a 100644
--- a/source/libs/planner/src/planUtil.c
+++ b/source/libs/planner/src/planUtil.c
@@ -18,7 +18,7 @@
 static char* getUsageErrFormat(int32_t errCode) {
   switch (errCode) {
     case TSDB_CODE_PLAN_EXPECTED_TS_EQUAL:
-      return "l.ts = r.ts is expected in join expression";
+      return "left.ts = right.ts is expected in join expression";
     case TSDB_CODE_PLAN_NOT_SUPPORT_CROSS_JOIN:
       return "not support cross join";
     default:
diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp
index 6c7b1d0a0e..77f9b5846c 100644
--- a/source/libs/planner/test/planOptimizeTest.cpp
+++ b/source/libs/planner/test/planOptimizeTest.cpp
@@ -28,6 +28,8 @@ TEST_F(PlanOptimizeTest, optimizeScanData) {
   run("SELECT COUNT(c1) FROM t1");
 
   run("SELECT COUNT(CAST(c1 AS BIGINT)) FROM t1");
+
+  run("SELECT PERCENTILE(c1, 40), COUNT(*) FROM t1");
 }
 
 TEST_F(PlanOptimizeTest, orderByPrimaryKey) {

From 2d89555b724646ef4a944a6bc1a5b3e65fa13a84 Mon Sep 17 00:00:00 2001
From: yihaoDeng <luomoxyz@126.com>
Date: Thu, 12 May 2022 17:38:16 +0800
Subject: [PATCH 27/35] fix(rpc): avoid fd leak

---
 source/libs/qcom/src/queryUtil.c | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c
index 3e3e393f5f..4b4c079649 100644
--- a/source/libs/qcom/src/queryUtil.c
+++ b/source/libs/qcom/src/queryUtil.c
@@ -153,11 +153,6 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra
                     .handle = pInfo->msgInfo.handle,
                     .persistHandle = persistHandle,
                     .code = 0};
-  if (pInfo->msgType == TDMT_VND_QUERY || pInfo->msgType == TDMT_VND_FETCH ||
-      pInfo->msgType == TDMT_VND_QUERY_CONTINUE) {
-    rpcMsg.persistHandle = 1;
-  }
-
   assert(pInfo->fp != NULL);
 
   rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx);
@@ -168,7 +163,7 @@ int32_t asyncSendMsgToServer(void* pTransporter, SEpSet* epSet, int64_t* pTransp
   return asyncSendMsgToServerExt(pTransporter, epSet, pTransporterId, pInfo, false, NULL);
 }
 
-char *jobTaskStatusStr(int32_t status) {
+char* jobTaskStatusStr(int32_t status) {
   switch (status) {
     case JOB_TASK_STATUS_NULL:
       return "NULL";
@@ -197,13 +192,10 @@ char *jobTaskStatusStr(int32_t status) {
 
 SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name) {
   SSchema s = {0};
-  s.type  = type;
+  s.type = type;
   s.bytes = bytes;
   s.colId = colId;
 
   tstrncpy(s.name, name, tListLen(s.name));
   return s;
 }
-
-
-

From ae8b3cc11d7980076b42626159caebf1e47f6655 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 12 May 2022 17:45:17 +0800
Subject: [PATCH 28/35] enh(tsdb): return oom err for submit msg

---
 source/dnode/vnode/src/vnd/vnodeSvr.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c
index fc2b6fe676..d2650725f0 100644
--- a/source/dnode/vnode/src/vnd/vnodeSvr.c
+++ b/source/dnode/vnode/src/vnd/vnodeSvr.c
@@ -562,6 +562,7 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
   int32_t        nRows;
   int32_t        tsize, ret;
   SEncoder       encoder = {0};
+  terrno = TSDB_CODE_SUCCESS;
 
   pRsp->code = 0;
 
@@ -576,6 +577,11 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in
   }
 
   submitRsp.pArray = taosArrayInit(pSubmitReq->numOfBlocks, sizeof(SSubmitBlkRsp));
+  if (!submitRsp.pArray) {
+    pRsp->code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+
   for (int i = 0;;) {
     tGetSubmitMsgNext(&msgIter, &pBlock);
     if (pBlock == NULL) break;
@@ -640,7 +646,12 @@ _exit:
 
   taosArrayDestroy(submitRsp.pArray);
 
-  tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+  // TODO: the partial success scenario and the error case
+  // TODO: refactor
+  if ((terrno == TSDB_CODE_SUCCESS || terrno == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) &&
+      (pRsp->code == TSDB_CODE_SUCCESS)) {
+    tsdbTriggerRSma(pVnode->pTsdb, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK);
+  }
 
   return 0;
 }

From d66bb65171a96a80e413684a4f0fd941eca8d6d0 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 12 May 2022 18:32:32 +0800
Subject: [PATCH 29/35] feat(tsdb): unit of days and keep support minute

---
 include/common/tglobal.h                    |  4 ++--
 source/common/src/tglobal.c                 | 10 +++++-----
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c |  8 ++++----
 source/dnode/vnode/src/inc/tsdb.h           |  8 ++++----
 source/dnode/vnode/src/tsdb/tsdbCommit.c    |  9 ++++-----
 source/dnode/vnode/src/tsdb/tsdbRead.c      |  6 +++---
 source/dnode/vnode/src/tsdb/tsdbSma.c       |  2 +-
 source/dnode/vnode/src/tsdb/tsdbWrite.c     |  4 ++--
 8 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/include/common/tglobal.h b/include/common/tglobal.h
index 97e6491b98..84aae46347 100644
--- a/include/common/tglobal.h
+++ b/include/common/tglobal.h
@@ -43,7 +43,7 @@ extern int32_t tsMaxNumOfDistinctResults;
 extern int32_t tsCompatibleModel;
 extern bool    tsEnableSlaveQuery;
 extern bool    tsPrintAuth;
-extern int64_t tsTickPerDay[3];
+extern int64_t tsTickPerMin[3];
 
 // multi-process
 extern bool    tsMultiProcess;
@@ -122,7 +122,7 @@ extern int32_t  tsDiskCfgNum;
 extern SDiskCfg tsDiskCfg[];
 
 // udf
-extern bool  tsStartUdfd;
+extern bool tsStartUdfd;
 
 // internal
 extern int32_t tsTransPullupInterval;
diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c
index 42c8e18f3e..0999cb4d2c 100644
--- a/source/common/src/tglobal.c
+++ b/source/common/src/tglobal.c
@@ -153,11 +153,11 @@ bool tsStreamSchedV = true;
 
 /*
  * minimum scale for whole system, millisecond by default
- * for TSDB_TIME_PRECISION_MILLI: 86400000L
- *     TSDB_TIME_PRECISION_MICRO: 86400000000L
- *     TSDB_TIME_PRECISION_NANO:  86400000000000L
+ * for TSDB_TIME_PRECISION_MILLI: 60000L
+ *     TSDB_TIME_PRECISION_MICRO: 60000000L
+ *     TSDB_TIME_PRECISION_NANO:  60000000000L
  */
-int64_t tsTickPerDay[] = {86400000L, 86400000000L, 86400000000000L};
+int64_t tsTickPerMin[] = {60000L, 60000000L, 60000000000L};
 
 // lossy compress 6
 char tsLossyColumns[32] = "";  // "float|double" means all float and double columns can be lossy compressed.  set empty
@@ -170,7 +170,7 @@ uint32_t tsCurRange = 100;                      // range
 char     tsCompressor[32] = "ZSTD_COMPRESSOR";  // ZSTD_COMPRESSOR or GZIP_COMPRESSOR
 
 // udf
-bool     tsStartUdfd = true;
+bool tsStartUdfd = true;
 
 // internal
 int32_t tsTransPullupInterval = 6;
diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index 6a0f4984b1..ae0f1b0d55 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -145,10 +145,10 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->szBuf = pCreate->buffer * 1024 * 1024;
   pCfg->isWeak = true;
   pCfg->tsdbCfg.precision = pCreate->precision;
-  pCfg->tsdbCfg.days = 10;
-  pCfg->tsdbCfg.keep0 = 3650;
-  pCfg->tsdbCfg.keep1 = 3650;
-  pCfg->tsdbCfg.keep2 = 3650;
+  pCfg->tsdbCfg.days = pCreate->daysPerFile;
+  pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0;
+  pCfg->tsdbCfg.keep1 = pCreate->daysToKeep1;
+  pCfg->tsdbCfg.keep2 = pCreate->daysToKeep2;
   pCfg->tsdbCfg.minRows = pCreate->minRows;
   pCfg->tsdbCfg.maxRows = pCreate->maxRows;
   for (size_t i = 0; i < taosArrayGetSize(pCreate->pRetensions); ++i) {
diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h
index b8cbb2d997..c9305f6af9 100644
--- a/source/dnode/vnode/src/inc/tsdb.h
+++ b/source/dnode/vnode/src/inc/tsdb.h
@@ -518,9 +518,9 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn);
 
 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) {
   if (key < 0) {
-    return (int)((key + 1) / tsTickPerDay[precision] / days - 1);
+    return (int)((key + 1) / tsTickPerMin[precision] / days - 1);
   } else {
-    return (int)((key / tsTickPerDay[precision] / days));
+    return (int)((key / tsTickPerMin[precision] / days));
   }
 }
 
@@ -770,8 +770,8 @@ static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet *pSrc, SDFileSet *pDest) {
 }
 
 static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY *minKey, TSKEY *maxKey) {
-  *minKey = fid * days * tsTickPerDay[precision];
-  *maxKey = *minKey + days * tsTickPerDay[precision] - 1;
+  *minKey = fid * days * tsTickPerMin[precision];
+  *maxKey = *minKey + days * tsTickPerMin[precision] - 1;
 }
 
 static FORCE_INLINE bool tsdbFSetIsOk(SDFileSet *pSet) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c
index c1813c787a..d180799e58 100644
--- a/source/dnode/vnode/src/tsdb/tsdbCommit.c
+++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c
@@ -216,9 +216,9 @@ void tsdbGetRtnSnap(STsdb *pRepo, SRtn *pRtn) {
   TSKEY         minKey, midKey, maxKey, now;
 
   now = taosGetTimestamp(pCfg->precision);
-  minKey = now - pCfg->keep2 * tsTickPerDay[pCfg->precision];
-  midKey = now - pCfg->keep1 * tsTickPerDay[pCfg->precision];
-  maxKey = now - pCfg->keep0 * tsTickPerDay[pCfg->precision];
+  minKey = now - pCfg->keep2 * tsTickPerMin[pCfg->precision];
+  midKey = now - pCfg->keep1 * tsTickPerMin[pCfg->precision];
+  maxKey = now - pCfg->keep0 * tsTickPerMin[pCfg->precision];
 
   pRtn->minKey = minKey;
   pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->days, pCfg->precision));
@@ -398,7 +398,7 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
       ++mIter;
     } else if (pIter && !pIter->pTable) {
       // When table already dropped during commit, pIter is not NULL but pIter->pTable is NULL.
-      ++mIter; // skip the table and do nothing
+      ++mIter;  // skip the table and do nothing
     } else if (pIdx) {
       if (tsdbMoveBlkIdx(pCommith, pIdx) < 0) {
         tsdbCloseCommitFile(pCommith, true);
@@ -948,7 +948,6 @@ static int tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx) {
 static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) {
   STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
 
-
   pCommith->pTable = pTable;
 
   if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) {
diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c
index 2511e3a570..b293f1399d 100644
--- a/source/dnode/vnode/src/tsdb/tsdbRead.c
+++ b/source/dnode/vnode/src/tsdb/tsdbRead.c
@@ -323,7 +323,7 @@ static int64_t getEarliestValidTimestamp(STsdb* pTsdb) {
   STsdbKeepCfg* pCfg = REPO_KEEP_CFG(pTsdb);
 
   int64_t now = taosGetTimestamp(pCfg->precision);
-  return now - (tsTickPerDay[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
+  return now - (tsTickPerMin[pCfg->precision] * pCfg->keep2) + 1;  // needs to add one tick
 }
 
 static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, SQueryTableDataCond* pCond) {
@@ -1047,10 +1047,10 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
   }
 
   if (key < 0) {
-    key -= (daysPerFile * tsTickPerDay[precision]);
+    key -= (daysPerFile * tsTickPerMin[precision]);
   }
 
-  int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerDay[precision]));  // set the starting fileId
+  int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerMin[precision]));  // set the starting fileId
   if (fid < 0L && llabs(fid) > INT32_MAX) {                                // data value overflow for INT32
     fid = INT32_MIN;
   }
diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c
index 61515a3be1..e878668654 100644
--- a/source/dnode/vnode/src/tsdb/tsdbSma.c
+++ b/source/dnode/vnode/src/tsdb/tsdbSma.c
@@ -1017,7 +1017,7 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe
   int32_t       daysPerFile = pCfg->days;
 
   if (storageLevel == SMA_STORAGE_LEVEL_TSDB) {
-    int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerDay[pCfg->precision]);
+    int32_t days = SMA_STORAGE_TSDB_TIMES * (interval / tsTickPerMin[pCfg->precision]);
     daysPerFile = days > SMA_STORAGE_TSDB_DAYS ? days : SMA_STORAGE_TSDB_DAYS;
   }
 
diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c
index 3107c6f5c7..341ab94ca4 100644
--- a/source/dnode/vnode/src/tsdb/tsdbWrite.c
+++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c
@@ -63,8 +63,8 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) {
   STSRow        *row = NULL;
   STsdbKeepCfg  *pCfg = REPO_KEEP_CFG(pTsdb);
   TSKEY          now = taosGetTimestamp(pCfg->precision);
-  TSKEY          minKey = now - tsTickPerDay[pCfg->precision] * pCfg->keep2;
-  TSKEY          maxKey = now + tsTickPerDay[pCfg->precision] * pCfg->days;
+  TSKEY          minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2;
+  TSKEY          maxKey = now + tsTickPerMin[pCfg->precision] * pCfg->days;
 
   terrno = TSDB_CODE_SUCCESS;
   // pMsg->length = htonl(pMsg->length);

From 910436a5647723b76852037ed29ec71299b51142 Mon Sep 17 00:00:00 2001
From: Cary Xu <klxu@taosdata.com>
Date: Thu, 12 May 2022 18:46:20 +0800
Subject: [PATCH 30/35] feat(tsdb): comp use cfg from params

---
 source/dnode/mgmt/mgmt_vnode/src/vmHandle.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
index ae0f1b0d55..914acce2ea 100644
--- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
+++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
@@ -144,6 +144,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
   pCfg->szCache = pCreate->pages;
   pCfg->szBuf = pCreate->buffer * 1024 * 1024;
   pCfg->isWeak = true;
+  pCfg->tsdbCfg.compression = pCreate->compression;
   pCfg->tsdbCfg.precision = pCreate->precision;
   pCfg->tsdbCfg.days = pCreate->daysPerFile;
   pCfg->tsdbCfg.keep0 = pCreate->daysToKeep0;

From d6f03eb7da95209406a1bb641d6b42a24d980330 Mon Sep 17 00:00:00 2001
From: Xiaoyu Wang <xiaoyuwang@taosdata.com>
Date: Thu, 12 May 2022 18:59:02 +0800
Subject: [PATCH 31/35] fix: some problems of parser

---
 source/libs/parser/src/parInsert.c    | 24 +++++++++++++++----
 source/libs/scheduler/src/scheduler.c | 33 +++++++++++++++++----------
 2 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c
index dfc56eff39..9c788202d3 100644
--- a/source/libs/parser/src/parInsert.c
+++ b/source/libs/parser/src/parInsert.c
@@ -53,6 +53,7 @@ typedef struct SInsertParseContext {
   SHashObj*          pTableBlockHashObj;  // global
   SHashObj*          pSubTableHashObj;    // global
   SArray*            pVgDataBlocks;       // global
+  SHashObj*          pTableNameHashObj;   // global
   int32_t            totalNum;
   SVnodeModifOpStmt* pOutput;
   SStmtCallback*     pStmtCb;
@@ -1111,6 +1112,8 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) {
     createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg);
     tNameExtractFullName(&name, tbFName);
 
+    CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName)));
+
     // USING cluase
     if (TK_USING == sToken.type) {
       CHECK_CODE(parseUsingClause(pCxt, &name, tbFName));
@@ -1193,7 +1196,8 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
       .pSql = (char*)pContext->pSql,
       .msg = {.buf = pContext->pMsg, .len = pContext->msgLen},
       .pTableMeta = NULL,
-      .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, false),
+      .pSubTableHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
+      .pTableNameHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), true, HASH_NO_LOCK),
       .totalNum = 0,
       .pOutput = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT),
       .pStmtCb = pContext->pStmtCb};
@@ -1202,12 +1206,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
     (*pContext->pStmtCb->getExecInfoFn)(pContext->pStmtCb->pStmt, &context.pVgroupsHashObj,
                                         &context.pTableBlockHashObj);
   } else {
-    context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
-    context.pTableBlockHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false);
+    context.pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
+    context.pTableBlockHashObj =
+        taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
   }
 
   if (NULL == context.pVgroupsHashObj || NULL == context.pTableBlockHashObj || NULL == context.pSubTableHashObj ||
-      NULL == context.pOutput) {
+      NULL == context.pTableNameHashObj || NULL == context.pOutput) {
     return TSDB_CODE_TSC_OUT_OF_MEMORY;
   }
 
@@ -1220,6 +1225,10 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
     if (NULL == *pQuery) {
       return TSDB_CODE_OUT_OF_MEMORY;
     }
+    (*pQuery)->pTableList = taosArrayInit(taosHashGetSize(context.pTableNameHashObj), sizeof(SName));
+    if (NULL == (*pQuery)->pTableList) {
+      return TSDB_CODE_OUT_OF_MEMORY;
+    }
     (*pQuery)->execMode = QUERY_EXEC_MODE_SCHEDULE;
     (*pQuery)->haveResultSet = false;
     (*pQuery)->msgType = TDMT_VND_SUBMIT;
@@ -1232,6 +1241,13 @@ int32_t parseInsertSql(SParseContext* pContext, SQuery** pQuery) {
   if (TSDB_CODE_SUCCESS == code) {
     code = parseInsertBody(&context);
   }
+  if (TSDB_CODE_SUCCESS == code) {
+    SName* pTable = taosHashIterate(context.pTableNameHashObj, NULL);
+    while (NULL != pTable) {
+      taosArrayPush((*pQuery)->pTableList, pTable);
+      pTable = taosHashIterate(context.pTableNameHashObj, pTable);
+    }
+  }
   destroyInsertParseContext(&context);
   return code;
 }
diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c
index 1ce074c49f..2710e54f95 100644
--- a/source/libs/scheduler/src/scheduler.c
+++ b/source/libs/scheduler/src/scheduler.c
@@ -1092,11 +1092,10 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
         if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) {
           for (int32_t i = 0; i < batchRsp.nRsps; ++i) {
             SVCreateTbRsp *rsp = batchRsp.pRsps + i;
-            if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) {
-              tDecoderClear(&coder);
-              SCH_ERR_JRET(rsp->code);
-            } else if (TSDB_CODE_SUCCESS != rsp->code) {
+            if (TSDB_CODE_SUCCESS != rsp->code) {
               code = rsp->code;
+              tDecoderClear(&coder);
+              SCH_ERR_JRET(code);
             }
           }
         }
@@ -1117,11 +1116,10 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
         if (TSDB_CODE_SUCCESS == code && batchRsp.nRsps > 0) {
           for (int32_t i = 0; i < batchRsp.nRsps; ++i) {
             SVDropTbRsp *rsp = batchRsp.pRsps + i;
-            if (NEED_CLIENT_HANDLE_ERROR(rsp->code)) {
-              tDecoderClear(&coder);
-              SCH_ERR_JRET(rsp->code);
-            } else if (TSDB_CODE_SUCCESS != rsp->code) {
+            if (TSDB_CODE_SUCCESS != rsp->code) {
               code = rsp->code;
+              tDecoderClear(&coder);
+              SCH_ERR_JRET(code);
             }
           }
         }
@@ -1137,7 +1135,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
       SCH_ERR_JRET(rspCode);
 
       if (msg) {
-        SDecoder coder = {0};
+        SDecoder    coder = {0};
         SSubmitRsp *rsp = taosMemoryMalloc(sizeof(*rsp));
         tDecoderInit(&coder, msg, msgSize);
         code = tDecodeSSubmitRsp(&coder, rsp);
@@ -1146,10 +1144,21 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
           tFreeSSubmitRsp(rsp);
           SCH_ERR_JRET(code);
         }
-        
+
+        if (rsp->nBlocks > 0) {
+          for (int32_t i = 0; i < rsp->nBlocks; ++i) {
+            SSubmitBlkRsp *blk = rsp->pBlocks + i;
+            if (TSDB_CODE_SUCCESS != blk->code) {
+              code = blk->code;
+              tFreeSSubmitRsp(rsp);
+              SCH_ERR_JRET(code);
+            }
+          }
+        }
+
         atomic_add_fetch_32(&pJob->resNumOfRows, rsp->affectedRows);
         SCH_TASK_DLOG("submit succeed, affectedRows:%d", rsp->affectedRows);
-        
+
         if (pJob->attr.needRes) {
           SCH_LOCK(SCH_WRITE, &pJob->resLock);
           if (pJob->resData) {
@@ -1160,7 +1169,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch
             memcpy(sum->pBlocks + sum->nBlocks - rsp->nBlocks, rsp->pBlocks, rsp->nBlocks * sizeof(*sum->pBlocks));
             taosMemoryFree(rsp->pBlocks);
             taosMemoryFree(rsp);
-          } else {        
+          } else {
             pJob->resData = rsp;
           }
           SCH_UNLOCK(SCH_WRITE, &pJob->resLock);

From c55d20b19159f1c777d1234a0cee716d656d386a Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 12 May 2022 19:09:53 +0800
Subject: [PATCH 32/35] test: error script for debug

---
 tests/system-test/99-TDcase/TD-15517.py | 372 ++++++++++++++++++++++++
 1 file changed, 372 insertions(+)
 create mode 100644 tests/system-test/99-TDcase/TD-15517.py

diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py
new file mode 100644
index 0000000000..e7837ba009
--- /dev/null
+++ b/tests/system-test/99-TDcase/TD-15517.py
@@ -0,0 +1,372 @@
+
+import taos
+import sys
+import time
+import socket
+import os
+import threading
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+from util.dnodes import *
+
+class TDTestCase:
+    hostname = socket.gethostname()
+    #rpcDebugFlagVal = '143'
+    #clientCfgDict = {'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
+    #clientCfgDict["rpcDebugFlag"]  = rpcDebugFlagVal
+    #updatecfgDict = {'clientCfg': {}, 'serverPort': '', 'firstEp': '', 'secondEp':'', 'rpcDebugFlag':'135', 'fqdn':''}
+    #updatecfgDict["rpcDebugFlag"] = rpcDebugFlagVal
+    #print ("===================: ", updatecfgDict)
+
+    def init(self, conn, logSql):
+        tdLog.debug(f"start to excute {__file__}")
+        #tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def getBuildPath(self):
+        selfPath = os.path.dirname(os.path.realpath(__file__))
+
+        if ("community" in selfPath):
+            projPath = selfPath[:selfPath.find("community")]
+        else:
+            projPath = selfPath[:selfPath.find("tests")]
+
+        for root, dirs, files in os.walk(projPath):
+            if ("taosd" in files):
+                rootRealPath = os.path.dirname(os.path.realpath(root))
+                if ("packaging" not in rootRealPath):
+                    buildPath = root[:len(root) - len("/build/bin")]
+                    break
+        return buildPath
+
+    def newcur(self,cfg,host,port):
+        user = "root"
+        password = "taosdata"
+        con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port)
+        cur=con.cursor()
+        print(cur)
+        return cur
+
+    def create_tables(self,tsql, dbName,vgroups,stbName,ctbNum,rowsPerTbl):
+        tsql.execute("create database if not exists %s vgroups %d"%(dbName, vgroups))        
+        tsql.execute("use %s" %dbName)
+        tsql.execute("create table %s (ts timestamp, c1 bigint, c2 binary(16)) tags(t1 int)"%stbName)
+        pre_create = "create table"
+        sql = pre_create
+        #tdLog.debug("doing create one  stable %s and %d  child table in %s  ..." %(stbname, count ,dbname))
+        for i in range(ctbNum):
+            sql += " %s_%d using %s tags(%d)"%(stbName,i,stbName,i+1)
+            if (i > 0) and (i%100 == 0):
+                tsql.execute(sql)
+                sql = pre_create
+        if sql != pre_create:
+            tsql.execute(sql)
+            
+        tdLog.debug("complete to create database[%s], stable[%s] and %d child tables" %(dbName, stbName, ctbNum))
+        return
+
+    def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs):
+        tdLog.debug("start to insert data ............")
+        tsql.execute("use %s" %dbName)
+        pre_insert = "insert into "
+        sql = pre_insert
+
+        #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows))
+        for i in range(ctbNum):
+            sql += " %s_%d values "%(stbName,i)
+            for j in range(rowsPerTbl):
+                sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j)
+                if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)):
+                    tsql.execute(sql)
+                    if j < rowsPerTbl - 1:
+                        sql = "insert into %s_%d values " %(stbName,i)
+                    else:
+                        sql = "insert into "
+        #end sql
+        if sql != pre_insert:
+            #print("insert sql:%s"%sql)
+            tsql.execute(sql)
+        tdLog.debug("insert data ............ [OK]")
+        return
+        
+    def prepareEnv(self, **parameterDict):
+        print ("input parameters:")
+        print (parameterDict)
+        # create new connector for my thread
+        tsql=self.newcur(parameterDict['cfg'], 'localhost', 6030)
+        self.create_tables(tsql,\
+                           parameterDict["dbName"],\
+                           parameterDict["vgroups"],\
+                           parameterDict["stbName"],\
+                           parameterDict["ctbNum"],\
+                           parameterDict["rowsPerTbl"])
+
+        self.insert_data(tsql,\
+                         parameterDict["dbName"],\
+                         parameterDict["stbName"],\
+                         parameterDict["ctbNum"],\
+                         parameterDict["rowsPerTbl"],\
+                         parameterDict["batchNum"],\
+                         parameterDict["startTs"])                           
+        return                   
+        
+    def run(self):
+        tdSql.prepare()
+
+        buildPath = self.getBuildPath()
+        if (buildPath == ""):
+            tdLog.exit("taosd not found!")
+        else:
+            tdLog.info("taosd found in %s" % buildPath)
+        cfgPath = buildPath + "/../sim/psim/cfg"
+        tdLog.info("cfgPath: %s" % cfgPath)
+
+        tdLog.printNoPrefix("======== test scenario 1: ")
+        tdLog.info("step 1: create database, stb, ctb and insert data")
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db',     \
+                         'vgroups':    1,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,   \
+                         'batchNum':   10,       \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+        time.sleep(2)
+        
+        # wait stb ready
+        while 1:
+            tdSql.query("show %s.stables"%parameterDict['dbName'])            
+            if tdSql.getRows() == 1:            
+                break
+            else:
+                time.sleep(1)
+
+        tdLog.info("create topics from super table")
+        topicFromStb = 'topic_stb_column'
+        topicFromCtb = 'topic_ctb_column'        
+        
+        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
+        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName']))
+        
+        time.sleep(1)
+        tdSql.query("show topics")
+        #tdSql.checkRows(2)
+        topic1 = tdSql.getData(0 , 0)
+        topic2 = tdSql.getData(1 , 0)
+        print (topic1)
+        print (topic2)
+        
+        print (topicFromStb)
+        print (topicFromCtb)
+        #tdLog.info("show topics: %s, %s"%topic1, topic2)
+        #if topic1 != topicFromStb or topic1 != topicFromCtb:
+        #    tdLog.exit("topic error1") 
+        #if topic2 != topicFromStb or topic2 != topicFromCtb:
+        #    tdLog.exit("topic error2") 
+         
+        tdLog.info("create consume info table and consume result table")
+        cdbName = parameterDict["dbName"]
+        tdSql.query("create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)")
+        tdSql.query("create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)")
+
+        consumerId   = 0
+        expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"]
+        expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"]
+        topicList    = topicFromStb
+        ifcheckdata  = 0
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        sql = "insert into consumeinfo values "
+        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata)
+        tdSql.query(sql)
+        
+        tdLog.info("check stb if there are data")
+        while 1:
+            tdSql.query("select count(*) from %s"%parameterDict["stbName"])
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            countOfStb = tdSql.getData(0, 0)
+            if countOfStb != 0:
+                tdLog.info("count from stb: %d"%countOfStb)
+                break
+            else:
+                time.sleep(1)
+        
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        
+        shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
+        shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) 
+        shellCmd += "> /dev/null 2>&1 &"        
+        tdLog.info(shellCmd)
+        os.system(shellCmd)        
+
+        # wait for data ready
+        prepareEnvThread.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        while 1:
+            tdSql.query("select * from consumeresult")
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            if tdSql.getRows() == 1:
+                break
+            else:
+                time.sleep(5)
+
+        expectrowcnt = parameterDict["rowsPerTbl"]  * parameterDict["ctbNum"]
+                    
+        tdSql.checkData(0 , 1, consumerId)
+        tdSql.checkData(0 , 2, expectmsgcnt)
+        tdSql.checkData(0 , 3, expectrowcnt)
+
+        tdSql.query("drop topic %s"%topicFromStb)
+        tdSql.query("drop topic %s"%topicFromCtb)
+        
+        # ==============================================================================
+        tdLog.printNoPrefix("======== test scenario 2: add child table with consuming ")
+        tdLog.info(" clean database")        
+        # create and start thread
+        parameterDict = {'cfg':        '',       \
+                         'dbName':     'db2',    \
+                         'vgroups':    1,        \
+                         'stbName':    'stb',    \
+                         'ctbNum':     10,       \
+                         'rowsPerTbl': 10000,   \
+                         'batchNum':   100,       \
+                         'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
+        parameterDict['cfg'] = cfgPath
+
+        prepareEnvThread = threading.Thread(target=self.prepareEnv, kwargs=parameterDict)
+        prepareEnvThread.start()
+        
+        # wait db ready
+        while 1:
+            tdSql.query("show databases")
+            if tdSql.getRows() == 3: 
+                print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),)           
+                break
+            else:
+                time.sleep(1)
+        
+        tdSql.query("use %s"%parameterDict['dbName'])
+        # wait stb ready
+        while 1:
+            tdSql.query("show %s.stables"%parameterDict['dbName'])
+            if tdSql.getRows() == 1:            
+                break
+            else:
+                time.sleep(1)
+
+        tdLog.info("create topics from super table")
+        topicFromStb = 'topic_stb_column2'
+        topicFromCtb = 'topic_ctb_column2'
+        
+        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s" %(topicFromStb, parameterDict['dbName'], parameterDict['stbName']))
+        tdSql.execute("create topic %s as select ts, c1, c2 from %s.%s_0" %(topicFromCtb, parameterDict['dbName'], parameterDict['stbName']))
+        
+        time.sleep(1)
+        tdSql.query("show topics")
+        topic1 = tdSql.getData(0 , 0)
+        topic2 = tdSql.getData(1 , 0)
+        print (topic1)
+        print (topic2)
+        
+        print (topicFromStb)
+        print (topicFromCtb)
+        #tdLog.info("show topics: %s, %s"%topic1, topic2)
+        #if topic1 != topicFromStb or topic1 != topicFromCtb:
+        #    tdLog.exit("topic error1") 
+        #if topic2 != topicFromStb or topic2 != topicFromCtb:
+        #    tdLog.exit("topic error2") 
+         
+        tdLog.info("create consume info table and consume result table")
+        cdbName = parameterDict["dbName"]
+        tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int)"%cdbName)
+        tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName)
+
+        consumerId   = 0
+        expectmsgcnt = (parameterDict["rowsPerTbl"] / parameterDict["batchNum"] ) * parameterDict["ctbNum"]
+        expectmsgcnt1 = expectmsgcnt + parameterDict["ctbNum"]
+        topicList    = topicFromStb
+        ifcheckdata  = 0
+        keyList      = 'group.id:cgrp1,\
+                        enable.auto.commit:false,\
+                        auto.commit.interval.ms:6000,\
+                        auto.offset.reset:earliest'
+        sql = "insert into consumeinfo values "
+        sql += "(now, %d, '%s', '%s', %d, %d)"%(consumerId, topicList, keyList, expectmsgcnt1, ifcheckdata)
+        tdSql.query(sql)
+        
+        tdLog.info("check stb if there are data")
+        while 1:
+            tdSql.query("select count(*) from %s"%parameterDict["stbName"])
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            countOfStb = tdSql.getData(0, 0)
+            if countOfStb != 0:
+                tdLog.info("count from stb: %d"%countOfStb)
+                break
+            else:
+                time.sleep(1)
+        
+        tdLog.info("start consume processor")
+        pollDelay = 5
+        showMsg   = 1
+        showRow   = 1
+        
+        shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath
+        shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, parameterDict["dbName"], showMsg, showRow, cdbName) 
+        shellCmd += "> /dev/null 2>&1 &"        
+        tdLog.info(shellCmd)
+        os.system(shellCmd)        
+
+        # create new child table and insert data
+        newCtbName = 'newctb'
+        rowsOfNewCtb = 1000
+        tdSql.query("create table %s.%s using %s.%s tags(9999)"%(parameterDict["dbName"], newCtbName, parameterDict["dbName"], parameterDict["stbName"]))
+        startTs = parameterDict["startTs"]
+        for j in range(rowsOfNewCtb):
+            sql = "insert into %s.%s values (%d, %d, 'tmqrow_%d') "%(parameterDict["dbName"], newCtbName, startTs + j, j, j)
+            tdSql.execute(sql)
+        tdLog.debug("insert data into new child table ............ [OK]")
+
+        # wait for data ready
+        prepareEnvThread.join()
+        
+        tdLog.info("insert process end, and start to check consume result")
+        while 1:
+            tdSql.query("select * from consumeresult")
+            #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3))
+            if tdSql.getRows() == 1:
+                break
+            else:
+                time.sleep(5)
+
+        expectmsgcnt += rowsOfNewCtb
+        expectrowcnt = parameterDict["rowsPerTbl"]  * parameterDict["ctbNum"] + rowsOfNewCtb
+                    
+        tdSql.checkData(0 , 1, consumerId)
+        tdSql.checkData(0 , 2, expectmsgcnt)
+        tdSql.checkData(0 , 3, expectrowcnt)
+
+        
+        # ==============================================================================
+        tdLog.printNoPrefix("======== test scenario 3: ")
+
+        #os.system('pkill tmq_sim')
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())

From 7ed7e1de40c0005a90d833008417b86a6964af27 Mon Sep 17 00:00:00 2001
From: Hongze Cheng <hzcheng@taosdata.com>
Date: Thu, 12 May 2022 11:21:44 +0000
Subject: [PATCH 33/35] feat: tag read

---
 include/common/tdataformat.h            | 2 +-
 source/dnode/vnode/inc/vnode.h          | 9 +++++----
 source/dnode/vnode/src/meta/metaQuery.c | 5 +++++
 3 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h
index 2e8a214a9d..c80b8db58a 100644
--- a/include/common/tdataformat.h
+++ b/include/common/tdataformat.h
@@ -384,7 +384,7 @@ static FORCE_INLINE int32_t comparTagId(const void *key1, const void *key2) {
   }
 }
 
-static FORCE_INLINE void *tdGetKVRowValOfCol(SKVRow row, int16_t colId) {
+static FORCE_INLINE void *tdGetKVRowValOfCol(const SKVRow row, int16_t colId) {
   void *ret = taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
   if (ret == NULL) return NULL;
   return kvRowColVal(row, (SColIdx *)ret);
diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h
index ebf49c644b..b84732c848 100644
--- a/source/dnode/vnode/inc/vnode.h
+++ b/source/dnode/vnode/inc/vnode.h
@@ -72,10 +72,11 @@ typedef struct SMeta       SMeta;  // todo: remove
 typedef struct SMetaReader SMetaReader;
 typedef struct SMetaEntry  SMetaEntry;
 
-void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
-void metaReaderClear(SMetaReader *pReader);
-int  metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
-int  metaReadNext(SMetaReader *pReader);
+void        metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
+void        metaReaderClear(SMetaReader *pReader);
+int         metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
+int         metaReadNext(SMetaReader *pReader);
+const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid);
 
 #if 1  // refact APIs below (TODO)
 typedef SVCreateTbReq   STbCfg;
diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c
index 7d0a87d79d..1e2c94679f 100644
--- a/source/dnode/vnode/src/meta/metaQuery.c
+++ b/source/dnode/vnode/src/meta/metaQuery.c
@@ -466,3 +466,8 @@ void *metaGetSmaInfoByIndex(SMeta *pMeta, int64_t indexUid, bool isDecode) {
 }
 
 #endif
+
+const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t cid) {
+  ASSERT(pEntry->type == TSDB_CHILD_TABLE);
+  return tdGetKVRowValOfCol((const SKVRow)pEntry->ctbEntry.pTags, cid);
+}
\ No newline at end of file

From d0c284d18310e256b515bf96f1800ee64a56913e Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 12 May 2022 19:31:33 +0800
Subject: [PATCH 34/35] test: update test case

---
 tests/system-test/99-TDcase/TD-15517.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py
index e7837ba009..44a64421d3 100644
--- a/tests/system-test/99-TDcase/TD-15517.py
+++ b/tests/system-test/99-TDcase/TD-15517.py
@@ -251,7 +251,7 @@ class TDTestCase:
         # wait db ready
         while 1:
             tdSql.query("show databases")
-            if tdSql.getRows() == 3: 
+            if tdSql.getRows() == 4: 
                 print (tdSql.getData(0,0), tdSql.getData(1,0),tdSql.getData(2,0),)           
                 break
             else:

From d2cfe7a8e0453d7550ef9d06205b9cadaf4e04f7 Mon Sep 17 00:00:00 2001
From: plum-lihui <huili@taosdata.com>
Date: Thu, 12 May 2022 19:39:40 +0800
Subject: [PATCH 35/35] test: update test case

---
 tests/system-test/99-TDcase/TD-15517.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system-test/99-TDcase/TD-15517.py b/tests/system-test/99-TDcase/TD-15517.py
index 44a64421d3..b7cac43954 100644
--- a/tests/system-test/99-TDcase/TD-15517.py
+++ b/tests/system-test/99-TDcase/TD-15517.py
@@ -131,7 +131,7 @@ class TDTestCase:
                          'vgroups':    1,        \
                          'stbName':    'stb',    \
                          'ctbNum':     10,       \
-                         'rowsPerTbl': 10000,   \
+                         'rowsPerTbl': 100,   \
                          'batchNum':   10,       \
                          'startTs':    1640966400000}  # 2022-01-01 00:00:00.000
         parameterDict['cfg'] = cfgPath