Merge branch '3.0' of https://github.com/taosdata/TDengine into feat/TD-26644

docs/en/12-taos-sql/06-select.md

@@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] select_list
 hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
 
 hint:
-    BATCH_SCAN | NO_BATCH_SCAN   
+    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP
 
 select_list:
     select_expr [, select_expr] ...
@@ -87,15 +87,17 @@ Hints are a means of user control over query optimization for individual stateme
 
 The list of currently supported Hints is as follows:
 
-|    **Hint**   |    **Params**  |         **Comment**        |       **Scopt**            |  
+|    **Hint**   |    **Params**  |         **Comment**        |       **Scopt**            |
-| :-----------: | -------------- | -------------------------- | -------------------------- |
+| :-----------: | -------------- | -------------------------- | -----------------------------------|
-| BATCH_SCAN    | None           | Batch table scan           | JOIN statment for stable   |         
+| BATCH_SCAN    | None           | Batch table scan           | JOIN statment for stable           |
-| NO_BATCH_SCAN | None           | Sequential table scan      | JOIN statment for stable   |         
+| NO_BATCH_SCAN | None           | Sequential table scan      | JOIN statment for stable           |
+| SORT_FOR_GROUP| None           | Use sort for partition     | With normal column in partition by list |
 
 For example:
 
 ```sql
 SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts;
+SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
 ```
 
 ## Lists

docs/en/12-taos-sql/16-operators.md

@@ -54,6 +54,7 @@ LIKE is used together with wildcards to match strings. Its usage is described as
 MATCH and NMATCH are used together with regular expressions to match strings. Their usage is described as follows:
 
 - Use POSIX regular expression syntax. For more information, see Regular Expressions.
+- The `MATCH` operator returns true when the regular expression is matched. The `NMATCH` operator returns true when the regular expression is not matched.
 - Regular expression can be used against only table names, i.e. `tbname`, and tags/columns of binary/nchar types.
 - The maximum length of regular expression string is 128 bytes. Configuration parameter `maxRegexStringLen` can be used to set the maximum allowed regular expression. It's a configuration parameter on the client side, and will take effect after restarting the client.
 

docs/en/12-taos-sql/24-show.md

@@ -73,10 +73,10 @@ Shows the SQL statement used to create the specified table. This statement can b
 ## SHOW DATABASES
 
 ```sql
-SHOW DATABASES;
+SHOW [USER | SYSTEM] DATABASES;
 ```
 
-Shows all user-created databases.
+Shows all databases. The `USER` qualifier specifies only user-created databases. The `SYSTEM` qualifier specifies only system databases.
 
 ## SHOW DNODES
 
@@ -183,10 +183,10 @@ Shows all subscriptions in the system.
 ## SHOW TABLES
 
 ```sql
-SHOW [db_name.]TABLES [LIKE 'pattern'];
+SHOW [NORMAL | CHILD] [db_name.]TABLES [LIKE 'pattern'];
 ```
 
-Shows all standard tables and subtables in the current database. You can use LIKE for fuzzy matching.
+Shows all standard tables and subtables in the current database. You can use LIKE for fuzzy matching. The `Normal` qualifier specifies standard tables. The `CHILD` qualifier specifies subtables.
 
 ## SHOW TABLE DISTRIBUTED
 

docs/zh/12-taos-sql/06-select.md

@@ -24,7 +24,7 @@ SELECT [hints] [DISTINCT] select_list
 hints: /*+ [hint([hint_param_list])] [hint([hint_param_list])] */
 
 hint:
-    BATCH_SCAN | NO_BATCH_SCAN   
+    BATCH_SCAN | NO_BATCH_SCAN | SORT_FOR_GROUP
 
 select_list:
     select_expr [, select_expr] ...
@@ -87,15 +87,17 @@ Hints 是用户控制单个语句查询优化的一种手段，当 Hint 不适
 
 目前支持的 Hints 列表如下：
 
-|    **Hint**   |    **参数**    |         **说明**           |       **适用范围**         |  
+|    **Hint**   |    **参数**    |         **说明**           |       **适用范围**         |
-| :-----------: | -------------- | -------------------------- | -------------------------- |
+| :-----------: | -------------- | -------------------------- | -----------------------------|
-| BATCH_SCAN    | 无             | 采用批量读表的方式         | 超级表 JOIN 语句           |         
+| BATCH_SCAN    | 无             | 采用批量读表的方式         | 超级表 JOIN 语句             |
-| NO_BATCH_SCAN | 无             | 采用顺序读表的方式         | 超级表 JOIN 语句           |         
+| NO_BATCH_SCAN | 无             | 采用顺序读表的方式         | 超级表 JOIN 语句             |
+| SORT_FOR_GROUP| 无             | 采用sort方式进行分组       | partition by 列表有普通列时  |
 
 举例： 
 
 ```sql
 SELECT /*+ BATCH_SCAN() */ a.ts FROM stable1 a, stable2 b where a.tag0 = b.tag0 and a.ts = b.ts;
+SELECT /*+ SORT_FOR_GROUP() */ count(*), c1 FROM stable1 PARTITION BY c1;
 ```
 
 ## 列表

docs/zh/12-taos-sql/16-operators.md

@@ -54,6 +54,7 @@ LIKE 条件使用通配符字符串进行匹配检查，规则如下：
 MATCH 条件和 NMATCH 条件使用正则表达式进行匹配，规则如下：
 
 - 支持符合 POSIX 规范的正则表达式，具体规范内容可参见 Regular Expressions。
+- MATCH 和正则表达式匹配时, 返回 TURE.  NMATCH 和正则表达式不匹配时, 返回 TRUE. 
 - 只能针对子表名（即 tbname）、字符串类型的标签值进行正则表达式过滤，不支持普通列的过滤。
 - 正则匹配字符串长度不能超过 128 字节。可以通过参数 maxRegexStringLen 设置和调整最大允许的正则匹配字符串，该参数是客户端配置参数，需要重启客户端才能生效
 

docs/zh/12-taos-sql/24-show.md

@@ -73,10 +73,10 @@ SHOW CREATE TABLE [db_name.]tb_name
 ## SHOW DATABASES
 
 ```sql
-SHOW DATABASES;
+SHOW [USER | SYSTEM] DATABASES;
 ```
 
-显示用户定义的所有数据库。
+显示定义的所有数据库。SYSTEM 指定只显示系统数据库。USER 指定只显示用户创建的数据库。
 
 ## SHOW DNODES
 
@@ -183,10 +183,10 @@ SHOW SUBSCRIPTIONS;
 ## SHOW TABLES
 
 ```sql
-SHOW [db_name.]TABLES [LIKE 'pattern'];
+SHOW [NORMAL | CHILD] [db_name.]TABLES [LIKE 'pattern'];
 ```
 
-显示当前数据库下的所有普通表和子表的信息。可以使用 LIKE 对表名进行模糊匹配。
+显示当前数据库下的所有普通表和子表的信息。可以使用 LIKE 对表名进行模糊匹配。NORMAL 指定只显示普通表信息， CHILD 指定只显示子表信息。
 
 ## SHOW TABLE DISTRIBUTED
 

include/common/ttokendef.h

@@ -362,6 +362,8 @@
 #define TK_WAL                            343
 
 
+
+
 #define TK_NK_SPACE         600
 #define TK_NK_COMMENT       601
 #define TK_NK_ILLEGAL       602

include/libs/nodes/cmdnodes.h

@@ -507,6 +507,7 @@ typedef struct SBalanceVgroupStmt {
 
 typedef struct SBalanceVgroupLeaderStmt {
   ENodeType type;
+  int32_t   vgId;
 } SBalanceVgroupLeaderStmt;
 
 typedef struct SMergeVgroupStmt {

include/util/tdef.h

@@ -382,6 +382,7 @@ typedef enum ELogicConditionType {
 #define TSDB_MAX_STT_TRIGGER     1
 #define TSDB_DEFAULT_SST_TRIGGER 1
 #endif
+#define TSDB_STT_TRIGGER_ARRAY_SIZE     16 // maximum of TSDB_MAX_STT_TRIGGER of TD_ENTERPRISE and TD_COMMUNITY
 #define TSDB_MIN_HASH_PREFIX     (2 - TSDB_TABLE_NAME_LEN)
 #define TSDB_MAX_HASH_PREFIX     (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_PREFIX 0

source/client/src/clientImpl.c

@@ -157,6 +157,10 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas
     tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, epSet.epSet.eps[0].fqdn, epSet.epSet.eps[0].port);
 
     pInst = &p;
+  } else {
+    ASSERTS((*pInst) && (*pInst)->pAppHbMgr, "*pInst:%p, pAppHgMgr:%p", *pInst, (*pInst) ? (*pInst)->pAppHbMgr : NULL);
+    // reset to 0 in case of conn with duplicated user key but its user has ever been dropped.
+    atomic_store_8(&(*pInst)->pAppHbMgr->connHbFlag, 0);
   }
 
   taosThreadMutexUnlock(&appInfo.mutex);

source/dnode/mnode/impl/src/mndTrans.c

@@ -544,7 +544,9 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
   if (pTrans == NULL) {
     terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
   } else {
+    #ifdef WINDOWS
     taosThreadMutexInit(&pTrans->mutex, NULL);
+    #endif
   }
   return pTrans;
 }

source/dnode/vnode/src/inc/tsdb.h

@@ -625,7 +625,7 @@ struct SDFileSet {
   SDataFile *pDataF;
   SSmaFile  *pSmaF;
   uint8_t    nSttF;
-  SSttFile  *aSttF[TSDB_MAX_STT_TRIGGER];
+  SSttFile  *aSttF[TSDB_STT_TRIGGER_ARRAY_SIZE];
 };
 
 struct STSDBRowIter {
@@ -694,7 +694,7 @@ struct SDataFWriter {
   SHeadFile fHead;
   SDataFile fData;
   SSmaFile  fSma;
-  SSttFile  fStt[TSDB_MAX_STT_TRIGGER];
+  SSttFile  fStt[TSDB_STT_TRIGGER_ARRAY_SIZE];
 
   uint8_t *aBuf[4];
 };
@@ -705,7 +705,7 @@ struct SDataFReader {
   STsdbFD   *pHeadFD;
   STsdbFD   *pDataFD;
   STsdbFD   *pSmaFD;
-  STsdbFD   *aSttFD[TSDB_MAX_STT_TRIGGER];
+  STsdbFD   *aSttFD[TSDB_STT_TRIGGER_ARRAY_SIZE];
   uint8_t   *aBuf[3];
 };
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -69,7 +69,7 @@ typedef struct {
     SDataIter *pIter;
     SRBTree    rbt;
     SDataIter  dataIter;
-    SDataIter  aDataIter[TSDB_MAX_STT_TRIGGER];
+    SDataIter  aDataIter[TSDB_STT_TRIGGER_ARRAY_SIZE];
     int8_t     toLastOnly;
   };
   struct {
@@ -865,7 +865,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
   TSDB_CHECK_CODE(code, lino, _exit);
 
   // merger
-  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
+  for (int32_t iStt = 0; iStt < TSDB_STT_TRIGGER_ARRAY_SIZE; iStt++) {
     SDataIter *pIter = &pCommitter->aDataIter[iStt];
     pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
     if (pIter->aSttBlk == NULL) {
@@ -915,7 +915,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
   tBlockDataDestroy(&pCommitter->dReader.bData);
 
   // merger
-  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
+  for (int32_t iStt = 0; iStt < TSDB_STT_TRIGGER_ARRAY_SIZE; iStt++) {
     SDataIter *pIter = &pCommitter->aDataIter[iStt];
     taosArrayDestroy(pIter->aSttBlk);
     tBlockDataDestroy(&pIter->bData);

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -1000,7 +1000,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
   tsdbCloseFile(&(*ppReader)->pSmaFD);
 
   // stt
-  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
+  for (int32_t iStt = 0; iStt < TSDB_STT_TRIGGER_ARRAY_SIZE; iStt++) {
     if ((*ppReader)->aSttFD[iStt]) {
       tsdbCloseFile(&(*ppReader)->aSttFD[iStt]);
     }

source/libs/parser/inc/parAst.h

@@ -244,7 +244,7 @@ SNode* createResumeStreamStmt(SAstCreateContext* pCxt, bool ignoreNotExists, boo
 SNode* createKillStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pId);
 SNode* createKillQueryStmt(SAstCreateContext* pCxt, const SToken* pQueryId);
 SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt);
-SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt);
+SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId);
 SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, const SToken* pVgId2);
 SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes);
 SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId);

source/libs/parser/inc/sql.y

@@ -691,11 +691,16 @@ cmd ::= KILL TRANSACTION NK_INTEGER(A).
 
 /************************************************ merge/redistribute/ vgroup ******************************************/
 cmd ::= BALANCE VGROUP.                                                           { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); }
-cmd ::= BALANCE VGROUP LEADER.                                                    { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt); }
+cmd ::= BALANCE VGROUP LEADER on_vgroup_id(A).                                    { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt, &A); }
 cmd ::= MERGE VGROUP NK_INTEGER(A) NK_INTEGER(B).                                 { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &A, &B); }
 cmd ::= REDISTRIBUTE VGROUP NK_INTEGER(A) dnode_list(B).                          { pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &A, B); }
 cmd ::= SPLIT VGROUP NK_INTEGER(A).                                               { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &A); }
 
+%type on_vgroup_id                                                                { SToken }
+%destructor on_vgroup_id                                                          { }
+on_vgroup_id(A) ::= .                                                             { A = nil_token; }
+on_vgroup_id(A) ::= ON NK_INTEGER(B).                                             { A = B; }
+
 %type dnode_list                                                                  { SNodeList* }
 %destructor dnode_list                                                            { nodesDestroyList($$); }
 dnode_list(A) ::= DNODE NK_INTEGER(B).                                            { A = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &B)); }

source/libs/parser/src/parAstCreater.c

@@ -2307,10 +2307,13 @@ SNode* createBalanceVgroupStmt(SAstCreateContext* pCxt) {
   return (SNode*)pStmt;
 }
 
-SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt) {
+SNode* createBalanceVgroupLeaderStmt(SAstCreateContext* pCxt, const SToken* pVgId) {
   CHECK_PARSER_STATUS(pCxt);
   SBalanceVgroupLeaderStmt* pStmt = (SBalanceVgroupLeaderStmt*)nodesMakeNode(QUERY_NODE_BALANCE_VGROUP_LEADER_STMT);
   CHECK_OUT_OF_MEM(pStmt);
+  if (NULL != pVgId && NULL != pVgId->z) {
+    pStmt->vgId = taosStr2Int32(pVgId->z, NULL, 10);
+  }
   return (SNode*)pStmt;
 }
 

source/libs/parser/src/parTranslater.c

@@ -7439,6 +7439,7 @@ static int32_t translateBalanceVgroup(STranslateContext* pCxt, SBalanceVgroupStm
 
 static int32_t translateBalanceVgroupLeader(STranslateContext* pCxt, SBalanceVgroupLeaderStmt* pStmt) {
   SBalanceVgroupLeaderReq req = {0};
+  req.vgId = pStmt->vgId;
   return buildCmdMsg(pCxt, TDMT_MND_BALANCE_VGROUP_LEADER, (FSerializeFunc)tSerializeSBalanceVgroupLeaderReq, &req);
 }
 

source/libs/transport/src/tmsgcb.c

@@ -26,6 +26,10 @@ void tmsgSetDefault(const SMsgCb* msgcb) { defaultMsgCb = *msgcb; }
 int32_t tmsgPutToQueue(const SMsgCb* msgcb, EQueueType qtype, SRpcMsg* pMsg) {
   int32_t code = (*msgcb->putToQueueFp)(msgcb->mgmt, qtype, pMsg);
   if (code != 0) {
+    SRpcMsg rsp = {.code = code, .info = pMsg->info};
+    if (rsp.info.handle != NULL) {
+      tmsgSendRsp(&rsp);
+    }
     rpcFreeCont(pMsg->pCont);
     pMsg->pCont = NULL;
   }

tests/script/api/passwdTest.c

@@ -48,6 +48,7 @@ void createUsers(TAOS *taos, const char *host, char *qstr);
 void passVerTestMulti(const char *host, char *qstr);
 void sysInfoTest(TAOS *taos, const char *host, char *qstr);
 void userDroppedTest(TAOS *taos, const char *host, char *qstr);
+void clearTestEnv(TAOS *taos, const char *host, char *qstr);
 
 int   nPassVerNotified = 0;
 int   nUserDropped = 0;
@@ -210,6 +211,7 @@ int main(int argc, char *argv[]) {
   passVerTestMulti(argv[1], qstr);
   sysInfoTest(taos, argv[1], qstr);
   userDroppedTest(taos, argv[1], qstr);
+  clearTestEnv(taos, argv[1], qstr);
 
   taos_close(taos);
   taos_cleanup();
@@ -267,9 +269,9 @@ void passVerTestMulti(const char *host, char *qstr) {
   queryDB(taos[0], "create database if not exists demo2 vgroups 1 minrows 10");
   queryDB(taos[0], "create database if not exists demo3 vgroups 1 minrows 10");
 
-  queryDB(taos[0], "create table demo1.stb (ts timestamp, c1 int) tags(t1 int)");
+  queryDB(taos[0], "create table if not exists demo1.stb (ts timestamp, c1 int) tags(t1 int)");
-  queryDB(taos[0], "create table demo2.stb (ts timestamp, c1 int) tags(t1 int)");
+  queryDB(taos[0], "create table if not exists demo2.stb (ts timestamp, c1 int) tags(t1 int)");
-  queryDB(taos[0], "create table demo3.stb (ts timestamp, c1 int) tags(t1 int)");
+  queryDB(taos[0], "create table if not exists demo3.stb (ts timestamp, c1 int) tags(t1 int)");
 
   strcpy(qstr, "alter user root pass 'taos'");
   queryDB(taos[0], qstr);
@@ -326,9 +328,9 @@ void sysInfoTest(TAOS *taosRoot, const char *host, char *qstr) {
   queryDB(taosRoot, "create database if not exists demo12 vgroups 1 minrows 10");
   queryDB(taosRoot, "create database if not exists demo13 vgroups 1 minrows 10");
 
-  queryDB(taosRoot, "create table demo11.stb (ts timestamp, c1 int) tags(t1 int)");
+  queryDB(taosRoot, "create table if not exists demo11.stb (ts timestamp, c1 int) tags(t1 int)");
-  queryDB(taosRoot, "create table demo12.stb (ts timestamp, c1 int) tags(t1 int)");
+  queryDB(taosRoot, "create table if not exists demo12.stb (ts timestamp, c1 int) tags(t1 int)");
-  queryDB(taosRoot, "create table demo13.stb (ts timestamp, c1 int) tags(t1 int)");
+  queryDB(taosRoot, "create table if not exists demo13.stb (ts timestamp, c1 int) tags(t1 int)");
 
   sprintf(qstr, "show grants");
   char      output[BUF_LEN];
@@ -387,10 +389,14 @@ _REP:
   fprintf(stderr, ">>> succeed to run sysInfoTest\n");
   fprintf(stderr, "######## %s #########\n", __func__);
 }
-
+static bool isDropUser = true;
 void userDroppedTest(TAOS *taos, const char *host, char *qstr) {
   // users
   int nTestUsers = nUser;
+  int nLoop = 0;
+_loop:
+  ++nLoop;
+  printf("\n\n%s:%d LOOP %d, nTestUsers:%d\n", __func__, __LINE__, nLoop, nTestUsers);
   for (int i = 0; i < nTestUsers; ++i) {
     // sprintf(users[i], "user%d", i);
     taosu[i] = taos_connect(host, users[i], "taos", NULL, 0);
@@ -426,7 +432,6 @@ void userDroppedTest(TAOS *taos, const char *host, char *qstr) {
   for (int i = 0; i < nTestUsers; ++i) {
     taos_close(taosu[i]);
     printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i);
-    sleep(1);
   }
 
   fprintf(stderr, "######## %s #########\n", __func__);
@@ -437,5 +442,32 @@ void userDroppedTest(TAOS *taos, const char *host, char *qstr) {
     exit(1);
   }
   fprintf(stderr, "######## %s #########\n", __func__);
-  // sleep(300);
+
+  if (nLoop < 5) {
+    nUserDropped = 0;
+    for (int i = 0; i < nTestUsers; ++i) {
+      sprintf(users[i], "user%d", i);
+      sprintf(qstr, "CREATE USER %s PASS 'taos'", users[i]);
+      fprintf(stderr, "%s:%d create user:%s\n", __func__, __LINE__, users[i]);
+      queryDB(taos, qstr);
+    }
+    goto _loop;
+  }
+  isDropUser = false;
+}
+
+void clearTestEnv(TAOS *taos, const char *host, char *qstr) {
+  fprintf(stderr, "######## %s start #########\n", __func__);
+  // restore  password
+  sprintf(qstr, "alter user root pass 'taosdata'");
+  queryDB(taos, qstr);
+
+  if (isDropUser) {
+    for (int i = 0; i < nUser; ++i) {
+      sprintf(qstr, "drop user %s", users[i]);
+      queryDB(taos, qstr);
+    }
+  }
+  // sleep(3000);
+  fprintf(stderr, "######## %s end #########\n", __func__);
 }