other: merge main.

packaging/tools/install.sh

@@ -378,8 +378,8 @@ function add_newHostname_to_hosts() {
 }
 
 function set_hostname() {
-  echo -e -n "${GREEN}Enter the public accessible IP address or fully qualified domain name TDengine will expose to users or applications (must not be 'localhost') :${NC}"
+  echo -e -n "${GREEN}Host name or IP (assigned to this machine) which can be accessed by your tools or apps: (must not be 'localhost') :${NC}"
-  read newHostname
+  read -e -p "" -i "$(hostname)" newHostname
   while true; do
     if [ -z "$newHostname" ]; then
       newHostname=$(hostname)
@@ -387,8 +387,8 @@ function set_hostname() {
     elif [ "$newHostname" != "localhost" ]; then
       break
     else
-      echo -e -n "${GREEN}Enter the public accessible IP address or fully qualified domain name TDengine will expose to users or applications (must not be 'localhost') :${NC}"
+      echo -e -n "${GREEN}Host name or IP (assigned to this machine) which can be accessed by your tools or apps: (must not be 'localhost') :${NC}"
-      read newHostname
+      read -e -p "" -i "$(hostname)" newHostname
     fi
   done
 

packaging/tools/install_client.sh

@@ -129,6 +129,7 @@ function install_bin() {
     if [ "$osType" != "Darwin" ]; then
         [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2}  || :
         [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2}  || :
+        [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2}  || :
     fi
     [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || :
   fi

packaging/tools/makeclient.sh

@@ -24,10 +24,12 @@ clientName2="${12}"
 productName="TDengine"
 clientName="taos"
 benchmarkName="taosBenchmark"
+dumpName="taosdump"
 configFile="taos.cfg"
 tarName="package.tar.gz"
 
 benchmarkName2="${clientName2}Benchmark"
+dumpName2="${clientName2}dump"
 
 if [ "$osType" != "Darwin" ]; then
   script_dir="$(dirname $(readlink -f $0))"
@@ -71,6 +73,7 @@ if [ "$osType" != "Darwin" ]; then
   else
     bin_files="${build_dir}/bin/${clientName} \
         ${build_dir}/bin/${benchmarkName} \
+        ${build_dir}/bin/${dumpName} \
         ${script_dir}/remove_client.sh \
         ${script_dir}/set_core.sh \
         ${script_dir}/get_client.sh"

source/dnode/mnode/impl/src/mndStb.c

@@ -858,22 +858,10 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat
   }
   return 0;
 }
-static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *tagname) {
+static int32_t mndGenIdxNameForFirstTag(char *fullname, char *dbname, char *stbname, char *tagname) {
-  char    randStr[TSDB_COL_NAME_LEN] = {0};
+  SName name = {0};
-  int32_t left = TSDB_COL_NAME_LEN - strlen(tagname) - 1;
+  tNameFromString(&name, stbname, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
-  if (left <= 1) {
+  return snprintf(fullname, TSDB_INDEX_FNAME_LEN, "%s.%s_%s", dbname, tagname, tNameGetTableName(&name));
-    sprintf(fullname, "%s.%s", dbname, tagname);
-  } else {
-    int8_t start = left < 8 ? 0 : 8;
-    int8_t end = left >= 24 ? 24 : left - 1;
-    // gen rand str len [base:end]
-    // note: ignore rand performance issues
-    int64_t len = taosRand() % (end - start + 1) + start;
-    taosRandStr2(randStr, len);
-    sprintf(fullname, "%s.%s_%s", dbname, tagname, randStr);
-  }
-
-  return 0;
 }
 
 static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCreate, SDbObj *pDb) {
@@ -889,12 +877,11 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea
   if (mndBuildStbFromReq(pMnode, &stbObj, pCreate, pDb) != 0) goto _OVER;
 
   SSchema *pSchema = &(stbObj.pTags[0]);
-  mndGenIdxNameForFirstTag(fullIdxName, pDb->name, pSchema->name);
+  mndGenIdxNameForFirstTag(fullIdxName, pDb->name, stbObj.name, pSchema->name);
   SSIdx idx = {0};
   if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) {
     terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST;
     mndReleaseIdx(pMnode, idx.pIdx);
-
     goto _OVER;
   }
 

source/dnode/vnode/src/tsdb/tsdbRead2.c

@@ -2645,6 +2645,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
   STableBlockScanInfo* pScanInfo = NULL;
   SFileDataBlockInfo*  pBlockInfo = getCurrentBlockInfo(pBlockIter);
   SLastBlockReader*    pLastBlockReader = pReader->status.fileIter.pLastBlockReader;
+  bool                 asc = ASCENDING_TRAVERSE(pReader->info.order);
 
   if (pReader->pIgnoreTables && taosHashGet(*pReader->pIgnoreTables, &pBlockInfo->uid, sizeof(pBlockInfo->uid))) {
     setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order);
@@ -2680,8 +2681,8 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
   } else {
     bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader);
     int64_t tsLast = bHasDataInLastBlock ? getCurrentKeyInLastBlock(pLastBlockReader) : INT64_MIN;
-    if (!bHasDataInLastBlock || ((ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.lastKey < tsLast) ||
+    if (!bHasDataInLastBlock ||
-                                 (!ASCENDING_TRAVERSE(pReader->info.order) && pBlockInfo->record.firstKey > tsLast))) {
+        ((asc && pBlockInfo->record.lastKey < tsLast) || (!asc && pBlockInfo->record.firstKey > tsLast))) {
       // whole block is required, return it directly
       SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info;
       pInfo->rows = pBlockInfo->record.numRow;
@@ -2703,26 +2704,28 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
       tBlockDataReset(pBData);
 
       SSDataBlock* pResBlock = pReader->resBlockInfo.pResBlock;
-      tsdbDebug("load data in last block firstly %s", pReader->idStr);
 
+      tsdbDebug("load data in last block firstly %s", pReader->idStr);
       int64_t st = taosGetTimestampUs();
 
-      while (1) {
+      // no data in last block, no need to proceed.
-        bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
+      while (hasDataInLastBlock(pLastBlockReader)) {
-
-        // no data in last block and block, no need to proceed.
-        if (hasBlockLData == false) {
-          break;
-        }
-
         code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
-        if (code) {
+        if (code != TSDB_CODE_SUCCESS) {
           return code;
         }
 
         if (pResBlock->info.rows >= pReader->resBlockInfo.capacity) {
           break;
         }
+
+        // data in stt now overlaps with current active file data block, need to composed with file data block.
+        int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader);
+        if ((keyInStt >= pBlockInfo->record.firstKey && asc) || (keyInStt <= pBlockInfo->record.lastKey && (!asc))) {
+          tsdbDebug("%p keyInStt:%" PRId64 ", overlap with file block, brange:%" PRId64 "-%" PRId64 " %s", pReader,
+                    keyInStt, pBlockInfo->record.firstKey, pBlockInfo->record.lastKey, pReader->idStr);
+          break;
+        }
       }
 
       double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -2735,7 +2738,6 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
                   pResBlock->info.rows, el, pReader->idStr);
       }
     }
-
   }
 
   return (pReader->code != TSDB_CODE_SUCCESS) ? pReader->code : code;

source/libs/stream/src/streamBackendRocksdb.c

@@ -1655,10 +1655,11 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) {
     stInfo("succ to open state %p on backendWrapper, %p, %s", pState, pBackendCfWrapper, pBackendCfWrapper->idstr);
 
     inst->pHandle = NULL;
-    inst->wOpt = NULL;
-    inst->rOpt = NULL;
     inst->cfOpt = NULL;
     inst->param = NULL;
+
+    inst->wOpt = NULL;
+    inst->rOpt = NULL;
     return 0;
   }
   taosThreadMutexUnlock(&handle->cfMutex);

source/os/src/osRand.c

@@ -86,8 +86,7 @@ void taosRandStr(char* str, int32_t size) {
 }
 
 void taosRandStr2(char* str, int32_t size) {
-
+  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789@";
-  const char* set = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ@";
   int32_t     len = strlen(set);
 
   for (int32_t i = 0; i < size; ++i) {

tests/docs-examples-test/test_R.sh

@@ -6,7 +6,7 @@ pgrep taosd || taosd >> /dev/null 2>&1 &
 pgrep taosadapter || taosadapter >> /dev/null 2>&1 &
 
 cd ../../docs/examples/R
-wget -N https://repo1.maven.org/maven2/com/taosdata/jdbc/taos-jdbcdriver/3.2.4/taos-jdbcdriver-3.2.4-dist.jar
+wget -N https://maven.aliyun.com/repository/central/com/taosdata/jdbc/taos-jdbcdriver/3.2.5/taos-jdbcdriver-3.2.5-dist.jar
 
 jar_path=`find . -name taos-jdbcdriver-*-dist.jar`
 echo jar_path=$jar_path

tools/shell/src/shellAuto.c

@@ -83,6 +83,11 @@ SWords shellCommands[] = {
     {"alter local \"asynclog\" \"1\";", 0, 0, NULL},
     {"alter topic", 0, 0, NULL},
     {"alter user <user_name> <user_actions> <anyword> ;", 0, 0, NULL},
+#ifdef TD_ENTERPRISE
+    {"balance vgroup;", 0, 0, NULL},
+    {"balance vgroup leader <vgroup_id>", 0, 0, NULL},
+#endif
+
     // 20
     {"create table <anyword> using <stb_name> tags(", 0, 0, NULL},
     {"create database <anyword> <db_options> <anyword> <db_options> <anyword> <db_options> <anyword> <db_options> "
@@ -127,9 +132,12 @@ SWords shellCommands[] = {
     {"kill query ", 0, 0, NULL},
     {"kill transaction ", 0, 0, NULL},
 #ifdef TD_ENTERPRISE
-    {"merge vgroup ", 0, 0, NULL},
+    {"merge vgroup <vgroup_id> <vgroup_id>", 0, 0, NULL},
 #endif
     {"pause stream <stream_name> ;", 0, 0, NULL},
+#ifdef TD_ENTERPRISE
+    {"redistribute vgroup <vgroup_id> dnode <dnode_id> ;", 0, 0, NULL},
+#endif
     {"resume stream <stream_name> ;", 0, 0, NULL},
     {"reset query cache;", 0, 0, NULL},
     {"restore dnode <dnode_id> ;", 0, 0, NULL},
@@ -187,7 +195,7 @@ SWords shellCommands[] = {
     {"show consumers;", 0, 0, NULL},
     {"show grants;", 0, 0, NULL},
 #ifdef TD_ENTERPRISE
-    {"split vgroup ", 0, 0, NULL},
+    {"split vgroup <vgroup_id>", 0, 0, NULL},
 #endif
     {"insert into <tb_name> values(", 0, 0, NULL},
     {"insert into <tb_name> using <stb_name> tags(", 0, 0, NULL},
@@ -268,7 +276,9 @@ char* db_options[] = {"keep ",
                       "wal_retention_size ",
                       "wal_segment_size "};
 
-char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "cachesize ", "wal_fsync_period ", "wal_level "};
+char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "stt_trigger ",
+                            "wal_retention_period ", "wal_retention_size ", "cachesize ", 
+                            "wal_fsync_period ", "buffer ", "pages " ,"wal_level "};
 
 char* data_types[] = {"timestamp",    "int",
                       "int unsigned", "varchar(16)",
@@ -312,26 +322,27 @@ bool    waitAutoFill = false;
 #define WT_VAR_TOPIC          5
 #define WT_VAR_STREAM         6
 #define WT_VAR_UDFNAME        7
+#define WT_VAR_VGROUPID       8
 
-#define WT_FROM_DB_MAX        7  // max get content from db
+#define WT_FROM_DB_MAX        8  // max get content from db
 #define WT_FROM_DB_CNT (WT_FROM_DB_MAX + 1)
 
-#define WT_VAR_ALLTABLE       8
+#define WT_VAR_ALLTABLE       9
-#define WT_VAR_FUNC           9
+#define WT_VAR_FUNC           10
-#define WT_VAR_KEYWORD        10
+#define WT_VAR_KEYWORD        11
-#define WT_VAR_TBACTION       11
+#define WT_VAR_TBACTION       12
-#define WT_VAR_DBOPTION       12
+#define WT_VAR_DBOPTION       13
-#define WT_VAR_ALTER_DBOPTION 13
+#define WT_VAR_ALTER_DBOPTION 14
-#define WT_VAR_DATATYPE       14
+#define WT_VAR_DATATYPE       15
-#define WT_VAR_KEYTAGS        15
+#define WT_VAR_KEYTAGS        16
-#define WT_VAR_ANYWORD        16
+#define WT_VAR_ANYWORD        17
-#define WT_VAR_TBOPTION       17
+#define WT_VAR_TBOPTION       18
-#define WT_VAR_USERACTION     18
+#define WT_VAR_USERACTION     19
-#define WT_VAR_KEYSELECT      19
+#define WT_VAR_KEYSELECT      20
-#define WT_VAR_SYSTABLE       20
+#define WT_VAR_SYSTABLE       21
-#define WT_VAR_LANGUAGE       21
+#define WT_VAR_LANGUAGE       22
 
-#define WT_VAR_CNT 22
+#define WT_VAR_CNT 23
 
 
 #define WT_TEXT 0xFF
@@ -345,11 +356,11 @@ TdThread* threads[WT_FROM_DB_CNT];
 // obtain var name  with sql from server
 char varTypes[WT_VAR_CNT][64] = {
     "<db_name>",    "<stb_name>",  "<tb_name>",  "<dnode_id>",  "<user_name>",    "<topic_name>", "<stream_name>",
-    "<udf_name>",   "<all_table>", "<function>", "<keyword>",    "<tb_actions>",   "<db_options>", "<alter_db_options>",
+    "<udf_name>",   "<vgroup_id>", "<all_table>", "<function>", "<keyword>",    "<tb_actions>",   "<db_options>", "<alter_db_options>",
     "<data_types>", "<key_tags>",  "<anyword>",  "<tb_options>", "<user_actions>", "<key_select>", "<sys_table>", "<udf_language>"};
 
 char varSqls[WT_FROM_DB_CNT][64] = {"show databases;", "show stables;", "show tables;", "show dnodes;",
-                                    "show users;",     "show topics;",  "show streams;", "show functions;"};
+                                    "show users;",     "show topics;",  "show streams;", "show functions;", "show vgroups;"};
 
 // var words current cursor, if user press any one key except tab, cursorVar can be reset to -1
 int  cursorVar = -1;
@@ -520,7 +531,10 @@ void showHelp() {
   printf(
       "\n\n\
   ----- special commands on enterpise version ----- \n\
+    balance vgroup; \n\
+    balance vgroup leader <vgroup_id> \n\
     compact database <db_name>; \n\
+    redistribute vgroup <vgroup_id> dnode <dnode_id> ;\n\
     split vgroup <vgroup_id>;");
 #endif
 
@@ -675,9 +689,9 @@ bool shellAutoInit() {
   // generate varType
   GenerateVarType(WT_VAR_FUNC, functions, sizeof(functions) / sizeof(char*));
   GenerateVarType(WT_VAR_KEYWORD, keywords, sizeof(keywords) / sizeof(char*));
+  GenerateVarType(WT_VAR_TBACTION, tb_actions, sizeof(tb_actions) / sizeof(char*));
   GenerateVarType(WT_VAR_DBOPTION, db_options, sizeof(db_options) / sizeof(char*));
   GenerateVarType(WT_VAR_ALTER_DBOPTION, alter_db_options, sizeof(alter_db_options) / sizeof(char*));
-  GenerateVarType(WT_VAR_TBACTION, tb_actions, sizeof(tb_actions) / sizeof(char*));
   GenerateVarType(WT_VAR_DATATYPE, data_types, sizeof(data_types) / sizeof(char*));
   GenerateVarType(WT_VAR_KEYTAGS, key_tags, sizeof(key_tags) / sizeof(char*));
   GenerateVarType(WT_VAR_TBOPTION, tb_options, sizeof(tb_options) / sizeof(char*));