Merge branch 'main' of https://github.com/taosdata/TDengine into main

packaging/tools/install.sh

@@ -481,10 +481,10 @@ function install_adapter_config() {
     ${csudo}mkdir -p ${cfg_install_dir}
     [ -f ${script_dir}/cfg/${adapterName}.toml ] && ${csudo}cp ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}
     [ -f ${cfg_install_dir}/${adapterName}.toml ] && ${csudo}chmod 644 ${cfg_install_dir}/${adapterName}.toml
-  fi
+  else
-
     [ -f ${script_dir}/cfg/${adapterName}.toml ] &&
       ${csudo}cp -f ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir}/${adapterName}.toml.new
+  fi
 
   [ -f ${cfg_install_dir}/${adapterName}.toml ] &&
     ${csudo}ln -s ${cfg_install_dir}/${adapterName}.toml ${install_main_dir}/cfg/${adapterName}.toml
@@ -499,9 +499,10 @@ function install_config() {
     ${csudo}mkdir -p ${cfg_install_dir}
     [ -f ${script_dir}/cfg/${configFile} ] && ${csudo}cp ${script_dir}/cfg/${configFile} ${cfg_install_dir}
     ${csudo}chmod 644 ${cfg_install_dir}/*
+  else
+    ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new
   fi
 
-  ${csudo}cp -f ${script_dir}/cfg/${configFile} ${cfg_install_dir}/${configFile}.new
   ${csudo}ln -s ${cfg_install_dir}/${configFile} ${install_main_dir}/cfg
 
   [ ! -z $1 ] && return 0 || : # only install client

source/common/src/systable.c

@@ -67,6 +67,8 @@ static const SSysDbTableSchema clusterSchema[] = {
     {.name = "name", .bytes = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
     {.name = "uptime", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true},
     {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
+    {.name = "version", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},  
+    {.name = "expire_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = true},
 };
 
 static const SSysDbTableSchema userDBSchema[] = {

source/dnode/mgmt/mgmt_mnode/src/mmFile.c

@@ -17,117 +17,97 @@
 #include "mmInt.h"
 #include "tjson.h"
 
-int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
+static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
-  int32_t   code = TSDB_CODE_INVALID_JSON_FORMAT;
+  int32_t code = 0;
-  int32_t   len = 0;
-  int32_t   maxLen = 4096;
-  char     *content = taosMemoryCalloc(1, maxLen + 1);
-  cJSON    *root = NULL;
-  char      file[PATH_MAX] = {0};
-  TdFilePtr pFile = NULL;
 
+  tjsonGetInt32ValueFromDouble(pJson, "deployed", pOption->deploy, code);
+  if (code < 0) return -1;
+  tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code);
+  if (code < 0) return 0;
+
+  SJson *replicas = tjsonGetObjectItem(pJson, "replicas");
+  if (replicas == NULL) return 0;
+  pOption->numOfReplicas = tjsonGetArraySize(replicas);
+
+  for (int32_t i = 0; i < pOption->numOfReplicas; ++i) {
+    SJson *replica = tjsonGetArrayItem(replicas, i);
+    if (replica == NULL) return -1;
+
+    SReplica *pReplica = pOption->replicas + i;
+    tjsonGetInt32ValueFromDouble(replica, "id", pReplica->id, code);
+    if (code < 0) return -1;
+    code = tjsonGetStringValue(replica, "fqdn", pReplica->fqdn);
+    if (code < 0) return -1;
+    tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code);
+    if (code < 0) return -1;
+  }
+
+  return 0;
+}
+
+int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
+  int32_t   code = -1;
+  TdFilePtr pFile = NULL;
+  char     *pData = NULL;
+  SJson    *pJson = NULL;
+  char      file[PATH_MAX] = {0};
   snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
+
+  if (taosStatFile(file, NULL, NULL) < 0) {
+    dInfo("mnode file:%s not exist", file);
+    return 0;
+  }
+
   pFile = taosOpenFile(file, TD_FILE_READ);
   if (pFile == NULL) {
-    code = 0;
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to open mnode file:%s since %s", file, terrstr());
     goto _OVER;
   }
 
-  len = (int32_t)taosReadFile(pFile, content, maxLen);
+  int64_t size = 0;
-  if (len <= 0) {
+  if (taosFStatFile(pFile, &size, NULL) < 0) {
-    dError("failed to read %s since content is null", file);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    dError("failed to fstat mnode file:%s since %s", file, terrstr());
     goto _OVER;
   }
 
-  content[len] = 0;
+  pData = taosMemoryMalloc(size + 1);
-  root = cJSON_Parse(content);
+  if (pData == NULL) {
-  if (root == NULL) {
+    terrno = TSDB_CODE_OUT_OF_MEMORY;
-    dError("failed to read %s since invalid json format", file);
     goto _OVER;
   }
 
-  cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
+  if (taosReadFile(pFile, pData, size) != size) {
-  if (!deployed || deployed->type != cJSON_Number) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
-    dError("failed to read %s since deployed not found", file);
+    dError("failed to read mnode file:%s since %s", file, terrstr());
-    goto _OVER;
-  }
-  pOption->deploy = deployed->valueint;
-
-  cJSON *selfIndex = cJSON_GetObjectItem(root, "selfIndex");
-  if (selfIndex) {
-    if (selfIndex->type != cJSON_Number) {
-      dError("failed to read %s since selfIndex not found", file);
-      goto _OVER;
-    }
-    pOption->selfIndex = selfIndex->valueint;
-  }
-
-  cJSON *replicas = cJSON_GetObjectItem(root, "replicas");
-  if (replicas) {
-    if (replicas->type != cJSON_Array) {
-      dError("failed to read %s since replicas not found", file);
     goto _OVER;
   }
 
-    int32_t numOfReplicas = cJSON_GetArraySize(replicas);
+  pData[size] = '\0';
-    if (numOfReplicas <= 0) {
+
-      dError("failed to read %s since numOfReplicas:%d invalid", file, numOfReplicas);
+  pJson = tjsonParse(pData);
+  if (pJson == NULL) {
+    terrno = TSDB_CODE_INVALID_JSON_FORMAT;
     goto _OVER;
   }
-    pOption->numOfReplicas = numOfReplicas;
 
-    for (int32_t i = 0; i < numOfReplicas; ++i) {
+  if (mmDecodeOption(pJson, pOption) < 0) {
-      SReplica *pReplica = pOption->replicas + i;
+    terrno = TSDB_CODE_INVALID_JSON_FORMAT;
-
-      cJSON *replica = cJSON_GetArrayItem(replicas, i);
-      if (replica == NULL) break;
-
-      cJSON *id = cJSON_GetObjectItem(replica, "id");
-      if (id) {
-        if (id->type != cJSON_Number) {
-          dError("failed to read %s since id not found", file);
     goto _OVER;
   }
-        if (pReplica) {
-          pReplica->id = id->valueint;
-        }
-      }
-
-      cJSON *fqdn = cJSON_GetObjectItem(replica, "fqdn");
-      if (fqdn) {
-        if (fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
-          dError("failed to read %s since fqdn not found", file);
-          goto _OVER;
-        }
-        if (pReplica) {
-          tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
-        }
-      }
-
-      cJSON *port = cJSON_GetObjectItem(replica, "port");
-      if (port) {
-        if (port->type != cJSON_Number) {
-          dError("failed to read %s since port not found", file);
-          goto _OVER;
-        }
-        if (pReplica) {
-          pReplica->port = (uint16_t)port->valueint;
-        }
-      }
-    }
-  }
 
   code = 0;
+  dInfo("succceed to read mnode file %s", file);
 
 _OVER:
-  if (content != NULL) taosMemoryFree(content);
+  if (pData != NULL) taosMemoryFree(pData);
-  if (root != NULL) cJSON_Delete(root);
+  if (pJson != NULL) cJSON_Delete(pJson);
   if (pFile != NULL) taosCloseFile(&pFile);
-  if (code == 0) {
-    dDebug("succcessed to read file %s, deployed:%d", file, pOption->deploy);
-  }
 
-  terrno = code;
+  if (code != 0) {
+    dError("failed to read mnode file:%s since %s", file, terrstr());
+  }
   return code;
 }
 

source/dnode/mnode/impl/src/mndCluster.c

@@ -20,6 +20,8 @@
 
 #define CLUSTER_VER_NUMBE    1
 #define CLUSTER_RESERVE_SIZE 60
+char    tsVersionName[16] = "community";
+int64_t tsExpireTime = 0;
 
 static SSdbRaw *mndClusterActionEncode(SClusterObj *pCluster);
 static SSdbRow *mndClusterActionDecode(SSdbRaw *pRaw);
@@ -291,6 +293,18 @@ static int32_t mndRetrieveClusters(SRpcMsg *pMsg, SShowObj *pShow, SSDataBlock *
     pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
     colDataAppend(pColInfo, numOfRows, (const char *)&pCluster->createdTime, false);
 
+    char ver[12] = {0};
+    STR_WITH_MAXSIZE_TO_VARSTR(ver, tsVersionName, pShow->pMeta->pSchemas[cols].bytes);
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    colDataAppend(pColInfo, numOfRows, (const char *)ver, false);
+
+    pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
+    if (tsExpireTime <= 0) {
+      colDataAppendNULL(pColInfo, numOfRows);
+    } else {
+      colDataAppend(pColInfo, numOfRows, (const char *)&tsExpireTime, false);
+    }
+
     sdbRelease(pSdb, pCluster);
     numOfRows++;
   }

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -2834,7 +2834,37 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
   TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader);
 
   if (pBlockInfo == NULL) {  // build data block from last data file
-    code = buildComposedDataBlock(pReader);
+    SBlockData* pBData = &pReader->status.fileBlockData;
+    tBlockDataReset(pBData);
+
+    SSDataBlock* pResBlock = pReader->pResBlock;
+    tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr);
+
+    int64_t st = taosGetTimestampUs();
+
+    while (1) {
+      bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
+
+      // no data in last block and block, no need to proceed.
+      if (hasBlockLData == false) {
+        break;
+      }
+
+      buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+      if (pResBlock->info.rows >= pReader->capacity) {
+        break;
+      }
+    }
+
+    double el = (taosGetTimestampUs() - st) / 1000.0;
+    updateComposedBlockInfo(pReader, el, pScanInfo);
+
+    if (pResBlock->info.rows > 0) {
+      tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
+                    " rows:%d, elapsed time:%.2f ms %s",
+                pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
+                pResBlock->info.rows, el, pReader->idStr);
+    }
   } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) {
     code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid);
     if (code != TSDB_CODE_SUCCESS) {
@@ -2853,10 +2883,38 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
       // only return the rows in last block
       int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader);
       ASSERT(tsLast >= pBlock->maxKey.ts);
-      tBlockDataReset(&pReader->status.fileBlockData);
 
+      SBlockData* pBData = &pReader->status.fileBlockData;
+      tBlockDataReset(pBData);
+
+      SSDataBlock* pResBlock = pReader->pResBlock;
       tsdbDebug("load data in last block firstly, due to desc scan data, %s", pReader->idStr);
-      code = buildComposedDataBlock(pReader);
+
+      int64_t st = taosGetTimestampUs();
+
+      while (1) {
+        bool hasBlockLData = hasDataInLastBlock(pLastBlockReader);
+
+        // no data in last block and block, no need to proceed.
+        if (hasBlockLData == false) {
+          break;
+        }
+
+        buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader);
+        if (pResBlock->info.rows >= pReader->capacity) {
+          break;
+        }
+      }
+
+      double el = (taosGetTimestampUs() - st) / 1000.0;
+      updateComposedBlockInfo(pReader, el, pScanInfo);
+
+      if (pResBlock->info.rows > 0) {
+        tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
+                      " rows:%d, elapsed time:%.2f ms %s",
+                  pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
+                  pResBlock->info.rows, el, pReader->idStr);
+      }
     } else {  // whole block is required, return it directly
       SDataBlockInfo* pInfo = &pReader->pResBlock->info;
       pInfo->rows = pBlock->nRow;

tests/script/tsim/parser/regressiontest.sim

@@ -58,4 +58,9 @@ if $data40 != @18-09-17 09:06:49.600@ then
  return -1
 endi
 
+sql select * from $tb order by ts desc;
+if $rows != 8198 then
+  return -1
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT