Merge branch '3.0' of https://github.com/taosdata/TDengine into 3.0

cmake/curl_CMakeLists.txt.in

@@ -1,7 +1,7 @@
 # curl
 ExternalProject_Add(curl
         URL  https://curl.se/download/curl-8.2.1.tar.gz
-        URL_HASH MD5 b25588a43556068be05e1624e0e74d41
+        URL_HASH MD5=b25588a43556068be05e1624e0e74d41
         DOWNLOAD_NO_PROGRESS 1
         DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
         #GIT_REPOSITORY https://github.com/curl/curl.git

docs/en/12-taos-sql/02-database.md

@@ -56,7 +56,7 @@ database_option: {
 - WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk.
 - MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096.
 - MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100.
-- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP).
+- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to three times of the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP).
 - PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB.
 - PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384.
 - PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms.

docs/zh/12-taos-sql/02-database.md

@@ -56,7 +56,7 @@ database_option: {
 - WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。
 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。
 - MINROWS：文件块中记录的最小条数，默认为 100 条。
-- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。
+- KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。
 - PAGES：一个 VNODE 中元数据存储引擎的缓存页个数，默认为 256，最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES，默认情况下为 1MB 内存。
 - PAGESIZE：一个 VNODE 中元数据存储引擎的页大小，单位为 KB，默认为 4 KB。范围为 1 到 16384，即 1 KB 到 16 MB。
 - PRECISION：数据库的时间戳精度。ms 表示毫秒，us 表示微秒，ns 表示纳秒，默认 ms 毫秒。

include/common/tgrant.h

@@ -51,6 +51,9 @@ typedef enum {
 } EGrantType;
 
 int32_t grantCheck(EGrantType grant);
+#ifdef TD_ENTERPRISE
+int32_t grantAlterActiveCode(const char* old, const char* new, char* out, int8_t type);
+#endif
 
 #ifndef GRANTS_CFG
 #ifdef TD_ENTERPRISE

include/libs/monitor/monitor.h

@@ -206,6 +206,11 @@ typedef struct {
   bool        comp;
 } SMonCfg;
 
+typedef struct {
+  int8_t state;
+  tsem_t sem;
+} SDmNotifyHandle;
+
 int32_t monInit(const SMonCfg *pCfg);
 void    monCleanup();
 void    monRecordLog(int64_t ts, ELogLevel level, const char *content);

include/util/taoserror.h

@@ -541,6 +541,13 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_GRANT_CPU_LIMITED             TAOS_DEF_ERROR_CODE(0, 0x080B)
 #define TSDB_CODE_GRANT_STABLE_LIMITED          TAOS_DEF_ERROR_CODE(0, 0x080C)
 #define TSDB_CODE_GRANT_TABLE_LIMITED           TAOS_DEF_ERROR_CODE(0, 0x080D)
+#define TSDB_CODE_GRANT_PAR_IVLD_ACTIVE         TAOS_DEF_ERROR_CODE(0, 0x080E)
+#define TSDB_CODE_GRANT_PAR_IVLD_KEY            TAOS_DEF_ERROR_CODE(0, 0x080F)
+#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KEY        TAOS_DEF_ERROR_CODE(0, 0x0810)
+#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN       TAOS_DEF_ERROR_CODE(0, 0x0811)
+#define TSDB_CODE_GRANT_GEN_IVLD_KEY            TAOS_DEF_ERROR_CODE(0, 0x0812)
+#define TSDB_CODE_GRANT_GEN_APP_LIMIT           TAOS_DEF_ERROR_CODE(0, 0x0813)
+#define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN       TAOS_DEF_ERROR_CODE(0, 0x0814)
 
 // sync
 // #define TSDB_CODE_SYN_INVALID_CONFIG         TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x

source/dnode/mgmt/mgmt_dnode/src/dmWorker.c

@@ -53,21 +53,26 @@ static void *dmStatusThreadFp(void *param) {
   return NULL;
 }
 
-tsem_t       dmNotifySem;
+SDmNotifyHandle dmNotifyHdl = {.state = 0};
 static void    *dmNotifyThreadFp(void *param) {
   SDnodeMgmt *pMgmt = param;
-  int64_t     lastTime = taosGetTimestampMs();
   setThreadName("dnode-notify");
 
-  if (tsem_init(&dmNotifySem, 0, 0) != 0) {
+  if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) {
     return NULL;
   }
 
+  bool wait = true;
   while (1) {
     if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
-
+    if (wait) tsem_wait(&dmNotifyHdl.sem);
-    tsem_wait(&dmNotifySem);
+    atomic_store_8(&dmNotifyHdl.state, 1);
     dmSendNotifyReq(pMgmt);
+    if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) {
+      wait = true;
+      continue;
+    }
+    wait = false;
   }
 
   return NULL;
@@ -189,11 +194,11 @@ int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
 
 void dmStopNotifyThread(SDnodeMgmt *pMgmt) {
   if (taosCheckPthreadValid(pMgmt->notifyThread)) {
-    tsem_post(&dmNotifySem);
+    tsem_post(&dmNotifyHdl.sem);
     taosThreadJoin(pMgmt->notifyThread, NULL);
     taosThreadClear(&pMgmt->notifyThread);
   }
-  tsem_destroy(&dmNotifySem);
+  tsem_destroy(&dmNotifyHdl.sem);
 }
 
 int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {

source/dnode/mnode/impl/inc/mndUser.h

@@ -47,7 +47,7 @@ void    mndUserFreeObj(SUserObj *pUser);
 
 int64_t mndGetIpWhiteVer(SMnode *pMnode);
 
-void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock);
+void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock);
 
 int32_t mndRefreshUserIpWhiteList(SMnode *pMnode);
 

source/dnode/mnode/impl/src/mndDb.c

@@ -294,6 +294,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
   pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0;
   pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1;
   pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2;
+  pOld->cfg.keepTimeOffset = pNew->cfg.keepTimeOffset;
   pOld->cfg.walFsyncPeriod = pNew->cfg.walFsyncPeriod;
   pOld->cfg.walLevel = pNew->cfg.walLevel;
   pOld->cfg.walRetentionPeriod = pNew->cfg.walRetentionPeriod;

source/dnode/mnode/impl/src/mndDnode.c

@@ -147,7 +147,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
 
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
   code = 0;
-  mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
+  mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
 
 _OVER:
   mndTransDrop(pTrans);
@@ -752,7 +752,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
   code = 0;
 
-  mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
+  mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
 _OVER:
   mndTransDrop(pTrans);
   sdbFreeRaw(pRaw);
@@ -763,7 +763,9 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
   SSdbRaw   *pRaw = NULL;
   STrans    *pTrans = NULL;
   SDnodeObj *pDnode = NULL;
+  SArray    *failRecord = NULL;
   bool       cfgAll = pCfgReq->dnodeId == -1;
+  int32_t    cfgAllErr = 0;
   int32_t    iter = 0;
 
   SSdb *pSdb = pMnode->pSdb;
@@ -777,28 +779,64 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
       goto _OVER;
     }
 
+    SDnodeObj tmpDnode = *pDnode;
+    if (action == DND_ACTIVE_CODE) {
+#ifndef TD_CHECK_ACTIVE
+      strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN);
+#else
+      if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
+        if (TSDB_CODE_DUP_KEY != terrno) {
+          mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
+                 pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
+          if (cfgAll) {  // alter all dnodes:
+            if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
+            if (failRecord) taosArrayPush(failRecord, &pDnode->id);
+            if (0 == cfgAllErr) cfgAllErr = terrno;  // output 1st terrno.
+          }
+        } else {
+          terrno = 0;  // no action for dup active code
+        }
+        if (cfgAll) continue;
+        goto _OVER;
+      }
+#endif
+    } else if (action == DND_CONN_ACTIVE_CODE) {
+#ifndef TD_CHECK_ACTIVE
+      strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN);
+#else
+      if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
+        if (TSDB_CODE_DUP_KEY != terrno) {
+          mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
+                 pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
+          if (cfgAll) {
+            if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
+            if (failRecord) taosArrayPush(failRecord, &pDnode->id);
+            if (0 == cfgAllErr) cfgAllErr = terrno;
+          }
+        } else {
+          terrno = 0;
+        }
+        if (cfgAll) continue;
+        goto _OVER;
+      }
+#endif
+    } else {
+      terrno = TSDB_CODE_INVALID_CFG;
+      goto _OVER;
+    }
+
     if (!pTrans) {
       pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "config-dnode");
       if (!pTrans) goto _OVER;
       if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
     }
 
-    SDnodeObj tmpDnode = *pDnode;
-    if (action == DND_ACTIVE_CODE) {
-      strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN);
-    } else if (action == DND_CONN_ACTIVE_CODE) {
-      strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN);
-    } else {
-      terrno = TSDB_CODE_INVALID_CFG;
-      goto _OVER;
-    }
-
     pRaw = mndDnodeActionEncode(&tmpDnode);
     if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
     (void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
     pRaw = NULL;
 
-    mInfo("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle,
+    mInfo("dnode:%d, config dnode:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle,
           pCfgReq->config, pCfgReq->value);
 
     if (cfgAll) {
@@ -816,12 +854,19 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
 _OVER:
   if (cfgAll) {
     sdbRelease(pSdb, pDnode);
+    if (cfgAllErr != 0) terrno = cfgAllErr;
+    int32_t nFail = taosArrayGetSize(failRecord);
+    if (nFail > 0) {
+      mError("config dnode, cfg:%d, app:%p config:%s value:%s. total:%d, fail:%d", pCfgReq->dnodeId, pReq->info.ahandle,
+             pCfgReq->config, pCfgReq->value, iter, nFail);
+    }
   } else {
     mndReleaseDnode(pMnode, pDnode);
   }
   sdbCancelFetch(pSdb, pIter);
   mndTransDrop(pTrans);
   sdbFreeRaw(pRaw);
+  taosArrayDestroy(failRecord);
   return terrno;
 }
 
@@ -1045,7 +1090,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
 
   if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
 
-  mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
+  mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
   code = 0;
 
 _OVER:
@@ -1191,7 +1236,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
 
     strcpy(dcfgReq.config, "monitor");
     snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
-} else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) {
+  } else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) {
     int32_t optLen = strlen("ttlpushinterval");
     int32_t flag = -1;
     int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
@@ -1262,7 +1307,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
     snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value);
 
     if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) {
-      mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr());
+      mError("dnode:%d, failed to config activeCode since %s. conf:%s, val:%s", cfgReq.dnodeId, terrstr(),
+             cfgReq.config, cfgReq.value);
       return -1;
     }
     return 0;

source/dnode/mnode/impl/src/mndGrant.c

@@ -131,6 +131,7 @@ void    grantAdd(EGrantType grant, uint64_t value) {}
 void    grantRestore(EGrantType grant, uint64_t value) {}
 int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
 int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
+int32_t grantAlterActiveCode(const char *old, const char *new, char *out, int8_t type) { return TSDB_CODE_SUCCESS; }
 
 #endif
 

source/dnode/mnode/impl/src/mndTrans.c

@@ -543,6 +543,8 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
   STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId);
   if (pTrans == NULL) {
     terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
+  } else {
+    taosThreadMutexInit(&pTrans->mutex, NULL);
   }
   return pTrans;
 }

source/dnode/mnode/impl/src/mndUser.c

@@ -103,7 +103,8 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) {
 
   for (int i = 0; i < taosArrayGetSize(fqdns); i++) {
     char *fqdn = taosArrayGetP(fqdns, i);
-    mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD);
+    update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD);
+    update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, IP_WHITE_ADD);
   }
 
   for (int i = 0; i < taosArrayGetSize(fqdns); i++) {
@@ -115,8 +116,7 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) {
   // for (int i = 0; i < taosArrayGetSize(pUserNames); i++) {
   //   taosMemoryFree(taosArrayGetP(pUserNames, i));
   // }
-  //taosArrayDestroy(pUserNames);
+  // taosArrayDestroy(pUserNames);
-  
 
   if (update) ipWhiteMgt.ver++;
 
@@ -302,7 +302,7 @@ int32_t mndRefreshUserIpWhiteList(SMnode *pMnode) {
 
   return 0;
 }
-void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) {
+void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) {
   if (lock) {
     taosThreadRwlockWrlock(&ipWhiteMgt.rw);
     if (ipWhiteMgt.ver == 0) {
@@ -313,6 +313,20 @@ void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_
   }
 
   bool update = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, type);
+
+  void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL);
+  while (pIter) {
+    size_t klen = 0;
+    char  *key = taosHashGetKey(pIter, &klen);
+
+    char *keyDup = taosMemoryCalloc(1, klen + 1);
+    memcpy(keyDup, key, klen);
+    update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, keyDup, fqdn, type);
+    taosMemoryFree(keyDup);
+
+    pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, pIter);
+  }
+
   if (update) ipWhiteMgt.ver++;
 
   if (lock) taosThreadRwlockUnlock(&ipWhiteMgt.rw);

source/dnode/vnode/inc/vnode.h

@@ -132,9 +132,10 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
 int32_t  metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
 int32_t  metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
                                int32_t payloadLen);
-bool     metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
+bool     metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type);
-int32_t  metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
+int32_t  metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type);
-int32_t  metaSizeOfTbFilterCache(void *pVnode, int8_t type);
+int32_t  metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type);
+int32_t  metaInitTbFilterCache(SMeta *pMeta);
 
 int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols);
 

source/dnode/vnode/src/meta/metaCache.c

@@ -14,6 +14,13 @@
  */
 #include "meta.h"
 
+#ifdef TD_ENTERPRISE
+extern const char* tkLogStb[];
+extern const char* tkAuditStb[];
+extern const int   tkLogStbNum;
+extern const int   tkAuditStbNum;
+#endif
+
 #define TAG_FILTER_RES_KEY_LEN  32
 #define META_CACHE_BASE_BUCKET  1024
 #define META_CACHE_STATS_BUCKET 16
@@ -69,6 +76,7 @@ struct SMetaCache {
 
   struct STbFilterCache {
     SHashObj* pStb;
+    SHashObj* pStbName;
   } STbFilterCache;
 };
 
@@ -178,6 +186,13 @@ int32_t metaCacheOpen(SMeta* pMeta) {
     goto _err2;
   }
 
+  pCache->STbFilterCache.pStbName =
+      taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
+  if (pCache->STbFilterCache.pStbName == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err2;
+  }
+
   pMeta->pCache = pCache;
   return code;
 
@@ -204,6 +219,7 @@ void metaCacheClose(SMeta* pMeta) {
     taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry);
 
     taosHashCleanup(pMeta->pCache->STbFilterCache.pStb);
+    taosHashCleanup(pMeta->pCache->STbFilterCache.pStbName);
 
     taosMemoryFree(pMeta->pCache);
     pMeta->pCache = NULL;
@@ -893,30 +909,59 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
   return TSDB_CODE_SUCCESS;
 }
 
-bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
+bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) {
-  SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
+  if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) {
+    return true;
+  }
 
-  if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) {
+  if (type == 1 && taosHashGet(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key))) {
     return true;
   }
 
   return false;
 }
 
-int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
+int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) {
-  SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
-
   if (type == 0) {
-    return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0);
+    return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0);
+  }
+
+  if (type == 1) {
+    return taosHashPut(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key), NULL, 0);
   }
 
   return 0;
 }
 
-int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) {
+int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type) {
-  SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
   if (type == 0) {
     return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb);
   }
   return 0;
 }
+
+int32_t metaInitTbFilterCache(SMeta* pMeta) {
+#ifdef TD_ENTERPRISE
+  int32_t      tbNum = 0;
+  const char** pTbArr = NULL;
+  const char*  dbName = NULL;
+
+  if (!(dbName = strchr(pMeta->pVnode->config.dbname, '.'))) return 0;
+  if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
+    tbNum = tkLogStbNum;
+    pTbArr = (const char**)&tkLogStb;
+  } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) {
+    tbNum = tkAuditStbNum;
+    pTbArr = (const char**)&tkAuditStb;
+  }
+  if (tbNum && pTbArr) {
+    for (int32_t i = 0; i < tbNum; ++i) {
+      if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) {
+        return terrno ? terrno : -1;
+      }
+    }
+  }
+#else
+#endif
+  return 0;
+}

source/dnode/vnode/src/meta/metaOpen.c

@@ -176,6 +176,10 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
     goto _err;
   }
 
+  if (metaInitTbFilterCache(pMeta) != 0) {
+    goto _err;
+  }
+
   metaDebug("vgId:%d, meta is opened", TD_VID(pVnode));
 
   *ppMeta = pMeta;

source/dnode/vnode/src/meta/metaTable.c

@@ -15,7 +15,7 @@
 
 #include "meta.h"
 
-extern tsem_t dmNotifySem;
+extern SDmNotifyHandle dmNotifyHdl;
 
 static int  metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
 static int  metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
@@ -28,7 +28,7 @@ static int  metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME);
 static int  metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
-static int  metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid);
+static int  metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl);
 static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey);
 // opt ins_tables query
 static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
@@ -198,7 +198,11 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) {
 #if defined(TD_ENTERPRISE) && !defined(_TD_DARWIN_64)
   int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0);
   int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries;
-  if (deltaTS > tsTimeSeriesThreshold) tsem_post(&dmNotifySem);
+  if (deltaTS > tsTimeSeriesThreshold) {
+    if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) {
+      tsem_post(&dmNotifyHdl.sem);
+    }
+  }
 #endif
 }
 
@@ -303,7 +307,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb
 
   for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) {
     tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild);
-    metaDropTableByUid(pMeta, uid, NULL, NULL);
+    metaDropTableByUid(pMeta, uid, NULL, NULL, NULL);
   }
 
   // drop super table
@@ -392,6 +396,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
   nStbEntry.stbEntry.schemaTag = pReq->schemaTag;
 
   int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols;
+  bool    updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1);
 
   metaWLock(pMeta);
   // compare two entry
@@ -407,15 +412,16 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
 
   // metaStatsCacheDrop(pMeta, nStbEntry.uid);
 
-  if (deltaCol != 0) {
+  if (updStat) {
     metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol);
   }
   metaULock(pMeta);
 
-  if (deltaCol != 0) {
+  if (updStat) {
     int64_t ctbNum;
     metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL);
     pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol);
+    metaTimeSeriesNotifyCheck(pMeta);
   }
 
 _exit:
@@ -760,6 +766,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
   }
   metaReaderClear(&mr);
 
+  bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1);
+
   // build SMetaEntry
   SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
   me.version = ver;
@@ -796,9 +804,12 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
 #endif
 
     ++pStats->numOfCTables;
+
+    if (!sysTbl) {
       int32_t nCols = 0;
       metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols);
       pStats->numOfTimeSeries += nCols - 1;
+    }
 
     metaWLock(pMeta);
     metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0);
@@ -852,6 +863,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
   int      rc = 0;
   tb_uid_t uid = 0;
   tb_uid_t suid = 0;
+  int8_t   sysTbl = 0;
   int      type;
 
   rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData);
@@ -862,12 +874,12 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
   uid = *(tb_uid_t *)pData;
 
   metaWLock(pMeta);
-  rc = metaDropTableByUid(pMeta, uid, &type, &suid);
+  rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl);
   metaULock(pMeta);
 
   if (rc < 0) goto _exit;
 
-  if (type == TSDB_CHILD_TABLE) {
+  if (!sysTbl && type == TSDB_CHILD_TABLE) {
     int32_t      nCols = 0;
     SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
     if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) {
@@ -898,9 +910,10 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) {
   for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
     tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i);
     tb_uid_t suid = 0;
+    int8_t   sysTbl = 0;
     int      type;
-    metaDropTableByUid(pMeta, uid, &type, &suid);
+    metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl);
-    if (type == TSDB_CHILD_TABLE && suid != 0 && suidHash) {
+    if (!sysTbl && type == TSDB_CHILD_TABLE && suid != 0 && suidHash) {
       int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t));
       if (pVal) {
         nCtbDropped = *pVal + 1;
@@ -1059,7 +1072,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
   return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx);
 }
 
-static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) {
+static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t* pSysTbl) {
   void      *pData = NULL;
   int        nData = 0;
   int        rc = 0;
@@ -1088,7 +1101,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
     void *tData = NULL;
     int   tLen = 0;
 
-
     if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
       STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version};
       if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
@@ -1098,6 +1110,8 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
         tDecoderInit(&tdc, tData, tLen);
         metaDecodeEntry(&tdc, &stbEntry);
 
+        if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0;
+
         SSchema        *pTagColumn = NULL;
         SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
         if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) {

source/dnode/vnode/src/vnd/vnodeQuery.c

@@ -566,8 +566,7 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
 }
 
 #ifdef TD_ENTERPRISE
-#define TK_LOG_STB_NUM 19
+const char *tkLogStb[] = {"cluster_info",
-static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
                           "data_dir",
                           "dnodes_info",
                           "d_info",
@@ -586,22 +585,36 @@ static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
                           "temp_dir",
                           "vgroups_info",
                           "vnodes_role"};
+const char *tkAuditStb[] = {"operations"};
+const int   tkLogStbNum = ARRAY_SIZE(tkLogStb);
+const int   tkAuditStbNum = ARRAY_SIZE(tkAuditStb);
 
 // exclude stbs of taoskeeper log
 static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) {
-  char *dbName = strchr(pVnode->config.dbname, '.');
+  int32_t      tbSize = 0;
-  if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
+  int32_t      tbNum = 0;
-    return 0;
+  const char **pTbArr = NULL;
+  const char  *dbName = NULL;
+
+  if (!(dbName = strchr(pVnode->config.dbname, '.'))) return 0;
+  if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
+    tbNum = tkLogStbNum;
+    pTbArr = (const char **)&tkLogStb;
+  } else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) {
+    tbNum = tkAuditStbNum;
+    pTbArr = (const char **)&tkAuditStb;
   }
-  int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0);
+  if (tbNum && pTbArr) {
-  if (tbSize < TK_LOG_STB_NUM) {
+    tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0);
-    for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) {
+    if (tbSize < tbNum) {
-      tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]);
+      for (int32_t i = 0; i < tbNum; ++i) {
+        tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, pTbArr[i]);
         if (suid != 0) {
-        metaPutTbToFilterCache(pVnode, suid, 0);
+          metaPutTbToFilterCache(pVnode->pMeta, &suid, 0);
         }
       }
-    tbSize = metaSizeOfTbFilterCache(pVnode, 0);
+      tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0);
+    }
   }
 
   return tbSize;
@@ -611,7 +624,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) {
 static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) {
   SVnode *pVnode = (SVnode *)arg1;
 
-  if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) {
+  if (metaTbInFilterCache(pVnode->pMeta, arg2, 0)) {
     return true;
   }
   return false;
@@ -626,9 +639,9 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
   }
 
   int32_t tbFilterSize = 0;
-  #ifdef TD_ENTERPRISE
+#ifdef TD_ENTERPRISE
   tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode);
-  #endif
+#endif
 
   if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) ||
       (tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) {

source/libs/parser/src/parTranslater.c

@@ -13,8 +13,8 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "parInt.h"
 #include "parTranslater.h"
+#include "parInt.h"
 
 #include "catalog.h"
 #include "cmdnodes.h"
@@ -1209,37 +1209,37 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
       break;
     }
     case TSDB_DATA_TYPE_VARBINARY: {
-      if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY){
+      if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY) {
         return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
       }
 
       void*    data = NULL;
       uint32_t size = 0;
       bool     isHexChar = isHex(pVal->literal, strlen(pVal->literal));
-      if(isHexChar){
+      if (isHexChar) {
-        if(!isValidateHex(pVal->literal, strlen(pVal->literal))){
+        if (!isValidateHex(pVal->literal, strlen(pVal->literal))) {
           return TSDB_CODE_PAR_INVALID_VARBINARY;
         }
-        if(taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0){
+        if (taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0) {
           return TSDB_CODE_OUT_OF_MEMORY;
         }
-      }else{
+      } else {
         size = pVal->node.resType.bytes;
         data = pVal->literal;
       }
 
       if (size + VARSTR_HEADER_SIZE > targetDt.bytes) {
-        if(isHexChar) taosMemoryFree(data);
+        if (isHexChar) taosMemoryFree(data);
         return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_VALUE_TOO_LONG, pVal->literal);
       }
       pVal->datum.p = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE);
       if (NULL == pVal->datum.p) {
-        if(isHexChar) taosMemoryFree(data);
+        if (isHexChar) taosMemoryFree(data);
         return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
       }
       varDataSetLen(pVal->datum.p, size);
       memcpy(varDataVal(pVal->datum.p), data, size);
-      if(isHexChar)  taosMemoryFree(data);
+      if (isHexChar) taosMemoryFree(data);
       break;
     }
     case TSDB_DATA_TYPE_VARCHAR:
@@ -1789,7 +1789,6 @@ static int32_t translateBlockDistFunc(STranslateContext* pCtx, SFunctionNode* pF
   return TSDB_CODE_SUCCESS;
 }
 
-
 static bool isStarParam(SNode* pNode) { return nodesIsStar(pNode) || nodesIsTableStar(pNode); }
 
 static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
@@ -2811,7 +2810,8 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
         pJoinTable->table.precision = calcJoinTablePrecision(pJoinTable);
         pJoinTable->table.singleTable = joinTableIsSingleTable(pJoinTable);
         code = translateExpr(pCxt, &pJoinTable->pOnCond);
-        pJoinTable->hasSubQuery = (nodeType(pJoinTable->pLeft) != QUERY_NODE_REAL_TABLE) || (nodeType(pJoinTable->pRight) != QUERY_NODE_REAL_TABLE);
+        pJoinTable->hasSubQuery = (nodeType(pJoinTable->pLeft) != QUERY_NODE_REAL_TABLE) ||
+                                  (nodeType(pJoinTable->pRight) != QUERY_NODE_REAL_TABLE);
         if (nodeType(pJoinTable->pLeft) == QUERY_NODE_JOIN_TABLE) {
           ((SJoinTableNode*)pJoinTable->pLeft)->isLowLevelJoin = true;
         }
@@ -2992,11 +2992,8 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 #ifndef TD_ENTERPRISE
-int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
+int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; }
-  return TSDB_CODE_SUCCESS;
-}
 #endif
 
 static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
@@ -3712,7 +3709,8 @@ static int32_t removeConstantValueFromList(SNodeList** pList) {
   SNode* pNode = NULL;
   WHERE_EACH(pNode, *pList) {
     if (nodeType(pNode) == QUERY_NODE_VALUE ||
-        (nodeType(pNode) == QUERY_NODE_FUNCTION && fmIsConstantResFunc((SFunctionNode*)pNode) && fmIsScalarFunc(((SFunctionNode*)pNode)->funcId))) {
+        (nodeType(pNode) == QUERY_NODE_FUNCTION && fmIsConstantResFunc((SFunctionNode*)pNode) &&
+         fmIsScalarFunc(((SFunctionNode*)pNode)->funcId))) {
       ERASE_NODE(*pList);
       continue;
     }
@@ -4557,9 +4555,9 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
     daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile);
     daysToKeep0 = (-1 == daysToKeep0 ? dbCfg.daysToKeep0 : daysToKeep0);
   }
-  if (daysPerFile > daysToKeep0) {
+  if (daysPerFile > daysToKeep0 / 3) {
     return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
-                                   "Invalid duration value, should be keep2 >= keep1 >= keep0 >= duration");
+                                   "Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration");
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -8018,9 +8016,29 @@ static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode* pCond) {
   if (pSelect->pWhere == NULL) {
     pSelect->pWhere = pCond;
   } else {
-    SNode* pWhere = NULL;
+    SNodeList* pLogicCondListWhere = NULL;
-    createLogicCondNode(pSelect->pWhere, pCond, &pWhere, LOGIC_COND_TYPE_AND);
+    SNodeList* pLogicCondList2 = NULL;
-    pSelect->pWhere = pWhere;
+    if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION &&
+        ((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) {
+        pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList;
+    } else {
+      nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere);
+    }
+
+    if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION &&
+        ((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) {
+         pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList;
+    } else {
+      nodesListMakeAppend(&pLogicCondList2, pCond);
+    }
+
+    nodesListAppendList(pLogicCondListWhere, pLogicCondList2);
+
+    SLogicConditionNode* pWhere = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
+    pWhere->condType = LOGIC_COND_TYPE_AND;
+    pWhere->pParameterList = pLogicCondListWhere;
+
+    pSelect->pWhere = (SNode*)pWhere;
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -8094,7 +8112,6 @@ static int32_t addShowKindCond(const SShowStmt* pShow, SSelectStmt* pSelect) {
   return TSDB_CODE_SUCCESS;
 }
 
-
 static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) {
   SNode* pDbCond = NULL;
   SNode* pTbCond = NULL;

source/util/src/terror.c

@@ -430,6 +430,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED,      "Query time limited by
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED,            "CPU cores limited by license")
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STABLE_LIMITED,         "STable creation limited by license")
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED,          "Table creation limited by license")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_ACTIVE,        "Invalid active code")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_KEY,           "Invalid key to parse active code")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KEY,       "Invalid key to decode active code")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN,      "Invalid klen to decode active code")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_IVLD_KEY,           "Invalid key to gen active code")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT,          "Limited app num to gen active code")
+TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN,      "Invalid klen to encode active code")
 
 // sync
 TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT,                  "Sync timeout")

tests/parallel_test/cases.task

@@ -815,6 +815,7 @@
 ,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py
 ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3
 ,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3
+,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py
 
 #tsim test
 ,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim

tests/script/tsim/parser/alter.sim

@@ -17,7 +17,7 @@ $db = $dbPrefix . $i
 $mt = $mtPrefix . $i
 
 sql drop database if exists $db
-sql create database $db duration 10 keep 20,20,20
+sql create database $db duration 3 keep 20,20,20
 sql use $db
 
 sql_error alter database $db keep "20"
@@ -27,8 +27,8 @@ sql_error alter database $db keep 20.0
 sql_error alter database $db keep 20.0,20.0,20.0
 sql_error alter database $db keep 0,0,0
 sql_error alter database $db keep -1,-1,-1
-sql_error alter database $db keep 9,20
+sql_error alter database $db keep 8,20
-sql_error alter database $db keep 9,9,9
+sql_error alter database $db keep 8,9,9
 sql_error alter database $db keep 20,20,19
 sql_error alter database $db keep 20,19,20
 sql_error alter database $db keep 20,19,19

tests/script/tsim/parser/alter__for_community_version.sim

@@ -17,7 +17,7 @@ $db = $dbPrefix . $i
 $mt = $mtPrefix . $i
 
 sql drop database if exists $db
-sql create database $db duration 10 keep 20
+sql create database $db duration 3 keep 20
 sql use $db
 sql select * from information_schema.ins_databases
 if $rows != 3 then
@@ -36,7 +36,7 @@ sql_error alter database $db keep 0,0,0
 sql_error alter database $db keep 3
 sql_error alter database $db keep -1,-1,-1
 sql alter database $db keep 20,20
-sql_error alter database $db keep 9,9,9
+sql_error alter database $db keep 8,9,9
 sql_error alter database $db keep 20,20,19
 sql_error alter database $db keep 20,19,20
 sql_error alter database $db keep 20,19,19

tests/script/tsim/parser/create_db.sim

@@ -150,26 +150,26 @@ sql_error create database $db keep 12,11,12
 sql_error create database $db keep 8
 sql_error create database $db keep 12,11
 sql_error create database $db keep 365001,365001,365001
-sql create database dbk0 keep 19
+sql create database dbk0 keep 39
 sql select * from information_schema.ins_databases
 if $rows != 3 then
   return -1
 endi
-if $data27 != 27360m,27360m,27360m then
+if $data27 != 56160m,56160m,56160m then
  return -1
 endi
 sql drop database dbk0
-sql create database dbka keep 19,20
+sql create database dbka keep 39,40
 sql select * from information_schema.ins_databases
 if $rows != 3 then
   return -1
 endi
-if $data27 != 27360m,28800m,28800m then
+if $data27 != 56160m,57600m,57600m then
  return -1
 endi
 sql drop database dbka
 
-sql create database dbk1 keep 11,11,11
+sql create database dbk1 duration 3 keep 11,11,11
 sql select * from information_schema.ins_databases
 if $rows != 3 then
   return -1
@@ -178,7 +178,7 @@ if $data27 != 15840m,15840m,15840m then
  return -1
 endi
 sql drop database dbk1
-sql create database dbk2 keep 11,12,13
+sql create database dbk2 duration 3 keep 11,12,13
 sql select * from information_schema.ins_databases
 if $rows != 3 then
   return -1
@@ -187,7 +187,7 @@ if $data27 !=  15840m,17280m,18720m then
  return -1
 endi
 sql drop database dbk2
-sql create database dbk3 keep 11,11,13
+sql create database dbk3 duration 3 keep 11,11,13
 sql select * from information_schema.ins_databases
 if $rows != 3 then
   return -1
@@ -196,7 +196,7 @@ if $data27 != 15840m,15840m,18720m then
  return -1
 endi
 sql drop database dbk3
-sql create database dbk4 keep 11,13,13
+sql create database dbk4 duration 3 keep 11,13,13
 sql select * from information_schema.ins_databases
 if $rows != 3 then
   return -1

tests/script/tsim/query/show_db_table_kind.sim

@@ -94,4 +94,14 @@ print $rows
 if $rows != 5 then
   return -1
 endi
+sql show child db2.tables like '%'
+print $rows
+if $rows != 5 then
+  return -1
+endi
+sql show normal db2.tables like '%'
+print $rows
+if $rows != 0 then
+  return -1
+endi
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/system-test/0-others/information_schema.py

@@ -242,11 +242,11 @@ class TDTestCase:
         self.str510 = self.str255 + self.str255
         tdSql.error('alter dnode 1 "activeCode" "a"')
         tdSql.error('alter dnode 1 "activeCode" "' + self.str107 + '"')
-        tdSql.execute('alter all dnodes "activeCode" "' + self.str108 + '"')
+        # tdSql.error('alter all dnodes "activeCode" "' + self.str108 + '"')
         tdSql.error('alter dnode 1 "activeCode" "' + self.str109 + '"')
         tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"')
         tdSql.query(f'select * from information_schema.ins_dnodes')
-        tdSql.checkEqual(tdSql.queryResult[0][8],self.str108)
+        tdSql.checkEqual(tdSql.queryResult[0][8],"")
         tdSql.execute('alter dnode 1 "activeCode" ""')
         tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes')
         tdSql.checkEqual(tdSql.queryResult[0][0],"")
@@ -257,14 +257,14 @@ class TDTestCase:
         tdSql.error('alter all dnodes "cActiveCode" "' + self.str255 + '"')
         tdSql.error('alter all dnodes "cActiveCode" "' + self.str256 + '"')
         tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"')
-        tdSql.execute('alter all dnodes "cActiveCode" "' + self.str254 + '"')
+        # tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"')
         tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"')
         tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes')
         tdSql.checkEqual(tdSql.queryResult[0][0],"")
-        tdSql.checkEqual(tdSql.queryResult[0][1],self.str254)
+        tdSql.checkEqual(tdSql.queryResult[0][1],"")
-        tdSql.execute('alter dnode 1 "cActiveCode" "' + self.str109 + '"')
+        # tdSql.error('alter dnode 1 "cActiveCode" "' + self.str109 + '"')
         tdSql.query(f'show dnodes')
-        tdSql.checkEqual(tdSql.queryResult[0][9],self.str109)
+        tdSql.checkEqual(tdSql.queryResult[0][9],"")
         tdSql.execute('alter all dnodes "cActiveCode" ""')
         tdSql.query(f'select c_active_code from information_schema.ins_dnodes')
         tdSql.checkEqual(tdSql.queryResult[0][0],'')

tests/system-test/0-others/multilevel.py

@@ -100,7 +100,7 @@ class TDTestCase:
         tdDnodes.deploy(1,cfg)
         tdDnodes.start(1)
 
-        tdSql.execute("create database test duration 1")
+        tdSql.execute("create database test duration 8h")
         tdSql.execute("use test")
 
         tdSql.execute("create table stb(ts timestamp, c int) tags(t int)")
@@ -117,7 +117,7 @@ class TDTestCase:
         tdLog.info("================= step3")
         tdSql.execute('drop database test')
         for i in range(10):
-            tdSql.execute("create database test%d duration 1" %(i))
+            tdSql.execute("create database test%d duration 8h" %(i))
             tdSql.execute("use test%d" %(i))
             tdSql.execute("create table tb (ts timestamp,i int)") 
             for j in range(10):           
@@ -216,7 +216,7 @@ class TDTestCase:
         tdDnodes.deploy(1,cfg)
         tdDnodes.start(1)
 
-        tdSql.execute('create database dbtest')
+        tdSql.execute('create database dbtest duration 3')
         tdSql.execute('use dbtest')
         tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)')
         tdSql.execute('create table tb1 using stb tags(1)')

tests/system-test/1-insert/keep_expired.py

@@ -15,7 +15,7 @@ class TDTestCase:
         self.stbname = "stb"
         self.ctbname = "ctb"
         self.keep_value = "2d,2d,2d"
-        self.duration_value = "1d"
+        self.duration_value = "16h"
         self.offset_time = 5
         self.sleep_time = self.offset_time*2
 

tests/system-test/eco-system/meta/database/keep_time_offset.py

@@ -0,0 +1,66 @@
+###################################################################
+#           Copyright (c) 2016 by TAOS Technologies, Inc.
+#                     All rights reserved.
+#
+#  This file is proprietary and confidential to TAOS Technologies.
+#  No part of this file may be reproduced, stored, transmitted,
+#  disclosed or used in any form or by any means other than as
+#  expressly provided by the written permission from Jianhui Tao
+#
+###################################################################
+
+# -*- coding: utf-8 -*-
+
+import re
+from util.log import *
+from util.cases import *
+from util.sql import *
+from util.common import *
+from util.sqlset import *
+
+class TDTestCase:
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug("start to execute %s" % __file__)
+        tdSql.init(conn.cursor())
+        self.setsql = TDSetSql()
+
+    def create_db(self):
+        hours = 8
+        # create
+        keep_str = f"KEEP_TIME_OFFSET {hours}"
+        tdSql.execute(f"create database db {keep_str}")
+
+        # check result
+        tdSql.query("select `keep_time_offset` from  information_schema.ins_databases where name='db'")
+        tdSql.checkData(0, 0, hours)
+
+        # alter
+        hours = 4
+        keep_str = f"KEEP_TIME_OFFSET {hours}"
+        tdSql.execute(f"alter database db {keep_str}")
+
+        # check result
+        tdSql.query("select `keep_time_offset` from  information_schema.ins_databases where name='db'")
+        tdSql.checkData(0, 0, hours)
+
+
+    def check_old_syntax(self):
+        # old syntax would not support again
+        tdSql.error("alter dnode 1 'keeptimeoffset 10';")
+
+
+    def run(self):
+        # check new syntax right
+        self.create_db()
+
+        # check old syntax error
+        self.check_old_syntax()
+        
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success("%s successfully executed" % __file__)
+
+tdCases.addWindows(__file__, TDTestCase())
+tdCases.addLinux(__file__, TDTestCase())

utils/TSZ/sz/src/Huffman.c

@@ -125,6 +125,9 @@ void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, u
 		huffmanTree->code[n->c] = (unsigned long*)malloc(2*sizeof(unsigned long));
 		if(len<=64)
 		{
+			if(len == 0)
+			  (huffmanTree->code[n->c])[0] = 0;
+			else 
 			  (huffmanTree->code[n->c])[0] = out1 << (64 - len);
 			(huffmanTree->code[n->c])[1] = out2;
 		}