Merge pull request #21007 from taosdata/feat/TS-3249

feat: support table_prefix/table_suffix new mode

include/common/ttokendef.h

@@ -346,6 +346,7 @@
 #define TK_VIEW                           328
 #define TK_WAL                            329
 
+
 #define TK_NK_SPACE   600
 #define TK_NK_COMMENT 601
 #define TK_NK_ILLEGAL 602

include/util/tdef.h

@@ -368,11 +368,11 @@ typedef enum ELogicConditionType {
 #define TSDB_MIN_STT_TRIGGER            1
 #define TSDB_MAX_STT_TRIGGER            16
 #define TSDB_DEFAULT_SST_TRIGGER        1
-#define TSDB_MIN_HASH_PREFIX            0
+#define TSDB_MIN_HASH_PREFIX            (2 - TSDB_TABLE_NAME_LEN)
-#define TSDB_MAX_HASH_PREFIX            128
+#define TSDB_MAX_HASH_PREFIX            (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_PREFIX        0
-#define TSDB_MIN_HASH_SUFFIX            0
+#define TSDB_MIN_HASH_SUFFIX            (2 - TSDB_TABLE_NAME_LEN)
-#define TSDB_MAX_HASH_SUFFIX            128
+#define TSDB_MAX_HASH_SUFFIX            (TSDB_TABLE_NAME_LEN - 2)
 #define TSDB_DEFAULT_HASH_SUFFIX        0
 
 #define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1

include/util/tutil.h

@@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
 
 static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
                                              int32_t suffix) {
-  if (prefix == 0 && suffix == 0) {
+  if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) {
     return MurmurHash3_32(tbname, tblen);
-  } else {
+  } else if (prefix > 0 || suffix > 0) {
-    if (tblen <= (prefix + suffix)) {
-      return MurmurHash3_32(tbname, tblen);
-    } else {
     return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
+  } else {
+    char tbName[TSDB_TABLE_FNAME_LEN];
+    int32_t offset = 0;
+    if (prefix < 0) {
+      offset = -1 * prefix;
+      strncpy(tbName, tbname, offset);
     }
+    if (suffix < 0) {
+      strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix);
+      offset += -1 *suffix;
+    }
+    return MurmurHash3_32(tbName, offset);
   }
 }
 

source/dnode/mnode/impl/src/mndDb.c

@@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
   if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
   if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
   if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
+  if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1;
+  if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1;
   if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
   if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
 
@@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
   if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
   if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
   if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
-  if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
-  if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
   if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
 }
 
@@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
     int32_t dbLen = strlen(dbObj.name) + 1;
     mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen);
     dbObj.cfg.hashPrefix += dbLen;
+  } else if (dbObj.cfg.hashPrefix < 0) {
+    int32_t dbLen = strlen(dbObj.name) + 1;
+    mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen);
+    dbObj.cfg.hashPrefix -= dbLen;
   }
 
   SVgObj *pVgroups = NULL;
@@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
     int16_t hashPrefix = pDb->cfg.hashPrefix;
     if (hashPrefix > 0) {
       hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
+    } else if (hashPrefix < 0) {
+      hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1;
     }
     colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false);
 

source/libs/command/src/command.c

@@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
 
   char* retentions = buildRetension(pCfg->pRetensions);
   int32_t dbFNameLen = strlen(dbFName);
-  int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0;
+  int32_t hashPrefix = 0;
+  if (pCfg->hashPrefix > 0) {
+    hashPrefix = pCfg->hashPrefix - dbFNameLen - 1;
+  } else if (pCfg->hashPrefix < 0) {
+    hashPrefix = pCfg->hashPrefix + dbFNameLen + 1;
+  }
 
   len += sprintf(
       buf2 + VARSTR_HEADER_SIZE,

source/libs/parser/inc/sql.y

@@ -221,8 +221,8 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C).
 db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C).                    { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); }
 db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C).                   { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); }
 db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C).                        { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
-db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C).                       { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); }
+db_options(A) ::= db_options(B) TABLE_PREFIX signed(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
-db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C).                       { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); }
+db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C).                           { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
 
 alter_db_options(A) ::= alter_db_option(B).                                       { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
 alter_db_options(A) ::= alter_db_options(B) alter_db_option(C).                   { A = setAlterDatabaseOption(pCxt, B, &C); }

source/libs/parser/src/parAstCreater.c

@@ -1024,12 +1024,28 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
     case DB_OPTION_STT_TRIGGER:
       pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
       break;
-    case DB_OPTION_TABLE_PREFIX:
+    case DB_OPTION_TABLE_PREFIX: {
-      pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+      SValueNode *pNode = (SValueNode *)pVal;
+      if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
+        pDbOptions->tablePrefix = taosStr2Int32(pNode->literal, NULL, 10);
+      } else {
+        snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type");
+        pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;        
+      }
+      nodesDestroyNode((SNode*)pNode);
       break;
-    case DB_OPTION_TABLE_SUFFIX:
+    }
-      pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
+    case DB_OPTION_TABLE_SUFFIX:{
+      SValueNode *pNode = (SValueNode *)pVal;
+      if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
+        pDbOptions->tableSuffix = taosStr2Int32(pNode->literal, NULL, 10);
+      } else {
+        snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type");
+        pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;        
+      }
+      nodesDestroyNode((SNode*)pNode);
       break;
+    }
     default:
       break;
   }

source/libs/parser/src/parTranslater.c

@@ -4177,6 +4177,34 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
   return TSDB_CODE_SUCCESS;
 }
 
+
+static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) {
+  if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_prefix: %d valid range: [%d, %d]", tbPrefix,
+                                   TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
+  }
+
+  if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_suffix: %d valid range: [%d, %d]", tbSuffix,
+                                   TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
+  }
+
+  if ((tbPrefix * tbSuffix) < 0) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_prefix & table_suffix: mixed usage not allowed");
+  }
+
+  if ((tbPrefix + tbSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) {
+    return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
+                                   "Invalid option table_prefix & table_suffix: exceed max table name length");
+  }
+
+  return TSDB_CODE_SUCCESS;
+}
+
+
 static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
   int32_t daysPerFile = pOptions->daysPerFile;
   int64_t daysToKeep0 = pOptions->keep[0];
@@ -4284,10 +4312,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
     code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER);
   }
   if (TSDB_CODE_SUCCESS == code) {
-    code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
+    code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
-  }
-  if (TSDB_CODE_SUCCESS == code) {
-    code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
   }
   if (TSDB_CODE_SUCCESS == code) {
     code = checkOptionsDependency(pCxt, pDbName, pOptions);

tests/parallel_test/cases.task

@@ -746,6 +746,7 @@
 ,,y,script,./test.sh -f tsim/db/show_create_table.sim
 ,,y,script,./test.sh -f tsim/db/tables.sim
 ,,y,script,./test.sh -f tsim/db/taosdlog.sim
+,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim
 ,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim
 ,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim
 ,,y,script,./test.sh -f tsim/dnode/balance1.sim

tests/script/tsim/db/table_prefix_suffix.sim

@@ -0,0 +1,182 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+sql drop database if exists db1;
+sql create database db1 vgroups 5 TABLE_PREFIX 1 TABLE_SUFFIX 2;
+sql use db1;
+sql create table atb1aa (ts timestamp, f1 int);
+sql create table btb1bb (ts timestamp, f1 int);
+sql create table ctb1cc (ts timestamp, f1 int);
+sql create table dtb1dd (ts timestamp, f1 int);
+sql create table atb2aa (ts timestamp, f1 int);
+sql create table btb2bb (ts timestamp, f1 int);
+sql create table ctb2cc (ts timestamp, f1 int);
+sql create table dtb2dd (ts timestamp, f1 int);
+sql create table etb2ee (ts timestamp, f1 int);
+sql show create database db1;
+sql select count(*) a from information_schema.ins_tables where db_name='db1' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db1;
+ 
+sql drop database if exists db2;
+sql create database db2 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX -2;
+sql use db2;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table tbbb11 (ts timestamp, f1 int);
+sql create table tccc11 (ts timestamp, f1 int);
+sql create table tddd11 (ts timestamp, f1 int);
+sql create table taaa22 (ts timestamp, f1 int);
+sql create table tbbb22 (ts timestamp, f1 int);
+sql create table tccc22 (ts timestamp, f1 int);
+sql create table tddd22 (ts timestamp, f1 int);
+sql create table teee22 (ts timestamp, f1 int);
+sql show create database db2;
+sql select count(*) a from information_schema.ins_tables where db_name='db2' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db2;
+ 
+sql drop database if exists db3;
+sql create database db3 vgroups 5 TABLE_PREFIX -1;
+sql use db3;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table tbbb11 (ts timestamp, f1 int);
+sql create table tccc11 (ts timestamp, f1 int);
+sql create table tddd11 (ts timestamp, f1 int);
+sql create table zaaa22 (ts timestamp, f1 int);
+sql create table zbbb22 (ts timestamp, f1 int);
+sql create table zccc22 (ts timestamp, f1 int);
+sql create table zddd22 (ts timestamp, f1 int);
+sql create table zeee22 (ts timestamp, f1 int);
+sql show create database db3;
+sql select count(*) a from information_schema.ins_tables where db_name='db3' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db3;
+ 
+sql drop database if exists db4;
+sql create database db4 vgroups 5 TABLE_SUFFIX -2;
+sql use db4;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table tbbb11 (ts timestamp, f1 int);
+sql create table tccc11 (ts timestamp, f1 int);
+sql create table tddd11 (ts timestamp, f1 int);
+sql create table zaaa22 (ts timestamp, f1 int);
+sql create table zbbb22 (ts timestamp, f1 int);
+sql create table zccc22 (ts timestamp, f1 int);
+sql create table zddd22 (ts timestamp, f1 int);
+sql create table zeee22 (ts timestamp, f1 int);
+sql show create database db4;
+sql select count(*) a from information_schema.ins_tables where db_name='db4' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db4;
+ 
+sql drop database if exists db5;
+sql create database db5 vgroups 5 TABLE_PREFIX 1;
+sql use db5;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table baaa11 (ts timestamp, f1 int);
+sql create table caaa11 (ts timestamp, f1 int);
+sql create table daaa11 (ts timestamp, f1 int);
+sql create table faaa11 (ts timestamp, f1 int);
+sql create table gbbb11 (ts timestamp, f1 int);
+sql create table hbbb11 (ts timestamp, f1 int);
+sql create table ibbb11 (ts timestamp, f1 int);
+sql create table jbbb11 (ts timestamp, f1 int);
+sql show create database db5;
+sql select count(*) a from information_schema.ins_tables where db_name='db5' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db5;
+ 
+sql drop database if exists db6;
+sql create database db6 vgroups 5 TABLE_SUFFIX 2;
+sql use db6;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table taaa12 (ts timestamp, f1 int);
+sql create table taaa13 (ts timestamp, f1 int);
+sql create table taaa14 (ts timestamp, f1 int);
+sql create table tbbb23 (ts timestamp, f1 int);
+sql create table tbbb24 (ts timestamp, f1 int);
+sql create table tbbb31 (ts timestamp, f1 int);
+sql create table tbbb32 (ts timestamp, f1 int);
+sql create table tbbb33 (ts timestamp, f1 int);
+sql show create database db6;
+sql select count(*) a from information_schema.ins_tables where db_name='db6' group by vgroup_id having(count(*) > 0) order by a;
+if $rows != 2 then
+  return -1
+endi
+if $data00 != 4 then
+  return -1
+endi
+if $data10 != 5 then
+  return -1
+endi
+sql drop database if exists db6;
+ 
+sql drop database if exists db7;
+sql create database db7 vgroups 5 TABLE_PREFIX -100 TABLE_SUFFIX -92;
+sql use db7;
+sql create table taaa11 (ts timestamp, f1 int);
+sql create table taaa12 (ts timestamp, f1 int);
+sql create table taaa13 (ts timestamp, f1 int);
+sql create table tbbb21 (ts timestamp, f1 int);
+sql create table tbbb22 (ts timestamp, f1 int);
+sql create table tbbb23 (ts timestamp, f1 int);
+sql create table tbbb24 (ts timestamp, f1 int);
+sql create table tccc31 (ts timestamp, f1 int);
+sql create table tccc32 (ts timestamp, f1 int);
+sql create table tccc33 (ts timestamp, f1 int);
+sql create table tddd24 (ts timestamp, f1 int);
+sql create table tddd31 (ts timestamp, f1 int);
+sql create table tddd32 (ts timestamp, f1 int);
+sql create table tddd33 (ts timestamp, f1 int);
+sql show create database db7;
+sql select count(*) a from information_schema.ins_tables where db_name='db7' group by vgroup_id having(count(*) > 0) order by a;
+sql drop database if exists db7;
+
+sql_error create database db8 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX 2;
+sql_error create database db8 vgroups 5 TABLE_PREFIX 191 TABLE_SUFFIX 192;
+sql_error create database db8 vgroups 5 TABLE_PREFIX -192 TABLE_SUFFIX -191;
+sql_error create database db8 vgroups 5 TABLE_PREFIX 100 TABLE_SUFFIX 92;
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT