xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #21007 from taosdata/feat/TS-3249 

feat: support table_prefix/table_suffix new mode
This commit is contained in:
dapan1121 2023-04-21 19:18:28 +08:00 committed by GitHub
parent 3f432cca2e 176f7740c3
commit f4d1ac4e93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 2662 additions and 1802 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/common/ttokendef.h
View File

@ -346,6 +346,7 @@
#define TK_VIEW 328
#define TK_WAL 329
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601
#define TK_NK_ILLEGAL 602

8
include/util/tdef.h
View File

@ -368,11 +368,11 @@ typedef enum ELogicConditionType {
#define TSDB_MIN_STT_TRIGGER 1
#define TSDB_MAX_STT_TRIGGER 16
#define TSDB_DEFAULT_SST_TRIGGER 1
#define TSDB_MIN_HASH_PREFIX 0
#define TSDB_MAX_HASH_PREFIX 128
#define TSDB_MIN_HASH_PREFIX (2 - TSDB_TABLE_NAME_LEN)
#define TSDB_MAX_HASH_PREFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_PREFIX 0
#define TSDB_MIN_HASH_SUFFIX 0
#define TSDB_MAX_HASH_SUFFIX 128
#define TSDB_MIN_HASH_SUFFIX (2 - TSDB_TABLE_NAME_LEN)
#define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2)
#define TSDB_DEFAULT_HASH_SUFFIX 0
#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1

18
include/util/tutil.h
View File

@ -81,14 +81,22 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar
static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix,
int32_t suffix) {
if (prefix == 0 && suffix == 0) {
if ((prefix == 0 && suffix == 0) || (tblen <= (prefix + suffix)) || (tblen <= -1 * (prefix + suffix)) || prefix * suffix < 0) {
return MurmurHash3_32(tbname, tblen);
} else {
if (tblen <= (prefix + suffix)) {
return MurmurHash3_32(tbname, tblen);
} else {
} else if (prefix > 0 || suffix > 0) {
return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix);
} else {
char tbName[TSDB_TABLE_FNAME_LEN];
int32_t offset = 0;
if (prefix < 0) {
offset = -1 * prefix;
strncpy(tbName, tbname, offset);
}
if (suffix < 0) {
strncpy(tbName + offset, tbname + tblen + suffix, -1 * suffix);
offset += -1 *suffix;
}
return MurmurHash3_32(tbName, offset);
}
}

10
source/dnode/mnode/impl/src/mndDb.c
View File

@ -373,6 +373,8 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;
if ((pCfg->hashSuffix * pCfg->hashPrefix) < 0) return -1;
if ((pCfg->hashPrefix + pCfg->hashSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) return -1;
if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1;
if (taosArrayGetSize(pCfg->pRetensions) != pCfg->numOfRetensions) return -1;
@ -409,8 +411,6 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) {
if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD;
if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE;
if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER;
if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX;
if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX;
if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE;
}
@ -553,6 +553,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
int32_t dbLen = strlen(dbObj.name) + 1;
mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen);
dbObj.cfg.hashPrefix += dbLen;
} else if (dbObj.cfg.hashPrefix < 0) {
int32_t dbLen = strlen(dbObj.name) + 1;
mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix - dbLen);
dbObj.cfg.hashPrefix -= dbLen;
}
SVgObj *pVgroups = NULL;
@ -1788,6 +1792,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb,
int16_t hashPrefix = pDb->cfg.hashPrefix;
if (hashPrefix > 0) {
hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1;
} else if (hashPrefix < 0) {
hashPrefix = pDb->cfg.hashPrefix + strlen(pDb->name) + 1;
}
colDataSetVal(pColInfo, rows, (const char *)&hashPrefix, false);

7
source/libs/command/src/command.c
View File

@ -278,7 +278,12 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbName, ch
char* retentions = buildRetension(pCfg->pRetensions);
int32_t dbFNameLen = strlen(dbFName);
int32_t hashPrefix = (pCfg->hashPrefix > (dbFNameLen + 1)) ? (pCfg->hashPrefix - dbFNameLen - 1) : 0;
int32_t hashPrefix = 0;
if (pCfg->hashPrefix > 0) {
hashPrefix = pCfg->hashPrefix - dbFNameLen - 1;
} else if (pCfg->hashPrefix < 0) {
hashPrefix = pCfg->hashPrefix + dbFNameLen + 1;
}
len += sprintf(
buf2 + VARSTR_HEADER_SIZE,

4
source/libs/parser/inc/sql.y Normal file → Executable file
View File

@ -221,8 +221,8 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C).
db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); }
db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); }
db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); }
db_options(A) ::= db_options(B) TABLE_PREFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, C); }
db_options(A) ::= db_options(B) TABLE_SUFFIX signed(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, C); }
alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); }
alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); }

24
source/libs/parser/src/parAstCreater.c
View File

@ -1024,12 +1024,28 @@ static SNode* setDatabaseOptionImpl(SAstCreateContext* pCxt, SNode* pOptions, ED
case DB_OPTION_STT_TRIGGER:
pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
break;
case DB_OPTION_TABLE_PREFIX:
pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
case DB_OPTION_TABLE_PREFIX: {
SValueNode *pNode = (SValueNode *)pVal;
if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
pDbOptions->tablePrefix = taosStr2Int32(pNode->literal, NULL, 10);
} else {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_prefix data type");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
}
nodesDestroyNode((SNode*)pNode);
break;
case DB_OPTION_TABLE_SUFFIX:
pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10);
}
case DB_OPTION_TABLE_SUFFIX:{
SValueNode *pNode = (SValueNode *)pVal;
if (TSDB_DATA_TYPE_BIGINT == pNode->node.resType.type || TSDB_DATA_TYPE_UBIGINT == pNode->node.resType.type) {
pDbOptions->tableSuffix = taosStr2Int32(pNode->literal, NULL, 10);
} else {
snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "invalid table_suffix data type");
pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR;
}
nodesDestroyNode((SNode*)pNode);
break;
}
default:
break;
}

33
source/libs/parser/src/parTranslater.c
View File

@ -4177,6 +4177,34 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbTbPrefixSuffixOptions(STranslateContext* pCxt, int32_t tbPrefix, int32_t tbSuffix) {
if (tbPrefix < TSDB_MIN_HASH_PREFIX || tbPrefix > TSDB_MAX_HASH_PREFIX) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix: %d valid range: [%d, %d]", tbPrefix,
TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
}
if (tbSuffix < TSDB_MIN_HASH_SUFFIX || tbSuffix > TSDB_MAX_HASH_SUFFIX) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_suffix: %d valid range: [%d, %d]", tbSuffix,
TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
}
if ((tbPrefix * tbSuffix) < 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix & table_suffix: mixed usage not allowed");
}
if ((tbPrefix + tbSuffix) >= (TSDB_TABLE_NAME_LEN - 1)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option table_prefix & table_suffix: exceed max table name length");
}
return TSDB_CODE_SUCCESS;
}
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions) {
int32_t daysPerFile = pOptions->daysPerFile;
int64_t daysToKeep0 = pOptions->keep[0];
@ -4284,10 +4312,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX);
code = checkDbTbPrefixSuffixOptions(pCxt, pOptions->tablePrefix, pOptions->tableSuffix);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkOptionsDependency(pCxt, pDbName, pOptions);

3516
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

1
tests/parallel_test/cases.task
View File

@ -746,6 +746,7 @@
,,y,script,./test.sh -f tsim/db/show_create_table.sim
,,y,script,./test.sh -f tsim/db/tables.sim
,,y,script,./test.sh -f tsim/db/taosdlog.sim
,,y,script,./test.sh -f tsim/db/table_prefix_suffix.sim
,,y,script,./test.sh -f tsim/dnode/balance_replica1.sim
,,y,script,./test.sh -f tsim/dnode/balance_replica3.sim
,,y,script,./test.sh -f tsim/dnode/balance1.sim

182
tests/script/tsim/db/table_prefix_suffix.sim Normal file
View File

@ -0,0 +1,182 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/exec.sh -n dnode1 -s start
sql connect
sql drop database if exists db1;
sql create database db1 vgroups 5 TABLE_PREFIX 1 TABLE_SUFFIX 2;
sql use db1;
sql create table atb1aa (ts timestamp, f1 int);
sql create table btb1bb (ts timestamp, f1 int);
sql create table ctb1cc (ts timestamp, f1 int);
sql create table dtb1dd (ts timestamp, f1 int);
sql create table atb2aa (ts timestamp, f1 int);
sql create table btb2bb (ts timestamp, f1 int);
sql create table ctb2cc (ts timestamp, f1 int);
sql create table dtb2dd (ts timestamp, f1 int);
sql create table etb2ee (ts timestamp, f1 int);
sql show create database db1;
sql select count(*) a from information_schema.ins_tables where db_name='db1' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db1;
sql drop database if exists db2;
sql create database db2 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX -2;
sql use db2;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table taaa22 (ts timestamp, f1 int);
sql create table tbbb22 (ts timestamp, f1 int);
sql create table tccc22 (ts timestamp, f1 int);
sql create table tddd22 (ts timestamp, f1 int);
sql create table teee22 (ts timestamp, f1 int);
sql show create database db2;
sql select count(*) a from information_schema.ins_tables where db_name='db2' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db2;
sql drop database if exists db3;
sql create database db3 vgroups 5 TABLE_PREFIX -1;
sql use db3;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table zaaa22 (ts timestamp, f1 int);
sql create table zbbb22 (ts timestamp, f1 int);
sql create table zccc22 (ts timestamp, f1 int);
sql create table zddd22 (ts timestamp, f1 int);
sql create table zeee22 (ts timestamp, f1 int);
sql show create database db3;
sql select count(*) a from information_schema.ins_tables where db_name='db3' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db3;
sql drop database if exists db4;
sql create database db4 vgroups 5 TABLE_SUFFIX -2;
sql use db4;
sql create table taaa11 (ts timestamp, f1 int);
sql create table tbbb11 (ts timestamp, f1 int);
sql create table tccc11 (ts timestamp, f1 int);
sql create table tddd11 (ts timestamp, f1 int);
sql create table zaaa22 (ts timestamp, f1 int);
sql create table zbbb22 (ts timestamp, f1 int);
sql create table zccc22 (ts timestamp, f1 int);
sql create table zddd22 (ts timestamp, f1 int);
sql create table zeee22 (ts timestamp, f1 int);
sql show create database db4;
sql select count(*) a from information_schema.ins_tables where db_name='db4' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db4;
sql drop database if exists db5;
sql create database db5 vgroups 5 TABLE_PREFIX 1;
sql use db5;
sql create table taaa11 (ts timestamp, f1 int);
sql create table baaa11 (ts timestamp, f1 int);
sql create table caaa11 (ts timestamp, f1 int);
sql create table daaa11 (ts timestamp, f1 int);
sql create table faaa11 (ts timestamp, f1 int);
sql create table gbbb11 (ts timestamp, f1 int);
sql create table hbbb11 (ts timestamp, f1 int);
sql create table ibbb11 (ts timestamp, f1 int);
sql create table jbbb11 (ts timestamp, f1 int);
sql show create database db5;
sql select count(*) a from information_schema.ins_tables where db_name='db5' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db5;
sql drop database if exists db6;
sql create database db6 vgroups 5 TABLE_SUFFIX 2;
sql use db6;
sql create table taaa11 (ts timestamp, f1 int);
sql create table taaa12 (ts timestamp, f1 int);
sql create table taaa13 (ts timestamp, f1 int);
sql create table taaa14 (ts timestamp, f1 int);
sql create table tbbb23 (ts timestamp, f1 int);
sql create table tbbb24 (ts timestamp, f1 int);
sql create table tbbb31 (ts timestamp, f1 int);
sql create table tbbb32 (ts timestamp, f1 int);
sql create table tbbb33 (ts timestamp, f1 int);
sql show create database db6;
sql select count(*) a from information_schema.ins_tables where db_name='db6' group by vgroup_id having(count(*) > 0) order by a;
if $rows != 2 then
return -1
endi
if $data00 != 4 then
return -1
endi
if $data10 != 5 then
return -1
endi
sql drop database if exists db6;
sql drop database if exists db7;
sql create database db7 vgroups 5 TABLE_PREFIX -100 TABLE_SUFFIX -92;
sql use db7;
sql create table taaa11 (ts timestamp, f1 int);
sql create table taaa12 (ts timestamp, f1 int);
sql create table taaa13 (ts timestamp, f1 int);
sql create table tbbb21 (ts timestamp, f1 int);
sql create table tbbb22 (ts timestamp, f1 int);
sql create table tbbb23 (ts timestamp, f1 int);
sql create table tbbb24 (ts timestamp, f1 int);
sql create table tccc31 (ts timestamp, f1 int);
sql create table tccc32 (ts timestamp, f1 int);
sql create table tccc33 (ts timestamp, f1 int);
sql create table tddd24 (ts timestamp, f1 int);
sql create table tddd31 (ts timestamp, f1 int);
sql create table tddd32 (ts timestamp, f1 int);
sql create table tddd33 (ts timestamp, f1 int);
sql show create database db7;
sql select count(*) a from information_schema.ins_tables where db_name='db7' group by vgroup_id having(count(*) > 0) order by a;
sql drop database if exists db7;
sql_error create database db8 vgroups 5 TABLE_PREFIX -1 TABLE_SUFFIX 2;
sql_error create database db8 vgroups 5 TABLE_PREFIX 191 TABLE_SUFFIX 192;
sql_error create database db8 vgroups 5 TABLE_PREFIX -192 TABLE_SUFFIX -191;
sql_error create database db8 vgroups 5 TABLE_PREFIX 100 TABLE_SUFFIX 92;
system sh/exec.sh -n dnode1 -s stop -x SIGINT