xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #16758 from taosdata/fix/stt_trigger_config 

refact: stt trigger config
This commit is contained in:
Hongze Cheng 2022-09-09 14:55:32 +08:00 committed by GitHub
parent 25e463d7c4 a0ba99e8a8
commit aafcd68895
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 36 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/util/tdef.h
View File

@ -359,8 +359,8 @@ typedef enum ELogicConditionType {
#define TSDB_DB_SCHEMALESS_ON 1
#define TSDB_DB_SCHEMALESS_OFF 0
#define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF
#define TSDB_MIN_SST_TRIGGER 1
#define TSDB_MAX_SST_TRIGGER 128
#define TSDB_MIN_STT_TRIGGER 1
#define TSDB_MAX_STT_TRIGGER 16
#define TSDB_DEFAULT_SST_TRIGGER 8
#define TSDB_MIN_HASH_PREFIX 0
#define TSDB_MAX_HASH_PREFIX 128

8
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -167,7 +167,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) {
pCfg->walCfg.segSize = pCreate->walSegmentSize;
pCfg->walCfg.level = pCreate->walLevel;
pCfg->sstTrigger = pCreate->sstTrigger;
pCfg->sttTrigger = pCreate->sstTrigger;
pCfg->hashBegin = pCreate->hashBegin;
pCfg->hashEnd = pCreate->hashEnd;
pCfg->hashMethod = pCreate->hashMethod;
@ -223,10 +223,10 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) {
}
dInfo("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d",
createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize,
createReq.sstTrigger);
createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize,
createReq.sstTrigger);
dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod,
createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix);
createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix);
vmGenerateVnodeCfg(&createReq, &vnodeCfg);
if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) {

2
source/dnode/mnode/impl/src/mndDb.c
View File

@ -338,7 +338,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) {
if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1;
if (pCfg->walRollPeriod < TSDB_DB_MIN_WAL_ROLL_PERIOD) return -1;
if (pCfg->walSegmentSize < TSDB_DB_MIN_WAL_SEGMENT_SIZE) return -1;
if (pCfg->sstTrigger < TSDB_MIN_SST_TRIGGER || pCfg->sstTrigger > TSDB_MAX_SST_TRIGGER) return -1;
if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1;
if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1;
if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1;

4
source/dnode/vnode/inc/vnode.h
View File

@ -125,6 +125,8 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur);
// typedef struct STsdb STsdb;
typedef struct STsdbReader STsdbReader;
#define TSDB_DEFAULT_STT_FILE 8
#define TIMEWINDOW_RANGE_CONTAINED 1
#define TIMEWINDOW_RANGE_EXTERNAL 2
@ -288,7 +290,7 @@ struct SVnodeCfg {
SVnodeStats vndStats;
uint32_t hashBegin;
uint32_t hashEnd;
int16_t sstTrigger;
int16_t sttTrigger;
int16_t hashPrefix;
int16_t hashSuffix;
};

8
source/dnode/vnode/src/inc/tsdb.h
View File

@ -69,8 +69,6 @@ typedef struct SLDataIter SLDataIter;
#define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F)
#define TSDB_MAX_SUBBLOCKS 8
#define TSDB_MAX_STT_FILE 16
#define TSDB_DEFAULT_STT_FILE 8
#define TSDB_FHDR_SIZE 512
#define TSDB_DEFAULT_PAGE_SIZE 4096
@ -574,7 +572,7 @@ struct SDFileSet {
SDataFile *pDataF;
SSmaFile *pSmaF;
uint8_t nSttF;
SSttFile *aSttF[TSDB_MAX_STT_FILE];
SSttFile *aSttF[TSDB_MAX_STT_TRIGGER];
};
struct SRowIter {
@ -624,7 +622,7 @@ struct SDataFWriter {
SHeadFile fHead;
SDataFile fData;
SSmaFile fSma;
SSttFile fStt[TSDB_MAX_STT_FILE];
SSttFile fStt[TSDB_MAX_STT_TRIGGER];
uint8_t *aBuf[4];
};
@ -635,7 +633,7 @@ struct SDataFReader {
STsdbFD *pHeadFD;
STsdbFD *pDataFD;
STsdbFD *pSmaFD;
STsdbFD *aSttFD[TSDB_MAX_STT_FILE];
STsdbFD *aSttFD[TSDB_MAX_STT_TRIGGER];
uint8_t *aBuf[3];
};

16
source/dnode/vnode/src/tsdb/tsdbCommit.c
View File

@ -45,7 +45,7 @@ typedef struct {
int32_t minRow;
int32_t maxRow;
int8_t cmprAlg;
int8_t maxLast;
int8_t sttTrigger;
SArray *aTbDataP; // memory
STsdbFS fs; // disk
// --------------
@ -66,7 +66,7 @@ typedef struct {
SDataIter *pIter;
SRBTree rbt;
SDataIter dataIter;
SDataIter aDataIter[TSDB_MAX_STT_FILE];
SDataIter aDataIter[TSDB_MAX_STT_TRIGGER];
int8_t toLastOnly;
};
struct {
@ -423,7 +423,7 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
pCommitter->toLastOnly = 0;
SDataFReader *pReader = pCommitter->dReader.pReader;
if (pReader) {
if (pReader->pSet->nSttF >= pCommitter->maxLast) {
if (pReader->pSet->nSttF >= pCommitter->sttTrigger) {
int8_t iIter = 0;
for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) {
pIter = &pCommitter->aDataIter[iIter];
@ -510,11 +510,11 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
SSttFile fStt = {.commitID = pCommitter->commitID};
SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
if (pRSet) {
ASSERT(pRSet->nSttF <= pCommitter->maxLast);
ASSERT(pRSet->nSttF <= pCommitter->sttTrigger);
fData = *pRSet->pDataF;
fSma = *pRSet->pSmaF;
wSet.diskId = pRSet->diskId;
if (pRSet->nSttF < pCommitter->maxLast) {
if (pRSet->nSttF < pCommitter->sttTrigger) {
for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) {
wSet.aSttF[iStt] = pRSet->aSttF[iStt];
}
@ -750,7 +750,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
pCommitter->maxLast = TSDB_DEFAULT_STT_FILE; // TODO: make it as a config
pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger;
pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
if (pCommitter->aTbDataP == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@ -780,7 +780,7 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
if (code) goto _exit;
// merger
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
SDataIter *pIter = &pCommitter->aDataIter[iStt];
pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
if (pIter->aSttBlk == NULL) {
@ -822,7 +822,7 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
tBlockDataDestroy(&pCommitter->dReader.bData, 1);
// merger
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
SDataIter *pIter = &pCommitter->aDataIter[iStt];
taosArrayDestroy(pIter->aSttBlk);
tBlockDataDestroy(&pIter->bData, 1);

2
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -780,7 +780,7 @@ int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
tsdbCloseFile(&(*ppReader)->pSmaFD);
// stt
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) {
if ((*ppReader)->aSttFD[iStt]) {
tsdbCloseFile(&(*ppReader)->aSttFD[iStt]);
}

2
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -51,7 +51,7 @@ struct STsdbSnapReader {
SDataFReader* pDataFReader;
SFDataIter* pIter;
SRBTree rbt;
SFDataIter aFDataIter[TSDB_MAX_STT_FILE + 1];
SFDataIter aFDataIter[TSDB_MAX_STT_TRIGGER + 1];
SBlockData bData;
SSkmInfo skmTable;
// for del file

9
source/dnode/vnode/src/vnd/vnodeCfg.c
View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnd.h"
#include "tutil.h"
#include "vnd.h"
const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
.dbname = "",
@ -48,7 +48,8 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1,
},
.hashBegin = 0,
.hashEnd = 0,
.hashMethod = 0};
.hashMethod = 0,
.sttTrigger = TSDB_DEFAULT_STT_FILE};
int vnodeCheckCfg(const SVnodeCfg *pCfg) {
// TODO
@ -107,7 +108,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) {
if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sstTrigger) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1;
if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1;
@ -209,7 +210,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) {
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sstTrigger, code);
tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code);
if (code < 0) return -1;
tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code);
if (code < 0) return -1;

2
source/dnode/vnode/src/vnd/vnodeOpen.c
View File

@ -60,6 +60,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) {
snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path);
info.config = vnodeCfgDefault;
// load vnode info
ret = vnodeLoadInfo(dir, &info);
if (ret < 0) {

4
source/libs/parser/src/parTranslater.c
View File

@ -3770,7 +3770,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_SST_TRIGGER, TSDB_MAX_SST_TRIGGER);
code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX);
@ -6606,7 +6606,7 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS
SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema);
if (QUERY_NODE_VALUE != pStmt->pVal->node.type) {
SValueNode *pVal = NULL;
SValueNode* pVal = NULL;
pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal);
if (pCxt->errCode) {
return pCxt->errCode;

8
source/libs/parser/test/parAlterToBalanceTest.cpp
View File

@ -88,7 +88,7 @@ TEST_F(ParserInitialATest, alterDnode) {
* | REPLICA int_value -- todo: enum 1, 3, default 1, unit replica
* | STRICT {'off' | 'on'} -- todo: default 'off'
* | WAL_LEVEL int_value -- enum 1, 2, default 1
* | SST_TRIGGER int_value -- rang [1, 128], default 8
* | SST_TRIGGER int_value -- rang [1, 16], default 8
* }
*/
TEST_F(ParserInitialATest, alterDatabase) {
@ -161,8 +161,8 @@ TEST_F(ParserInitialATest, alterDatabase) {
setAlterDbFsync(200);
setAlterDbWal(1);
setAlterDbCacheModel(TSDB_CACHE_MODEL_LAST_ROW);
setAlterDbSstTrigger(20);
run("ALTER DATABASE test CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 WAL_LEVEL 1 SST_TRIGGER 20");
setAlterDbSstTrigger(16);
run("ALTER DATABASE test CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 WAL_LEVEL 1 SST_TRIGGER 16");
clearAlterDbReq();
initAlterDb("test");
@ -237,7 +237,7 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) {
run("ALTER DATABASE test WAL_LEVEL 0", TSDB_CODE_PAR_INVALID_DB_OPTION);
run("ALTER DATABASE test WAL_LEVEL 3", TSDB_CODE_PAR_INVALID_DB_OPTION);
run("ALTER DATABASE test SST_TRIGGER 0", TSDB_CODE_PAR_INVALID_DB_OPTION);
run("ALTER DATABASE test SST_TRIGGER 129", TSDB_CODE_PAR_INVALID_DB_OPTION);
run("ALTER DATABASE test SST_TRIGGER 17", TSDB_CODE_PAR_INVALID_DB_OPTION);
// Regardless of the specific sentence
run("ALTER DATABASE db WAL_LEVEL 0 # td-14436", TSDB_CODE_PAR_SYNTAX_ERROR, PARSER_STAGE_PARSE);
}