diff --git a/include/common/tmsg.h b/include/common/tmsg.h index d3db2f318c..dfce7e54d0 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1201,6 +1201,7 @@ typedef struct { int16_t sstTrigger; int16_t hashPrefix; int16_t hashSuffix; + int32_t tsdbPageSize; } SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index f77320c5ac..496806d877 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -102,9 +102,10 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, - {.name = "sst_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "stt_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, }; static const SSysDbTableSchema userFuncSchema[] = { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 7d8461ca60..cbd6bc3ed7 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3782,6 +3782,7 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI16(&encoder, pReq->sstTrigger) < 0) return -1; if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1; if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1; + if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; tEndEncode(&encoder); @@ -3857,6 +3858,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI16(&decoder, &pReq->sstTrigger) < 0) return -1; if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1; if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 981576efec..e3ad1b9389 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -173,6 +173,7 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->hashMethod = pCreate->hashMethod; pCfg->hashPrefix = pCreate->hashPrefix; pCfg->hashSuffix = pCreate->hashSuffix; + pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->standby = pCfg->standby; pCfg->syncCfg.myIndex = pCreate->selfIndex; @@ -222,9 +223,11 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d", - createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize, - createReq.sstTrigger); + dInfo( + "vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d " + "tsdbPageSize:%d", + createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize, + createReq.sstTrigger, createReq.tsdbPageSize); dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod, createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix); vmGenerateVnodeCfg(&createReq, &vnodeCfg); diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9632be1b24..cd08ee4ad5 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -308,6 +308,7 @@ typedef struct { int16_t hashPrefix; int16_t hashSuffix; int16_t sstTrigger; + int32_t tsdbPageSize; int32_t numOfRetensions; SArray* pRetensions; int32_t walRetentionPeriod; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 9e58b74017..d42ba1f7ff 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -31,7 +31,7 @@ #include "systable.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 58 +#define DB_RESERVE_SIZE 54 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); @@ -128,6 +128,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT16(pRaw, dataPos, pDb->cfg.sstTrigger, _OVER) SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashPrefix, _OVER) SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -214,10 +215,20 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.sstTrigger, _OVER) SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashPrefix, _OVER) SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); + if (pDb->cfg.tsdbPageSize <= TSDB_MIN_TSDB_PAGESIZE) { + mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize, + TSDB_DEFAULT_TSDB_PAGESIZE); + } + + if (pDb->cfg.sstTrigger <= TSDB_MIN_STT_TRIGGER) { + mInfo("db:%s, sstTrigger set from %d to default %d", pDb->name, pDb->cfg.sstTrigger, TSDB_DEFAULT_SST_TRIGGER); + } + terrno = 0; _OVER: @@ -262,6 +273,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.cacheLast = pNew->cfg.cacheLast; pOld->cfg.replications = pNew->cfg.replications; pOld->cfg.sstTrigger = pNew->cfg.sstTrigger; + pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; taosWUnLockLatch(&pOld->lock); return 0; } @@ -341,6 +353,7 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; + if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; terrno = 0; return terrno; @@ -377,6 +390,7 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; + if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; } static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { @@ -496,6 +510,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .sstTrigger = pCreate->sstTrigger, .hashPrefix = pCreate->hashPrefix, .hashSuffix = pCreate->hashSuffix, + .tsdbPageSize = pCreate->tsdbPageSize, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; @@ -1726,6 +1741,9 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false); } taosMemoryFree(buf); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index de29dea511..a95bdef323 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -237,6 +237,7 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.sstTrigger = pDb->cfg.sstTrigger; createReq.hashPrefix = pDb->cfg.hashPrefix; createReq.hashSuffix = pDb->cfg.hashSuffix; + createReq.tsdbPageSize = pDb->cfg.tsdbPageSize; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 9ed72eeecd..5adb2eb359 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -115,6 +115,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; @@ -215,7 +216,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code); - if (code < 0) return -1; + if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER; tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code); @@ -223,9 +224,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code); - if (code < 0) return -1; + if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code); - if (code < 0) return -1; + if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); if (code < 0) return -1; @@ -255,6 +256,7 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { } tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code); + if (code < 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024; return 0; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index bb2151ed9d..2cc62d3d6a 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -185,10 +185,10 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // ref page one more time so the page will not be release tdbRefPage(pPage); tdbDebug("pcache/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); - /* + // Set page as dirty pPage->isDirty = 1; - + /* // Add page to dirty list(TODO: NOT use O(n^2) algorithm) for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage); ppPage = &((*ppPage)->pDirtyNext)) { @@ -260,6 +260,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { pPage->isDirty = 0; + // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } @@ -345,15 +346,19 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { } */ // 3, release the dirty pages - for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) { - pPager->pDirty = pPage->pDirtyNext; - pPage->pDirtyNext = NULL; + SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); + SRBTreeNode *pNode = NULL; + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; pPage->isDirty = 0; + // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } + tRBTreeCreate(&pPager->rbt, pageCmpFn); + // 4, remove the journal file tdbOsClose(pPager->jfd); tdbOsRemove(pPager->jFileName); diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 068d212ac4..f620a4b18a 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -81,6 +81,7 @@ class TDTestCase: 'binary':self.binary_val, 'nchar':self.nchar_val } + def insert_base_data(self,col_type,tbname,rows,base_data): for i in range(rows): if col_type.lower() == 'tinyint': @@ -290,6 +291,9 @@ class TDTestCase: self.delete_data_ntb() self.delete_data_ctb() self.delete_data_stb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_ntb() def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index deff4b42a1..417adc39ac 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -154,10 +154,12 @@ class TDTestCase: up_bool = random.randint(0,100)%2 up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - binary_length = random.randint(0,self.str_length) - nchar_length = random.randint(0,self.str_length) - up_binary = tdCom.getLongName(binary_length) - up_nchar = tdCom.getLongName(nchar_length) + binary_length = [] + for i in range(self.str_length+1): + binary_length.append(i) + nchar_length = [] + for i in range(self.str_length+1): + nchar_length.append(i) for col_name,col_type in column_dict.items(): if tb_type == 'ntb': tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') @@ -188,9 +190,13 @@ class TDTestCase: elif col_type.lower() == 'double': self.update_and_check_data(tbname,col_name,col_type,up_double,dbname) elif 'binary' in col_type.lower(): - self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) + for i in binary_length: + up_binary = tdCom.getLongName(i) + self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) elif 'nchar' in col_type.lower(): - self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) + for i in nchar_length: + up_nchar = tdCom.getLongName(i) + self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) elif col_type.lower() == 'timestamp': self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname) tdSql.execute(f'insert into {tbname} values({self.ts},null)')