From 8c6f452f09d3bc9fe66915d7415bfac553628c52 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Fri, 27 Oct 2023 15:51:52 +0800 Subject: [PATCH 01/13] tsdb/write: make last tier writable to mem --- source/dnode/vnode/src/tsdb/tsdbWrite.c | 6 +++--- source/dnode/vnode/src/vnd/vnodeSvr.c | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 5949b103d5..1e6526da48 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -80,8 +80,8 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { TSKEY minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep2; TSKEY maxKey = tsMaxKeyByPrecision[pCfg->precision]; int32_t size = taosArrayGetSize(pMsg->aSubmitTbData); - int32_t nlevel = tfsGetLevel(pTsdb->pVnode->pTfs); - + /* + int32_t nlevel = tfsGetLevel(pTsdb->pVnode->pTfs); if (nlevel > 1 && tsS3Enabled) { if (nlevel == 3) { minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep1; @@ -89,7 +89,7 @@ int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq2 *pMsg) { minKey = now - tsTickPerMin[pCfg->precision] * pCfg->keep0; } } - + */ for (int32_t i = 0; i < size; ++i) { SSubmitTbData *pData = TARRAY_GET_ELEM(pMsg->aSubmitTbData, i); if (pData->flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index c46ea15111..ed3b9460a6 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ +#include "audit.h" #include "tencode.h" #include "tmsg.h" #include "vnd.h" #include "vndCos.h" #include "vnode.h" #include "vnodeInt.h" -#include "audit.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -177,7 +177,7 @@ static int32_t vnodePreProcessDropTtlMsg(SVnode *pVnode, SRpcMsg *pMsg) { ttlReq.pTbUids = tbUids; } - { // prepare new content + { // prepare new content int32_t reqLenNew = tSerializeSVDropTtlTableReq(NULL, 0, &ttlReq); int32_t contLenNew = reqLenNew + sizeof(SMsgHead); @@ -262,8 +262,9 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int now *= 1000000; } - int32_t nlevel = tfsGetLevel(pVnode->pTfs); int32_t keep = pVnode->config.tsdbCfg.keep2; + /* + int32_t nlevel = tfsGetLevel(pVnode->pTfs); if (nlevel > 1 && tsS3Enabled) { if (nlevel == 3) { keep = pVnode->config.tsdbCfg.keep1; @@ -271,7 +272,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int keep = pVnode->config.tsdbCfg.keep0; } } - + */ TSKEY minKey = now - tsTickPerMin[pVnode->config.tsdbCfg.precision] * keep; TSKEY maxKey = tsMaxKeyByPrecision[pVnode->config.tsdbCfg.precision]; if (submitTbData.flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { @@ -584,7 +585,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg } } break; case TDMT_VND_STREAM_TASK_RESET: { - if (pVnode->restored/* && vnodeIsLeader(pVnode)*/) { + if (pVnode->restored /* && vnodeIsLeader(pVnode)*/) { tqProcessTaskResetReq(pVnode->pTq, pMsg); } } break; @@ -947,7 +948,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); - if(tsEnableAuditCreateTable){ + if (tsEnableAuditCreateTable) { int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; @@ -1460,7 +1461,6 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in int32_t nRow = TARRAY_SIZE(pSubmitTbData->aRowP); SRow **aRow = (SRow **)TARRAY_DATA(pSubmitTbData->aRowP); for (int32_t iRow = 0; iRow < nRow; ++iRow) { - if (aRow[iRow]->ts < minKey || aRow[iRow]->ts > maxKey || (iRow > 0 && aRow[iRow]->ts <= aRow[iRow - 1]->ts)) { code = TSDB_CODE_INVALID_MSG; vError("vgId:%d %s failed since %s, version:%" PRId64, TD_VID(pVnode), __func__, tstrerror(code), ver); @@ -1569,7 +1569,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { code = terrno; vError("vgId:%d failed to create table:%s, code:%s", TD_VID(pVnode), pSubmitTbData->pCreateTbReq->name, - tstrerror(terrno)); + tstrerror(terrno)); goto _exit; } terrno = 0; From 4fede02da4b285940ddd8578f1ed485edf632458 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 30 Oct 2023 10:06:42 +0800 Subject: [PATCH 02/13] tsdb/retention: remove s3 evict cache hook --- source/dnode/vnode/src/tsdb/tsdbRetention.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index f2665dcf26..8fb7648c99 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -342,6 +342,7 @@ static int32_t tsdbDoRetention2(void *arg) { code = tsdbMigrateDataFileS3(rtner, fobj, &did); TSDB_CHECK_CODE(code, lino, _exit); } else { + /* if (tsS3Enabled) { int64_t fsize = 0; if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { @@ -352,7 +353,7 @@ static int32_t tsdbDoRetention2(void *arg) { } s3EvictCache(fobj->fname, fsize * 2); } - + */ code = tsdbDoMigrateFileObj(rtner, fobj, &did); TSDB_CHECK_CODE(code, lino, _exit); } From 84d2f9c379b5299b874ec969759619bda512f273 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 30 Oct 2023 14:58:22 +0800 Subject: [PATCH 03/13] tsdb/upload: delay data file uploading --- source/common/src/tglobal.c | 25 ++++++++-- source/dnode/vnode/src/inc/vndCos.h | 1 + source/dnode/vnode/src/tsdb/tsdbCommit2.c | 38 ++++++++++++-- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 + source/dnode/vnode/src/tsdb/tsdbRetention.c | 49 ++++++++++++------- 5 files changed, 87 insertions(+), 28 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0a155f4ea1..a170e5436e 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -120,7 +120,7 @@ char tsSmlTsDefaultName[TSDB_COL_NAME_LEN] = "_ts"; char tsSmlTagName[TSDB_COL_NAME_LEN] = "_tag_null"; char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table name can be specified in tag value. char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; - // If set to empty system will generate table name using MD5 hash. +// If set to empty system will generate table name using MD5 hash. // true means that the name and order of cols in each line are the same(only for influx protocol) // bool tsSmlDataFormat = false; // int32_t tsSmlBatchSize = 10000; @@ -273,6 +273,7 @@ char tsS3Hostname[TSDB_FQDN_LEN] = ""; int32_t tsS3BlockSize = 4096; // number of tsdb pages int32_t tsS3BlockCacheSize = 16; // number of blocks +int32_t tsS3UploadDelaySec = 60 * 60; int32_t tsCheckpointInterval = 300; @@ -453,7 +454,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) + return -1; if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1; if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1; @@ -665,7 +667,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointTickInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointTickInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) + return -1; if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1; @@ -688,6 +691,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, 2048, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER) != 0) + return -1; // min free disk space used to check if the disk is full [50MB, 1GB] if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, @@ -946,7 +951,8 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { return -1; } - tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN); + tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, + TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN); @@ -1116,6 +1122,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32; tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32; + tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32; GRANT_CFG_GET; return 0; @@ -1413,7 +1420,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("smlChildTableName", name) == 0) { tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); } else if (strcasecmp("smlAutoChildTableNameDelimiter", name) == 0) { - tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, TSDB_TABLE_NAME_LEN); + tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, + TSDB_TABLE_NAME_LEN); } else if (strcasecmp("smlTagName", name) == 0) { tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); // } else if (strcasecmp("smlDataFormat", name) == 0) { @@ -1708,6 +1716,13 @@ void taosCfgDynamicOptions(const char *option, const char *value) { return; } + if (strcasecmp(option, "s3UploadDelaySec") == 0) { + int32_t newS3UploadDelaysec = atoi(value); + uInfo("s3UploadDelaySec set from %d to %d", tsS3UploadDelaySec, newS3UploadDelaysec); + tsS3UploadDelaySec = newS3UploadDelaysec; + return; + } + if (strcasecmp(option, "ttlPushInterval") == 0) { int32_t newTtlPushInterval = atoi(value); uInfo("ttlPushInterval set from %d to %d", tsTtlPushIntervalSec, newTtlPushInterval); diff --git a/source/dnode/vnode/src/inc/vndCos.h b/source/dnode/vnode/src/inc/vndCos.h index bb4d284f0e..6612b5c653 100644 --- a/source/dnode/vnode/src/inc/vndCos.h +++ b/source/dnode/vnode/src/inc/vndCos.h @@ -27,6 +27,7 @@ extern "C" { extern int8_t tsS3Enabled; extern int32_t tsS3BlockSize; extern int32_t tsS3BlockCacheSize; +extern int32_t tsS3UploadDelaySec; int32_t s3Init(); void s3CleanUp(); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 6dc492f420..c77c444125 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -46,6 +46,7 @@ typedef struct { STFileSet *fset; TABLEID tbid[1]; bool hasTSData; + bool skipTsRow; } ctx[1]; // reader @@ -127,18 +128,18 @@ static int32_t tsdbCommitTSData(SCommitter2 *committer) { continue; } } - + /* extern int8_t tsS3Enabled; int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); - bool skipRow = false; + committer->ctx->skipTsRow = false; if (tsS3Enabled && nlevel > 1 && committer->ctx->did.level == nlevel - 1) { - skipRow = true; + committer->ctx->skipTsRow = true; } - + */ int64_t ts = TSDBROW_TS(&row->row); - if (skipRow && ts <= committer->ctx->maxKey) { + if (committer->ctx->skipTsRow && ts <= committer->ctx->maxKey) { ts = committer->ctx->maxKey + 1; } @@ -397,6 +398,33 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { // reset nextKey committer->ctx->nextKey = TSKEY_MAX; + committer->ctx->skipTsRow = false; + + extern int8_t tsS3Enabled; + extern int32_t tsS3UploadDelaySec; + long s3Size(const char *object_name); + int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); + committer->ctx->skipTsRow = false; + if (tsS3Enabled && nlevel > 1 /* && committer->ctx->did.level == nlevel - 1*/) { + STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA]; + if (committer->ctx->fset && fobj) { + // if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay + const char *object_name = taosDirEntryBaseName((char *)fobj->fname); + + if (taosCheckExistFile(fobj->fname)) { + int32_t mtime = 0; + taosStatFile(fobj->fname, NULL, &mtime, NULL); + if (mtime < committer->ctx->now - tsS3UploadDelaySec) { + committer->ctx->skipTsRow = true; + } + } else if (s3Size(object_name) > 0) { + committer->ctx->skipTsRow = true; + } + } + + // new fset can be written with ts data + } + _exit: if (code) { TSDB_ERROR_LOG(TD_VID(tsdb->pVnode), lino, code); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index f3bcfef703..6904249adb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -131,6 +131,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) { } if (pFD->s3File) { + tsdbWarn("%s file:%d", __func__, pFD->path); return code; } if (pFD->pgno > 0) { @@ -286,6 +287,7 @@ int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; if (pFD->s3File) { + tsdbWarn("%s file:%d", __func__, pFD->path); return code; } code = tsdbWriteFilePage(pFD); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 8fb7648c99..4a5c8300cd 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -304,7 +304,8 @@ static int32_t tsdbDoRetention2(void *arg) { if (fobj == NULL) continue; int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) { + if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1 && + !taosCheckExistFile(fobj->fname)) { code = tsdbRemoveFileObjectS3(rtner, fobj); TSDB_CHECK_CODE(code, lino, _exit); } else { @@ -335,28 +336,40 @@ static int32_t tsdbDoRetention2(void *arg) { for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = rtner->ctx->fset->farr[ftype], 1); ++ftype) { if (fobj == NULL) continue; - if (fobj->f->did.level == did.level) continue; - int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) { - code = tsdbMigrateDataFileS3(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - /* - if (tsS3Enabled) { - int64_t fsize = 0; - if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(terrno); - tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode), __func__, - fobj->fname, tstrerror(code)); + + if (fobj->f->did.level == did.level) { + if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1 && + taosCheckExistFile(fobj->fname)) { + int32_t mtime = 0; + taosStatFile(fobj->fname, NULL, &mtime, NULL); + if (mtime < rtner->now - tsS3UploadDelaySec) { + code = tsdbMigrateDataFileS3(rtner, fobj, &did); TSDB_CHECK_CODE(code, lino, _exit); } - s3EvictCache(fobj->fname, fsize * 2); } - */ - code = tsdbDoMigrateFileObj(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); + + continue; } + /* + if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) { + code = tsdbMigrateDataFileS3(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + + if (tsS3Enabled) { + int64_t fsize = 0; + if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(terrno); + tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode), + __func__, fobj->fname, tstrerror(code)); TSDB_CHECK_CODE(code, lino, _exit); + } + s3EvictCache(fobj->fname, fsize * 2); + } + */ + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + //} } // stt From 800cbfbb056a763ed6ffac35d0a1d98cab6bbf32 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 30 Oct 2023 17:36:49 +0800 Subject: [PATCH 04/13] tsdb/file: new s3flag in mem --- source/dnode/mnode/impl/src/mndDnode.c | 34 ++++++++++++++++++++- source/dnode/vnode/src/tsdb/tsdbCommit2.c | 5 ++- source/dnode/vnode/src/tsdb/tsdbFile2.c | 1 + source/dnode/vnode/src/tsdb/tsdbFile2.h | 1 + source/dnode/vnode/src/tsdb/tsdbRetention.c | 2 ++ 5 files changed, 39 insertions(+), 4 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index b53dee7bff..50e5ec46b8 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -720,7 +720,7 @@ static int32_t mndProcessNotifyReq(SRpcMsg *pReq) { mndReleaseVgroup(pMnode, pVgroup); } } - mndUpdClusterInfo(pReq); + mndUpdClusterInfo(pReq); _OVER: tFreeSNotifyReq(¬ifyReq); return code; @@ -1235,6 +1235,38 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "monitor"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strncasecmp(cfgReq.config, "s3blockcachesize", 16) == 0) { + int32_t optLen = strlen("s3blockcachesize"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; + + if (flag < 4 || flag > 1024 * 1024) { + mError("dnode:%d, failed to config s3BlockCacheSize since value:%d. Valid range: [4, 1024 * 1024]", + cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + tFreeSMCfgDnodeReq(&cfgReq); + return -1; + } + + strcpy(dcfgReq.config, "s3blockcachesize"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strncasecmp(cfgReq.config, "s3uploaddelaysec", 16) == 0) { + int32_t optLen = strlen("s3uploaddelaysec"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; + + if (flag < 600 || flag > 60 * 60 * 24 * 30) { + mError("dnode:%d, failed to config s3UploadDelaySec since value:%d. Valid range: [600, 60 * 60 * 24 * 30]", + cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + tFreeSMCfgDnodeReq(&cfgReq); + return -1; + } + + strcpy(dcfgReq.config, "s3uploaddelaysec"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) { int32_t optLen = strlen("ttlpushinterval"); int32_t flag = -1; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index c77c444125..5357c52723 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -405,9 +405,9 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { long s3Size(const char *object_name); int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); committer->ctx->skipTsRow = false; - if (tsS3Enabled && nlevel > 1 /* && committer->ctx->did.level == nlevel - 1*/) { + if (tsS3Enabled && nlevel > 1 && committer->ctx->fset) { STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA]; - if (committer->ctx->fset && fobj) { + if (fobj && fobj->f->did.level == nlevel - 1) { // if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay const char *object_name = taosDirEntryBaseName((char *)fobj->fname); @@ -421,7 +421,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { committer->ctx->skipTsRow = true; } } - // new fset can be written with ts data } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 963c5bad34..9edb03d35b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -303,6 +303,7 @@ bool tsdbIsSameTFile(const STFile *f1, const STFile *f2) { if (f1->did.id != f2->did.id) return false; if (f1->fid != f2->fid) return false; if (f1->cid != f2->cid) return false; + if (f1->s3flag != f2->s3flag) return false; return true; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.h b/source/dnode/vnode/src/tsdb/tsdbFile2.h index 33d8ac5478..9da198c1f0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h @@ -58,6 +58,7 @@ int32_t tsdbTFileObjCmpr(const STFileObj **fobj1, const STFileObj **fobj2); struct STFile { tsdb_ftype_t type; SDiskID did; // disk id + int32_t s3flag; int32_t fid; // file id int64_t cid; // commit id int64_t size; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 4a5c8300cd..cfeca531fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -211,6 +211,8 @@ static int32_t tsdbMigrateDataFileS3(SRTNer *rtner, const STFileObj *fobj, const }, }; + op.nf.s3flag = true; + code = TARRAY2_APPEND(rtner->fopArr, op); TSDB_CHECK_CODE(code, lino, _exit); From 867ce92dcc9f0c8e58591183a0672c442e3ec9ff Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 30 Oct 2023 19:28:18 +0800 Subject: [PATCH 05/13] tsdb/file-rw: fix log msg --- source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 6904249adb..65f0f1b94d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -131,7 +131,7 @@ static int32_t tsdbWriteFilePage(STsdbFD *pFD) { } if (pFD->s3File) { - tsdbWarn("%s file:%d", __func__, pFD->path); + tsdbWarn("%s file: %s", __func__, pFD->path); return code; } if (pFD->pgno > 0) { @@ -287,7 +287,7 @@ int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; if (pFD->s3File) { - tsdbWarn("%s file:%d", __func__, pFD->path); + tsdbWarn("%s file: %s", __func__, pFD->path); return code; } code = tsdbWriteFilePage(pFD); From a61502411df454c474ff2f70aadb4d311fc970bc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 1 Nov 2023 10:49:05 +0800 Subject: [PATCH 06/13] tsdb/pg-cache: new page cache for tsdb s3 read file page --- source/dnode/vnode/src/inc/tsdb.h | 16 ++- source/dnode/vnode/src/tsdb/tsdbCache.c | 79 +++++++++++- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 117 +++++++++++++++++- 3 files changed, 200 insertions(+), 12 deletions(-) diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 79112babc3..857aef78dc 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -382,6 +382,8 @@ struct STsdb { TdThreadMutex biMutex; SLRUCache *bCache; TdThreadMutex bMutex; + SLRUCache *pgCache; + TdThreadMutex pgMutex; struct STFileSystem *pFS; // new SRocksCache rCache; }; @@ -677,9 +679,9 @@ typedef struct STSnapRange STSnapRange; typedef TARRAY2(STSnapRange *) TSnapRangeArray; // disjoint snap ranges // util -int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); -void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); +int32_t tSerializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); +int32_t tDeserializeSnapRangeArray(void *buf, int32_t bufLen, TSnapRangeArray *pSnapR); +void tsdbSnapRangeArrayDestroy(TSnapRangeArray **ppSnap); SHashObj *tsdbGetSnapRangeHash(TSnapRangeArray *pRanges); // snap partition list @@ -873,8 +875,8 @@ int32_t tMergeTreeOpen2(SMergeTree *pMTree, SMergeTreeConf *pConf); void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); bool tMergeTreeNext(SMergeTree *pMTree); -void tMergeTreePinSttBlock(SMergeTree* pMTree); -void tMergeTreeUnpinSttBlock(SMergeTree* pMTree); +void tMergeTreePinSttBlock(SMergeTree *pMTree); +void tMergeTreeUnpinSttBlock(SMergeTree *pMTree); bool tMergeTreeIgnoreEarlierTs(SMergeTree *pMTree); void tMergeTreeClose(SMergeTree *pMTree); @@ -909,7 +911,9 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h); int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle); -int32_t tsdbBCacheRelease(SLRUCache *pCache, LRUHandle *h); +int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHandle **handle); +int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage); +int32_t tsdbCacheRelease(SLRUCache *pCache, LRUHandle *h); int32_t tsdbCacheDeleteLastrow(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ca3fb7027f..7cea469c34 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -87,6 +87,41 @@ static void tsdbCloseBCache(STsdb *pTsdb) { } } +static int32_t tsdbOpenPgCache(STsdb *pTsdb) { + int32_t code = 0; + // SLRUCache *pCache = taosLRUCacheInit(10 * 1024 * 1024, 0, .5); + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; + + SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3BlockCacheSize * szPage, 0, .5); + if (pCache == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + taosLRUCacheSetStrictCapacity(pCache, false); + + taosThreadMutexInit(&pTsdb->pgMutex, NULL); + +_err: + pTsdb->pgCache = pCache; + return code; +} + +static void tsdbClosePgCache(STsdb *pTsdb) { + SLRUCache *pCache = pTsdb->pgCache; + if (pCache) { + int32_t elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); + taosLRUCacheEraseUnrefEntries(pCache); + elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); + + taosLRUCacheCleanup(pCache); + + taosThreadMutexDestroy(&pTsdb->bMutex); + } +} + #define ROCKS_KEY_LEN (sizeof(tb_uid_t) + sizeof(int16_t) + sizeof(int8_t)) typedef struct { @@ -1191,6 +1226,12 @@ int32_t tsdbOpenCache(STsdb *pTsdb) { goto _err; } + code = tsdbOpenPgCache(pTsdb); + if (code != TSDB_CODE_SUCCESS) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tsdbOpenRocksCache(pTsdb); if (code != TSDB_CODE_SUCCESS) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1221,6 +1262,7 @@ void tsdbCloseCache(STsdb *pTsdb) { tsdbCloseBICache(pTsdb); tsdbCloseBCache(pTsdb); + tsdbClosePgCache(pTsdb); tsdbCloseRocksCache(pTsdb); } @@ -3057,7 +3099,6 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) { } */ int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage; - // int64_t size = 4096; code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, ppBlock); if (code != TSDB_CODE_SUCCESS) { // taosMemoryFree(pBlock); @@ -3123,10 +3164,42 @@ int32_t tsdbCacheGetBlockS3(SLRUCache *pCache, STsdbFD *pFD, LRUHandle **handle) return code; } -int32_t tsdbBCacheRelease(SLRUCache *pCache, LRUHandle *h) { +int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHandle **handle) { int32_t code = 0; + char key[128] = {0}; + int keyLen = 0; - taosLRUCacheRelease(pCache, h, false); + getBCacheKey(pFD->fid, pFD->cid, pFD->blkno, key, &keyLen); + *handle = taosLRUCacheLookup(pCache, key, keyLen); + + return code; +} + +int32_t tsdbCacheSetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, uint8_t *pPage) { + int32_t code = 0; + char key[128] = {0}; + int keyLen = 0; + LRUHandle *handle = NULL; + + getBCacheKey(pFD->fid, pFD->cid, pgno, key, &keyLen); + taosThreadMutexLock(&pFD->pTsdb->pgMutex); + handle = taosLRUCacheLookup(pFD->pTsdb->pgCache, key, keyLen); + if (!handle) { + size_t charge = pFD->szPage; + _taos_lru_deleter_t deleter = deleteBCache; + uint8_t *pPg = taosMemoryMalloc(charge); + memcpy(pPg, pPage, charge); + + LRUStatus status = + taosLRUCacheInsert(pCache, key, keyLen, pPg, charge, deleter, &handle, TAOS_LRU_PRIORITY_LOW, NULL); + if (status != TAOS_LRU_STATUS_OK) { + // ignore cache updating if not ok + // code = TSDB_CODE_OUT_OF_MEMORY; + } + } + taosThreadMutexUnlock(&pFD->pTsdb->pgMutex); + + tsdbCacheRelease(pFD->pTsdb->pgCache, handle); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 65f0f1b94d..def9a73d10 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -178,7 +178,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { pFD->blkno = (pgno + tsS3BlockSize - 1) / tsS3BlockSize; code = tsdbCacheGetBlockS3(pFD->pTsdb->bCache, pFD, &handle); if (code != TSDB_CODE_SUCCESS || handle == NULL) { - tsdbBCacheRelease(pFD->pTsdb->bCache, handle); + tsdbCacheRelease(pFD->pTsdb->bCache, handle); if (code == TSDB_CODE_SUCCESS && !handle) { code = TSDB_CODE_OUT_OF_MEMORY; } @@ -190,7 +190,7 @@ static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { int64_t blk_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage; memcpy(pFD->pBuf, pBlock + (offset - blk_offset), pFD->szPage); - tsdbBCacheRelease(pFD->pTsdb->bCache, handle); + tsdbCacheRelease(pFD->pTsdb->bCache, handle); } else { // seek int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); @@ -254,7 +254,7 @@ _exit: return code; } -int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { +static int32_t tsdbReadFileImp(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { int32_t code = 0; int64_t n = 0; int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); @@ -283,6 +283,117 @@ _exit: return code; } +static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { + int32_t code = 0; + int64_t n = 0; + int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); + int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); + int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); + int64_t bOffset = fOffset % pFD->szPage; + + ASSERT(bOffset < szPgCont); + + // 1, find pgnoStart & pgnoEnd to fetch from s3, if all pgs are local, no need to fetch + // 2, fetch pgnoStart ~ pgnoEnd from s3 + // 3, store pgs to pcache & last pg to pFD->pBuf + // 4, deliver pgs to [pBuf, pBuf + size) + + while (n < size) { + if (pFD->pgno != pgno) { + LRUHandle *handle = NULL; + code = tsdbCacheGetPageS3(pFD->pTsdb->pgCache, pFD, pgno, &handle); + if (code != TSDB_CODE_SUCCESS) { + if (handle) { + tsdbCacheRelease(pFD->pTsdb->pgCache, handle); + } + goto _exit; + } + + if (!handle) { + break; + } + + uint8_t *pPage = (uint8_t *)taosLRUCacheValue(pFD->pTsdb->pgCache, handle); + memcpy(pFD->pBuf, pPage, pFD->szPage); + tsdbCacheRelease(pFD->pTsdb->pgCache, handle); + + // check + if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + pFD->pgno = pgno; + } + + int64_t nRead = TMIN(szPgCont - bOffset, size - n); + memcpy(pBuf + n, pFD->pBuf + bOffset, nRead); + + n += nRead; + pgno++; + bOffset = 0; + } + + if (n < size) { + // 2, retrieve pgs from s3 + uint8_t *pBlock = NULL; + int64_t retrieve_offset = PAGE_OFFSET(pgno, pFD->szPage); + int64_t pgnoEnd = pgno - 1 + (size - n + szPgCont - 1) / szPgCont; + int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage; + code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, &pBlock); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + // 3, Store Pages in Cache + int nPage = pgnoEnd - pgno + 1; + for (int i = 0; i < nPage; ++i) { + tsdbCacheSetPageS3(pFD->pTsdb->pgCache, pFD, pgno, pBlock + i * pFD->szPage); + + memcpy(pFD->pBuf, pBlock + i * pFD->szPage, pFD->szPage); + + // check + if (pgno > 1 && !taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; + } + + pFD->pgno = pgno; + + int64_t nRead = TMIN(szPgCont - bOffset, size - n); + memcpy(pBuf + n, pFD->pBuf + bOffset, nRead); + + n += nRead; + pgno++; + bOffset = 0; + } + + taosMemoryFree(pBlock); + } + +_exit: + return code; +} + +int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { + int32_t code = 0; + if (!pFD->pFD) { + code = tsdbOpenFileImpl(pFD); + if (code) { + goto _exit; + } + } + + if (pFD->s3File && tsS3BlockSize < 0) { + return tsdbReadFileS3(pFD, offset, pBuf, size); + } else { + return tsdbReadFileImp(pFD, offset, pBuf, size); + } + +_exit: + return code; +} + int32_t tsdbFsyncFile(STsdbFD *pFD) { int32_t code = 0; From 221afff9eefdbde51126cc080375b356c092145c Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 1 Nov 2023 11:00:47 +0800 Subject: [PATCH 07/13] vnode/write: remove keep modify code block --- source/dnode/vnode/src/vnd/vnodeSvr.c | 46 +++++++++++++-------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b3675380c7..b31463ac00 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -16,12 +16,11 @@ #include "audit.h" #include "tencode.h" #include "tmsg.h" +#include "tstrbuild.h" #include "vnd.h" #include "vndCos.h" #include "vnode.h" #include "vnodeInt.h" -#include "audit.h" -#include "tstrbuild.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -275,6 +274,7 @@ static int32_t vnodePreProcessSubmitTbData(SVnode *pVnode, SDecoder *pCoder, int } } */ + TSKEY minKey = now - tsTickPerMin[pVnode->config.tsdbCfg.precision] * keep; TSKEY maxKey = tsMaxKeyByPrecision[pVnode->config.tsdbCfg.precision]; if (submitTbData.flags & SUBMIT_REQ_COLUMN_DATA_FORMAT) { @@ -906,7 +906,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, rsp.pArray = taosArrayInit(req.nReqs, sizeof(cRsp)); tbUids = taosArrayInit(req.nReqs, sizeof(int64_t)); - tbNames = taosArrayInit(req.nReqs, sizeof(char*)); + tbNames = taosArrayInit(req.nReqs, sizeof(char *)); if (rsp.pArray == NULL || tbUids == NULL || tbNames == NULL) { rcode = -1; terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -952,8 +952,8 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosArrayPush(rsp.pArray, &cRsp); - if(tsEnableAuditCreateTable){ - char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + if (tsEnableAuditCreateTable) { + char *str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); strcpy(str, pCreateReq->name); taosArrayPush(tbNames, &str); } @@ -978,24 +978,24 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen); tEncodeSVCreateTbBatchRsp(&encoder, &rsp); - if(tsEnableAuditCreateTable){ + if (tsEnableAuditCreateTable) { int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); SStringBuilder sb = {0}; - for(int32_t iReq = 0; iReq < req.nReqs; iReq++){ - char** key = (char**)taosArrayGet(tbNames, iReq); + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + char **key = (char **)taosArrayGet(tbNames, iReq); taosStringBuilderAppendStringLen(&sb, *key, strlen(*key)); - if(iReq < req.nReqs - 1){ + if (iReq < req.nReqs - 1) { taosStringBuilderAppendChar(&sb, ','); } taosMemoryFreeClear(*key); } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); + size_t len = 0; + char *keyJoined = taosStringBuilderGetResult(&sb, &len); auditRecord(NULL, clusterId, "createTable", name.dbname, "", keyJoined, len); @@ -1007,7 +1007,7 @@ _exit: pCreateReq = req.pReqs + iReq; taosMemoryFree(pCreateReq->sql); taosMemoryFree(pCreateReq->comment); - taosArrayDestroy(pCreateReq->ctb.tagName); + taosArrayDestroy(pCreateReq->ctb.tagName); } taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); taosArrayDestroy(tbUids); @@ -1166,7 +1166,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in // process req tbUids = taosArrayInit(req.nReqs, sizeof(int64_t)); rsp.pArray = taosArrayInit(req.nReqs, sizeof(SVDropTbRsp)); - tbNames = taosArrayInit(req.nReqs, sizeof(char*)); + tbNames = taosArrayInit(req.nReqs, sizeof(char *)); if (tbUids == NULL || rsp.pArray == NULL || tbNames == NULL) goto _exit; for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { @@ -1188,9 +1188,9 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in } taosArrayPush(rsp.pArray, &dropTbRsp); - - if(tsEnableAuditCreateTable){ - char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + + if (tsEnableAuditCreateTable) { + char *str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); strcpy(str, pDropTbReq->name); taosArrayPush(tbNames, &str); } @@ -1199,30 +1199,30 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in tqUpdateTbUidList(pVnode->pTq, tbUids, false); tdUpdateTbUidList(pVnode->pSma, pStore, false); - if(tsEnableAuditCreateTable){ + if (tsEnableAuditCreateTable) { int64_t clusterId = pVnode->config.syncCfg.nodeInfo[0].clusterId; SName name = {0}; tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); SStringBuilder sb = {0}; - for(int32_t iReq = 0; iReq < req.nReqs; iReq++){ - char** key = (char**)taosArrayGet(tbNames, iReq); + for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { + char **key = (char **)taosArrayGet(tbNames, iReq); taosStringBuilderAppendStringLen(&sb, *key, strlen(*key)); - if(iReq < req.nReqs - 1){ + if (iReq < req.nReqs - 1) { taosStringBuilderAppendChar(&sb, ','); } taosMemoryFreeClear(*key); } - size_t len = 0; - char* keyJoined = taosStringBuilderGetResult(&sb, &len); + size_t len = 0; + char *keyJoined = taosStringBuilderGetResult(&sb, &len); auditRecord(NULL, clusterId, "dropTable", name.dbname, "", keyJoined, len); taosStringBuilderDestroy(&sb); } - + _exit: taosArrayDestroy(tbUids); tdUidStoreFree(pStore); From 0f5a124cb1a43fa7d1ba17fa91a3cc43cc20bbfc Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 1 Nov 2023 11:08:39 +0800 Subject: [PATCH 08/13] config/block-size: enable dynamic config for debugging --- source/common/src/tglobal.c | 13 +++++++------ source/dnode/mnode/impl/src/mndDnode.c | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3d1a1ccb16..7cf1586abb 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -105,9 +105,9 @@ bool tsEnableAuditCreateTable = true; // telem #ifdef TD_ENTERPRISE -bool tsEnableTelem = false; +bool tsEnableTelem = false; #else -bool tsEnableTelem = true; +bool tsEnableTelem = true; #endif int32_t tsTelemInterval = 43200; char tsTelemServer[TSDB_FQDN_LEN] = "telemetry.tdengine.com"; @@ -115,9 +115,9 @@ uint16_t tsTelemPort = 80; char *tsTelemUri = "/report"; #ifdef TD_ENTERPRISE -bool tsEnableCrashReport = false; +bool tsEnableCrashReport = false; #else -bool tsEnableCrashReport = true; +bool tsEnableCrashReport = true; #endif char *tsClientCrashReportUri = "/ccrashreport"; char *tsSvrCrashReportUri = "/dcrashreport"; @@ -677,7 +677,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) + return -1; if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1; @@ -695,7 +696,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, 2048, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 6d77a48110..3b3e9eb03b 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1228,6 +1228,22 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "monitor"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { + int32_t optLen = strlen("s3blocksize"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; + + if (flag > 1024 * 1024) { + mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId, + flag); + terrno = TSDB_CODE_INVALID_CFG; + tFreeSMCfgDnodeReq(&cfgReq); + return -1; + } + + strcpy(dcfgReq.config, "s3blocksize"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "s3blockcachesize", 16) == 0) { int32_t optLen = strlen("s3blockcachesize"); int32_t flag = -1; From 6f0fa5d47e99d1ffe16f1a5aa45053ce23700e8b Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 1 Nov 2023 14:56:57 +0800 Subject: [PATCH 09/13] tsdb/retention: remove incorrect right brace --- source/dnode/vnode/src/tsdb/tsdbRetention.c | 59 ++++++++++----------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 40561ddb42..0fc1e1b64b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -324,41 +324,40 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { for (int32_t ftype = 0; ftype < TSDB_FTYPE_MAX && (fobj = fset->farr[ftype], 1); ++ftype) { if (fobj == NULL) continue; - int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); + int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); - if (fobj->f->did.level == did.level) { - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1 && - taosCheckExistFile(fobj->fname)) { - int32_t mtime = 0; - taosStatFile(fobj->fname, NULL, &mtime, NULL); - if (mtime < rtner->now - tsS3UploadDelaySec) { - code = tsdbMigrateDataFileS3(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } + if (fobj->f->did.level == did.level) { + if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1 && + taosCheckExistFile(fobj->fname)) { + int32_t mtime = 0; + taosStatFile(fobj->fname, NULL, &mtime, NULL); + if (mtime < rtner->now - tsS3UploadDelaySec) { + code = tsdbMigrateDataFileS3(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); } - - continue; } - /* - if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) { - code = tsdbMigrateDataFileS3(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - } else { - if (tsS3Enabled) { - int64_t fsize = 0; - if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { - code = TAOS_SYSTEM_ERROR(terrno); - tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode), - __func__, fobj->fname, tstrerror(code)); TSDB_CHECK_CODE(code, lino, _exit); - } - s3EvictCache(fobj->fname, fsize * 2); - } - */ - code = tsdbDoMigrateFileObj(rtner, fobj, &did); - TSDB_CHECK_CODE(code, lino, _exit); - //} + continue; } + /* + if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && did.level == nlevel - 1) { + code = tsdbMigrateDataFileS3(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + } else { + + if (tsS3Enabled) { + int64_t fsize = 0; + if (taosStatFile(fobj->fname, &fsize, NULL, NULL) < 0) { + code = TAOS_SYSTEM_ERROR(terrno); + tsdbError("vgId:%d %s failed since file:%s stat failed, reason:%s", TD_VID(rtner->tsdb->pVnode), + __func__, fobj->fname, tstrerror(code)); TSDB_CHECK_CODE(code, lino, _exit); + } + s3EvictCache(fobj->fname, fsize * 2); + } + */ + code = tsdbDoMigrateFileObj(rtner, fobj, &did); + TSDB_CHECK_CODE(code, lino, _exit); + //} } // stt From 52fd71ce13f07ba1788893b228b562e5191d4352 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 1 Nov 2023 18:00:31 +0800 Subject: [PATCH 10/13] config/block-size: default to -1 to enable page cache --- source/common/src/tglobal.c | 7 ++++--- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 7cf1586abb..485b67e1a7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -280,8 +280,8 @@ int8_t tsS3Enabled = false; int8_t tsS3Https = true; char tsS3Hostname[TSDB_FQDN_LEN] = ""; -int32_t tsS3BlockSize = 4096; // number of tsdb pages -int32_t tsS3BlockCacheSize = 16; // number of blocks +int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) +int32_t tsS3BlockCacheSize = 1024; // number of blocks/pages (16) int32_t tsS3UploadDelaySec = 60 * 60; #ifndef _STORAGE @@ -697,7 +697,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER) != 0) + return -1; if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 3b3e9eb03b..e2c21ed012 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1250,7 +1250,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); if (code < 0) return code; - if (flag < 4 || flag > 1024 * 1024) { + if (flag < 4 || flag > 1024 * 1024 * 1024) { mError("dnode:%d, failed to config s3BlockCacheSize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId, flag); terrno = TSDB_CODE_INVALID_CFG; From d0327e07b833bb3086a294ad44ba562e82e7c18f Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 2 Nov 2023 09:42:19 +0800 Subject: [PATCH 11/13] tsdb/pg-cache: use pgno for cache key instead of blkno --- source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 7cea469c34..94e6152487 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -3169,7 +3169,7 @@ int32_t tsdbCacheGetPageS3(SLRUCache *pCache, STsdbFD *pFD, int64_t pgno, LRUHan char key[128] = {0}; int keyLen = 0; - getBCacheKey(pFD->fid, pFD->cid, pFD->blkno, key, &keyLen); + getBCacheKey(pFD->fid, pFD->cid, pgno, key, &keyLen); *handle = taosLRUCacheLookup(pCache, key, keyLen); return code; From 46d3be1f08eeebee07e6d3c53eb05cbbc649e4e6 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 2 Nov 2023 10:10:03 +0800 Subject: [PATCH 12/13] config/page-cache-size: new parameter for pcache size --- source/common/src/tglobal.c | 17 +++++++++++++---- source/dnode/mnode/impl/src/mndDnode.c | 18 +++++++++++++++++- source/dnode/vnode/src/inc/vndCos.h | 1 + source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 485b67e1a7..3c16554ac6 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -280,8 +280,9 @@ int8_t tsS3Enabled = false; int8_t tsS3Https = true; char tsS3Hostname[TSDB_FQDN_LEN] = ""; -int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) -int32_t tsS3BlockCacheSize = 1024; // number of blocks/pages (16) +int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) +int32_t tsS3BlockCacheSize = 16; // number of blocks +int32_t tsS3PageCacheSize = 1024; // number of pages int32_t tsS3UploadDelaySec = 60 * 60; #ifndef _STORAGE @@ -697,8 +698,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER) != 0) - return -1; + if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER) != 0) return -1; @@ -1132,6 +1133,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsS3BlockSize = cfgGetItem(pCfg, "s3BlockSize")->i32; tsS3BlockCacheSize = cfgGetItem(pCfg, "s3BlockCacheSize")->i32; + tsS3PageCacheSize = cfgGetItem(pCfg, "s3PageCacheSize")->i32; tsS3UploadDelaySec = cfgGetItem(pCfg, "s3UploadDelaySec")->i32; GRANT_CFG_GET; @@ -1726,6 +1728,13 @@ void taosCfgDynamicOptions(const char *option, const char *value) { return; } + if (strcasecmp(option, "s3PageCacheSize") == 0) { + int32_t newS3PageCacheSize = atoi(value); + uInfo("s3PageCacheSize set from %d to %d", tsS3PageCacheSize, newS3PageCacheSize); + tsS3PageCacheSize = newS3PageCacheSize; + return; + } + if (strcasecmp(option, "s3UploadDelaySec") == 0) { int32_t newS3UploadDelaysec = atoi(value); uInfo("s3UploadDelaySec set from %d to %d", tsS3UploadDelaySec, newS3UploadDelaysec); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index e2c21ed012..46f7c6c3e2 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1250,7 +1250,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); if (code < 0) return code; - if (flag < 4 || flag > 1024 * 1024 * 1024) { + if (flag < 4 || flag > 1024 * 1024) { mError("dnode:%d, failed to config s3BlockCacheSize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId, flag); terrno = TSDB_CODE_INVALID_CFG; @@ -1260,6 +1260,22 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { strcpy(dcfgReq.config, "s3blockcachesize"); snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strncasecmp(cfgReq.config, "s3pagecachesize", 16) == 0) { + int32_t optLen = strlen("s3pagecachesize"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; + + if (flag < 4 || flag > 1024 * 1024 * 1024) { + mError("dnode:%d, failed to config s3PageCacheSize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId, + flag); + terrno = TSDB_CODE_INVALID_CFG; + tFreeSMCfgDnodeReq(&cfgReq); + return -1; + } + + strcpy(dcfgReq.config, "s3pagecachesize"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); } else if (strncasecmp(cfgReq.config, "s3uploaddelaysec", 16) == 0) { int32_t optLen = strlen("s3uploaddelaysec"); int32_t flag = -1; diff --git a/source/dnode/vnode/src/inc/vndCos.h b/source/dnode/vnode/src/inc/vndCos.h index 6612b5c653..8581b039f8 100644 --- a/source/dnode/vnode/src/inc/vndCos.h +++ b/source/dnode/vnode/src/inc/vndCos.h @@ -27,6 +27,7 @@ extern "C" { extern int8_t tsS3Enabled; extern int32_t tsS3BlockSize; extern int32_t tsS3BlockCacheSize; +extern int32_t tsS3PageCacheSize; extern int32_t tsS3UploadDelaySec; int32_t s3Init(); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 94e6152487..bf6f0cf4d6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -92,7 +92,7 @@ static int32_t tsdbOpenPgCache(STsdb *pTsdb) { // SLRUCache *pCache = taosLRUCacheInit(10 * 1024 * 1024, 0, .5); int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; - SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3BlockCacheSize * szPage, 0, .5); + SLRUCache *pCache = taosLRUCacheInit((int64_t)tsS3PageCacheSize * szPage, 0, .5); if (pCache == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; From 1cd12845202a6d3d1272f88c70aa3726a6234423 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 2 Nov 2023 12:43:06 +0800 Subject: [PATCH 13/13] config/pcache: default page cache to 4096 --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3c16554ac6..8e824b4d67 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -282,7 +282,7 @@ char tsS3Hostname[TSDB_FQDN_LEN] = ""; int32_t tsS3BlockSize = -1; // number of tsdb pages (4096) int32_t tsS3BlockCacheSize = 16; // number of blocks -int32_t tsS3PageCacheSize = 1024; // number of pages +int32_t tsS3PageCacheSize = 4096; // number of pages int32_t tsS3UploadDelaySec = 60 * 60; #ifndef _STORAGE