From 159873692cfedac7bc855bd1eb3f6fa2575a0c5a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 2 Nov 2023 20:13:29 +0800 Subject: [PATCH 01/20] feat:checkpoint bakeup using rsync --- include/common/tglobal.h | 4 + .../src/inc/vndCos.h => include/util/cos.h | 3 +- include/util/rsync.h | 24 +++ source/common/src/tglobal.c | 10 +- source/dnode/snode/src/snode.c | 4 + source/dnode/vnode/CMakeLists.txt | 1 - source/dnode/vnode/src/tsdb/tsdbCache.c | 2 +- source/dnode/vnode/src/tsdb/tsdbFS2.c | 2 +- .../dnode/vnode/src/tsdb/tsdbReaderWriter.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRetention.c | 3 +- source/dnode/vnode/src/tsdb/tsdbWrite.c | 2 +- source/dnode/vnode/src/vnd/vnodeModule.c | 2 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 2 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/stream/inc/streamInt.h | 11 ++ source/libs/stream/src/streamCheckpoint.c | 132 +++++++++++++ source/libs/stream/test/CMakeLists.txt | 11 ++ source/libs/stream/test/checkpointTest.cpp | 61 ++++++ .../src/vnd/vnodeCos.c => util/src/cos.c} | 2 +- source/util/src/rsync.c | 178 ++++++++++++++++++ 20 files changed, 445 insertions(+), 13 deletions(-) rename source/dnode/vnode/src/inc/vndCos.h => include/util/cos.h (96%) create mode 100644 include/util/rsync.h create mode 100644 source/libs/stream/test/checkpointTest.cpp rename source/{dnode/vnode/src/vnd/vnodeCos.c => util/src/cos.c} (99%) create mode 100644 source/util/src/rsync.c diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 3e29703070..ccae8f02e2 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -82,6 +82,10 @@ extern int32_t tsHeartbeatTimeout; // vnode extern int64_t tsVndCommitMaxIntervalMs; +// snode +extern char tsSnodeIp[]; +extern char tsCheckpointBackupDir[]; + // mnode extern int64_t tsMndSdbWriteDelta; extern int64_t tsMndLogRetention; diff --git a/source/dnode/vnode/src/inc/vndCos.h b/include/util/cos.h similarity index 96% rename from source/dnode/vnode/src/inc/vndCos.h rename to include/util/cos.h index bb4d284f0e..9b91afea8d 100644 --- a/source/dnode/vnode/src/inc/vndCos.h +++ b/include/util/cos.h @@ -16,7 +16,7 @@ #ifndef _TD_VND_COS_H_ #define _TD_VND_COS_H_ -#include "vnd.h" +#include "os.h" #ifdef __cplusplus extern "C" { @@ -24,6 +24,7 @@ extern "C" { #define S3_BLOCK_CACHE +extern int8_t tsS3StreamEnabled; extern int8_t tsS3Enabled; extern int32_t tsS3BlockSize; extern int32_t tsS3BlockCacheSize; diff --git a/include/util/rsync.h b/include/util/rsync.h new file mode 100644 index 0000000000..50b27b95e0 --- /dev/null +++ b/include/util/rsync.h @@ -0,0 +1,24 @@ +// +// Created by mingming wanng on 2023/11/2. +// + +#ifndef TDENGINE_RSYNC_H +#define TDENGINE_RSYNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tarray.h" + +void stopRsync(); +void startRsync(); +int uploadRsync(char* id, SArray* fileList); +int downloadRsync(char* id, char* path); +int deleteRsync(char* id); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_RSYNC_H diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c6cff27011..241481df6a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -133,6 +133,10 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; // bool tsSmlDataFormat = false; // int32_t tsSmlBatchSize = 10000; +// checkpoint backup +char tsSnodeIp[TSDB_FQDN_LEN] = {0}; +char tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/"; + // tmq int32_t tmqMaxTopicNum = 20; // query @@ -275,6 +279,7 @@ char tsS3AccessKeySecret[TSDB_FQDN_LEN] = ""; char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_FQDN_LEN] = ""; int8_t tsS3Enabled = false; +int8_t tsS3StreamEnabled = false; int8_t tsS3Https = true; char tsS3Hostname[TSDB_FQDN_LEN] = ""; @@ -338,9 +343,10 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN); } } - if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) { + if (tsS3BucketName[0] != '<') { #if defined(USE_COS) || defined(USE_S3) - tsS3Enabled = true; + if(tsDiskCfgNum > 1) tsS3Enabled = true; + tsS3StreamEnabled = true; #endif } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 6451dba2da..9a6e51db31 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "rsync.h" #include "executor.h" #include "sndInt.h" #include "tstream.h" @@ -120,6 +121,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { goto FAIL; } + stopRsync(); + startRsync(); + // todo fix it: send msg to mnode to rollback to an existed checkpoint streamMetaInitForSnode(pSnode->pMeta); return pSnode; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index dcc9f9a115..27cb0f93f7 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -8,7 +8,6 @@ set( "src/vnd/vnodeCommit.c" "src/vnd/vnodeQuery.c" "src/vnd/vnodeModule.c" - "src/vnd/vnodeCos.c" "src/vnd/vnodeSvr.c" "src/vnd/vnodeSync.c" "src/vnd/vnodeSnapshot.c" diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index ca3fb7027f..1e73fad8f6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -12,11 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "cos.h" #include "tsdb.h" #include "tsdbDataFileRW.h" #include "tsdbReadUtil.h" #include "vnd.h" -#include "vndCos.h" #define ROCKS_BATCH_SIZE (4096) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 38d221d978..348397272d 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -14,9 +14,9 @@ */ #include "tsdbFS2.h" +#include "cos.h" #include "tsdbUpgrade.h" #include "vnd.h" -#include "vndCos.h" #define BLOCK_COMMIT_FACTOR 3 diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index f3bcfef703..cc15fb85ca 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#include "cos.h" #include "tsdb.h" -#include "vndCos.h" static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 6c41b46c73..2400c41d52 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -15,7 +15,8 @@ #include "tsdb.h" #include "tsdbFS2.h" -#include "vndCos.h" +#include "cos.h" +#include "vnd.h" typedef struct { STsdb *tsdb; diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 5949b103d5..e75079403e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#include "cos.h" #include "tsdb.h" -#include "vndCos.h" /** * @brief max key by precision diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 6ccce5c9d7..df08fb8a2b 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#include "cos.h" #include "vnd.h" -#include "vndCos.h" typedef struct SVnodeTask SVnodeTask; struct SVnodeTask { diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 3bdecee79b..ff79e83d72 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ +#include "cos.h" #include "sync.h" #include "tsdb.h" #include "vnd.h" -#include "vndCos.h" int32_t vnodeGetPrimaryDir(const char *relPath, int32_t diskPrimary, STfs *pTfs, char *buf, size_t bufLen) { if (pTfs) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b6a4aaf388..8f6f5df850 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -16,7 +16,7 @@ #include "tencode.h" #include "tmsg.h" #include "vnd.h" -#include "vndCos.h" +#include "cos.h" #include "vnode.h" #include "vnodeInt.h" #include "audit.h" diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 4cd8319a07..806124bc58 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -137,6 +137,17 @@ void* streamQueueNextItem(SStreamQueue* pQueue); void streamFreeQitem(SStreamQueueItem* data); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); +//#define CHECKPOINT_PATH_LEN 128 +//typedef struct SChekpointDataHeader{ +// int64_t size; +// char name[CHECKPOINT_PATH_LEN]; +// char id[CHECKPOINT_PATH_LEN]; +//} SChekpointDataHeader; + +int uploadCheckpoint(char* id, SArray* fileList); +int downloadCheckpoint(char* id, char* path); +int deleteCheckpoint(char* id); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 2cde368195..238c7c2329 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -14,6 +14,7 @@ */ #include "streamInt.h" +#include "rsync.h" int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; @@ -357,3 +358,134 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { return code; } + + +//static int64_t kBlockSize = 64 * 1024; +//static int sendCheckpointToS3(char* id, SArray* fileList){ +// code = s3PutObjectFromFile2(from->fname, object_name); +// return 0; +//} +//static int sendCheckpointToSnode(char* id, SArray* fileList){ +// if(strlen(id) >= CHECKPOINT_PATH_LEN){ +// tqError("uploadCheckpoint id name too long, name:%s", id); +// return -1; +// } +// uint8_t* buf = taosMemoryCalloc(1, sizeof(SChekpointDataHeader) + kBlockSize); +// if(buf == NULL){ +// tqError("uploadCheckpoint malloc failed"); +// return -1; +// } +// +// SChekpointDataHeader* pHdr = (SChekpointDataHeader*)buf; +// strcpy(pHdr->id, id); +// +// TdFilePtr fd = NULL; +// for(int i = 0; i < taosArrayGetSize(fileList); i++){ +// char* name = (char*)taosArrayGetP(fileList, i); +// if(strlen(name) >= CHECKPOINT_PATH_LEN){ +// tqError("uploadCheckpoint file name too long, name:%s", name); +// return -1; +// } +// int64_t offset = 0; +// +// fd = taosOpenFile(name, TD_FILE_READ); +// tqDebug("uploadCheckpoint open file %s, file index: %d", name, i); +// +// while(1){ +// int64_t nread = taosPReadFile(fd, buf + sizeof(SChekpointDataHeader), kBlockSize, offset); +// if (nread == -1) { +// taosCloseFile(&fd); +// taosMemoryFree(buf); +// tqError("uploadCheckpoint failed to read file name:%s,reason:%d", name, errno); +// return -1; +// } else if (nread == 0){ +// tqDebug("uploadCheckpoint no data read, close file:%s, move to next file, open and read", name); +// taosCloseFile(&fd); +// break; +// } else if (nread == kBlockSize){ +// offset += nread; +// } else { +// taosCloseFile(&fd); +// offset = 0; +// } +// tqDebug("uploadCheckpoint read file %s, size:%" PRId64 ", current offset:%" PRId64, name, nread, offset); +// +// +// pHdr->size = nread; +// strcpy(pHdr->name, name); +// +// SRpcMsg rpcMsg = {0}; +// int32_t bytes = sizeof(SChekpointDataHeader) + nread; +// rpcMsg.pCont = rpcMallocCont(bytes); +// rpcMsg.msgType = TDMT_SYNC_SNAPSHOT_SEND; +// rpcMsg.contLen = bytes; +// if (rpcMsg.pCont == NULL) { +// tqError("uploadCheckpoint malloc failed"); +// taosCloseFile(&fd); +// taosMemoryFree(buf); +// return -1; +// } +// memcpy(rpcMsg.pCont, buf, bytes); +// int try = 3; +// int32_t code = 0; +// while(try-- > 0){ +// code = tmsgSendReq(pEpSet, &rpcMsg); +// if(code == 0) +// break; +// taosMsleep(10); +// } +// if(code != 0){ +// tqError("uploadCheckpoint send request failed code:%d", code); +// taosCloseFile(&fd); +// taosMemoryFree(buf); +// return -1; +// } +// +// if(offset == 0){ +// break; +// } +// } +// } +// +// taosMemoryFree(buf); + +//} + +int uploadCheckpoint(char* id, SArray* fileList){ + if(id == NULL || fileList == NULL || strlen(id) == 0 || taosArrayGetSize(fileList) == 0){ + stError("uploadCheckpoint parameters invalid"); + return -1; + } + if(strlen(tsSnodeIp) != 0){ + uploadRsync(id, fileList); +// }else if(tsS3StreamEnabled){ + + } + return 0; +} + +int downloadCheckpoint(char* id, char* path){ + if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0){ + stError("downloadCheckpoint parameters invalid"); + return -1; + } + if(strlen(tsSnodeIp) != 0){ + downloadRsync(id, path); +// }else if(tsS3StreamEnabled){ + + } + return 0; +} + +int deleteCheckpoint(char* id){ + if(id == NULL || strlen(id) == 0){ + stError("deleteCheckpoint parameters invalid"); + return -1; + } + if(strlen(tsSnodeIp) != 0){ + deleteRsync(id); +// }else if(tsS3StreamEnabled){ + + } + return 0; +} diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index 629b04ae51..d756b71e29 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -18,6 +18,17 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" ) +ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) +TARGET_LINK_LIBRARIES( + checkpointTest + PUBLIC os common gtest stream executor qcom index transport util +) + +TARGET_INCLUDE_DIRECTORIES( + checkpointTest + PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +) + add_test( NAME streamUpdateTest COMMAND streamUpdateTest diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp new file mode 100644 index 0000000000..56614cc537 --- /dev/null +++ b/source/libs/stream/test/checkpointTest.cpp @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "rsync.h" +#include "streamInt.h" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + + strcpy(tsSnodeIp, "127.0.0.1"); + return RUN_ALL_TESTS(); +} + +TEST(testCase, checkpointUpload_Test) { + stopRsync(); + startRsync(); + + taosSsleep(5); + SArray* fileList = taosArrayInit(0, POINTER_BYTES); + char* id = "2013892036"; + char* file1 = "/Users/mingmingwanng/wal1"; + char* file2 = "/Users/mingmingwanng/java_error_in_clion.hprof"; + taosArrayPush(fileList, &file1); + taosArrayPush(fileList, &file2); + + uploadCheckpoint(id, fileList); +} + +TEST(testCase, checkpointDownload_Test) { + char* id = "2013892036"; + downloadRsync(id, "/Users/mingmingwanng/rsync/tmp"); +} + +TEST(testCase, checkpointDelete_Test) { + char* id = "2013892036"; + deleteRsync(id); +} diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/util/src/cos.c similarity index 99% rename from source/dnode/vnode/src/vnd/vnodeCos.c rename to source/util/src/cos.c index 6e36739f5a..23b2d53990 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/util/src/cos.c @@ -1,6 +1,6 @@ #define ALLOW_FORBID_FUNC -#include "vndCos.h" +#include "cos.h" extern char tsS3Endpoint[]; extern char tsS3AccessKeyId[]; diff --git a/source/util/src/rsync.c b/source/util/src/rsync.c new file mode 100644 index 0000000000..940c631c65 --- /dev/null +++ b/source/util/src/rsync.c @@ -0,0 +1,178 @@ +// +// Created by mingming wanng on 2023/11/2. +// +#include "rsync.h" +#include +#include "tglobal.h" + +#define ERRNO_ERR_FORMAT "errno:%d,msg:%s" +#define ERRNO_ERR_DATA errno,strerror(errno) + +// deleteRsync function produce empty directories, traverse base directory to remove them +static void removeEmptyDir(){ + TdDirPtr pDir = taosOpenDir(tsCheckpointBackupDir); + if (pDir == NULL) return; + + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + if (!taosDirEntryIsDir(de)) { + continue; + } + + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; + + char filename[PATH_MAX] = {0}; + snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de)); + + TdDirPtr pDirTmp = taosOpenDir(filename); + TdDirEntryPtr deTmp = NULL; + bool empty = true; + while ((deTmp = taosReadDir(pDirTmp)) != NULL){ + if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue; + empty = false; + } + if(empty) taosRemoveDir(filename); + taosCloseDir(&pDirTmp); + } + + taosCloseDir(&pDir); +} + +static int generateConfigFile(char* confDir){ + TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + uError("[rsync] open conf file error, dir:%s,"ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); + return -1; + } + + char confContent[PATH_MAX*4] = {0}; + snprintf(confContent, PATH_MAX*4, + "uid = root\n" + "gid = root\n" + "use chroot = false\n" + "max connections = 200\n" + "timeout = 100\n" + "lock file = %srsync.lock\n" + "log file = %srsync.log\n" + "ignore errors = true\n" + "read only = false\n" + "list = false\n" + "[checkpoint]\n" + "path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir, tsCheckpointBackupDir); + uDebug("[rsync] conf:%s", confContent); + if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){ + uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + taosCloseFile(&pFile); + return -1; + } + + taosCloseFile(&pFile); + return 0; +} + +static int execCommand(char* command){ + int try = 3; + int32_t code = 0; + while(try-- > 0) { + code = system(command); + if (code == 0) { + break; + } + taosMsleep(10); + } + return code; +} + +void stopRsync(){ + int code = system("pkill rsync"); + if(code != 0){ + uError("[rsync] stop rsync server failed,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + return; + } + uDebug("[rsync] stop rsync server successful"); +} + +void startRsync(){ + if(taosMulMkDir(tsCheckpointBackupDir) != 0){ + uError("[rsync] build checkpoint backup dir failed, dir:%s,"ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA); + return; + } + removeEmptyDir(); + + char confDir[PATH_MAX] = {0}; + snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir); + + int code = generateConfigFile(confDir); + if(code != 0){ + return; + } + + char cmd[PATH_MAX] = {0}; + snprintf(cmd, PATH_MAX, "rsync --daemon --config=%s", confDir); + // start rsync service to backup checkpoint + code = system(cmd); + if(code != 0){ + uError("[rsync] start server failed, code:%d,"ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return; + } + uDebug("[rsync] start server successful"); +} + +int uploadRsync(char* id, SArray* fileList){ + for(int i = 0; i < taosArrayGetSize(fileList); i++) { + char* fullName = (char*)taosArrayGetP(fileList, i); + char command[PATH_MAX] = {0}; +// char* name = strrchr(fullName, '/'); +// if(name == NULL){ +// uError("[rsync] file name invalid, name:%s", name); +// return -1; +// } +// name = name + 1; + snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/", + fullName, tsSnodeIp, id); + + int code = execCommand(command); + if(code != 0){ + uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + } + uDebug("[rsync] upload data:%s successful", id); + return 0; +} + +int downloadRsync(char* id, char* path){ + char command[PATH_MAX] = {0}; + snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s", + tsSnodeIp, id, path); + + int code = execCommand(command); + if(code != 0){ + uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + uDebug("[rsync] down data:%s successful", id); + return 0; +} + +int deleteRsync(char* id){ + char* tmp = "./tmp_empty/"; + int code = taosMkDir(tmp); + if(code != 0){ + uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + char command[PATH_MAX] = {0}; + snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/", + tmp, tsSnodeIp, id); + + code = execCommand(command); + taosRemoveDir(tmp); + if(code != 0){ + uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + uDebug("[rsync] delete data:%s successful", id); + + return 0; +} \ No newline at end of file From 3ac11eba0eceeae8e2a9785c806c8e66aa9f472f Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 3 Nov 2023 11:33:18 +0800 Subject: [PATCH 02/20] fix:modify upload checkpoint interface --- include/util/rsync.h | 2 +- source/libs/stream/inc/streamInt.h | 2 +- source/libs/stream/src/streamCheckpoint.c | 6 +++--- source/libs/stream/test/checkpointTest.cpp | 7 +------ source/util/src/rsync.c | 24 ++++++++++++---------- 5 files changed, 19 insertions(+), 22 deletions(-) diff --git a/include/util/rsync.h b/include/util/rsync.h index 50b27b95e0..6cce645d1e 100644 --- a/include/util/rsync.h +++ b/include/util/rsync.h @@ -13,7 +13,7 @@ extern "C" { void stopRsync(); void startRsync(); -int uploadRsync(char* id, SArray* fileList); +int uploadRsync(char* id, char* path); int downloadRsync(char* id, char* path); int deleteRsync(char* id); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 806124bc58..85e4e923e5 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -144,7 +144,7 @@ int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); // char id[CHECKPOINT_PATH_LEN]; //} SChekpointDataHeader; -int uploadCheckpoint(char* id, SArray* fileList); +int uploadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path); int deleteCheckpoint(char* id); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 238c7c2329..e02a591cff 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -451,13 +451,13 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { //} -int uploadCheckpoint(char* id, SArray* fileList){ - if(id == NULL || fileList == NULL || strlen(id) == 0 || taosArrayGetSize(fileList) == 0){ +int uploadCheckpoint(char* id, char* path){ + if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0){ stError("uploadCheckpoint parameters invalid"); return -1; } if(strlen(tsSnodeIp) != 0){ - uploadRsync(id, fileList); + uploadRsync(id, path); // }else if(tsS3StreamEnabled){ } diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 56614cc537..6b53f13c71 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -40,14 +40,9 @@ TEST(testCase, checkpointUpload_Test) { startRsync(); taosSsleep(5); - SArray* fileList = taosArrayInit(0, POINTER_BYTES); char* id = "2013892036"; - char* file1 = "/Users/mingmingwanng/wal1"; - char* file2 = "/Users/mingmingwanng/java_error_in_clion.hprof"; - taosArrayPush(fileList, &file1); - taosArrayPush(fileList, &file2); - uploadCheckpoint(id, fileList); + uploadCheckpoint(id, "/Users/mingmingwanng/rsync/"); } TEST(testCase, checkpointDownload_Test) { diff --git a/source/util/src/rsync.c b/source/util/src/rsync.c index 940c631c65..b31526dc48 100644 --- a/source/util/src/rsync.c +++ b/source/util/src/rsync.c @@ -118,24 +118,26 @@ void startRsync(){ uDebug("[rsync] start server successful"); } -int uploadRsync(char* id, SArray* fileList){ - for(int i = 0; i < taosArrayGetSize(fileList); i++) { - char* fullName = (char*)taosArrayGetP(fileList, i); - char command[PATH_MAX] = {0}; +int uploadRsync(char* id, char* path){ + char command[PATH_MAX] = {0}; // char* name = strrchr(fullName, '/'); // if(name == NULL){ // uError("[rsync] file name invalid, name:%s", name); // return -1; // } // name = name + 1; - snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/", - fullName, tsSnodeIp, id); + if(path[strlen(path) - 1] != '/'){ + snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/", + path, tsSnodeIp, id); + }else{ + snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/", + path, tsSnodeIp, id); + } - int code = execCommand(command); - if(code != 0){ - uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); - return -1; - } + int code = execCommand(command); + if(code != 0){ + uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; } uDebug("[rsync] upload data:%s successful", id); return 0; From 5bd0745a762682e2ebbed243a9e837e2f836654b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 3 Nov 2023 11:46:21 +0800 Subject: [PATCH 03/20] fix:add config for checkpoint --- source/common/src/tglobal.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 0eebb9ff22..39410dc8ae 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -657,6 +657,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "snodeIp", tsSnodeIp, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; @@ -1081,6 +1084,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); + tstrncpy(tsSnodeIp, cfgGetItem(pCfg, "snodeIp")->str, TSDB_FQDN_LEN); + tstrncpy(tsCheckpointBackupDir, cfgGetItem(pCfg, "checkpointBackupDir")->str, PATH_MAX); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32; From 7a39b3296157439d66239e4d48ca1eb96635c4ea Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 3 Nov 2023 16:33:48 +0800 Subject: [PATCH 04/20] fix:add windows support --- source/common/src/tglobal.c | 4 ++++ source/util/src/rsync.c | 15 ++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 39410dc8ae..639e300179 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -135,7 +135,11 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; // checkpoint backup char tsSnodeIp[TSDB_FQDN_LEN] = {0}; +#ifdef WINDOWS +char tsCheckpointBackupDir[PATH_MAX] = "/c/TDengine/data/backup/checkpoint/"; +#else char tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/"; +#endif // tmq int32_t tmqMaxTopicNum = 20; diff --git a/source/util/src/rsync.c b/source/util/src/rsync.c index b31526dc48..e3908e53ba 100644 --- a/source/util/src/rsync.c +++ b/source/util/src/rsync.c @@ -47,8 +47,10 @@ static int generateConfigFile(char* confDir){ char confContent[PATH_MAX*4] = {0}; snprintf(confContent, PATH_MAX*4, +#ifndef WINDOWS "uid = root\n" "gid = root\n" +#endif "use chroot = false\n" "max connections = 200\n" "timeout = 100\n" @@ -84,7 +86,12 @@ static int execCommand(char* command){ } void stopRsync(){ - int code = system("pkill rsync"); + int code = +#ifdef WINDOWS + system("taskkill /f /mi rsync.exe"); +#else + system("pkill rsync"); +#endif if(code != 0){ uError("[rsync] stop rsync server failed,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); return; @@ -120,12 +127,6 @@ void startRsync(){ int uploadRsync(char* id, char* path){ char command[PATH_MAX] = {0}; -// char* name = strrchr(fullName, '/'); -// if(name == NULL){ -// uError("[rsync] file name invalid, name:%s", name); -// return -1; -// } -// name = name + 1; if(path[strlen(path) - 1] != '/'){ snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/", path, tsSnodeIp, id); From 572dd8ec3953ea1e878f689b2759c8fc9642ff94 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 3 Nov 2023 17:14:04 +0800 Subject: [PATCH 05/20] fix:add windows support --- source/common/src/tglobal.c | 2 +- source/util/src/rsync.c | 30 ++++++++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 639e300179..51c141c754 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -136,7 +136,7 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; // checkpoint backup char tsSnodeIp[TSDB_FQDN_LEN] = {0}; #ifdef WINDOWS -char tsCheckpointBackupDir[PATH_MAX] = "/c/TDengine/data/backup/checkpoint/"; +char tsCheckpointBackupDir[PATH_MAX] = "C:\TDengine\data\backup\checkpoint\"; #else char tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/"; #endif diff --git a/source/util/src/rsync.c b/source/util/src/rsync.c index e3908e53ba..6316be7f79 100644 --- a/source/util/src/rsync.c +++ b/source/util/src/rsync.c @@ -38,6 +38,21 @@ static void removeEmptyDir(){ taosCloseDir(&pDir); } +#ifdef WINDOWS +// C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/ +static void changeDirFromWindowsToLinux(char* from, char* to){ + to[0] = '/'; + to[1] = from[0]; + for(int i = 2; i < strlen(from); i++) { + if (from[i] == '\\') { + to[i] = '/'; + } else { + to[i] = from[i]; + } + } +} +#endif + static int generateConfigFile(char* confDir){ TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -45,6 +60,11 @@ static int generateConfigFile(char* confDir){ return -1; } +#ifdef WINDOWS + char path[PATH_MAX] = {0}; + changeDirFromWindowsToLinux(tsCheckpointBackupDir, path); +#endif + char confContent[PATH_MAX*4] = {0}; snprintf(confContent, PATH_MAX*4, #ifndef WINDOWS @@ -60,7 +80,13 @@ static int generateConfigFile(char* confDir){ "read only = false\n" "list = false\n" "[checkpoint]\n" - "path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir, tsCheckpointBackupDir); + "path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir, +#ifdef WINDOWS + path +#else + tsCheckpointBackupDir +#endif + ); uDebug("[rsync] conf:%s", confContent); if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){ uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); @@ -88,7 +114,7 @@ static int execCommand(char* command){ void stopRsync(){ int code = #ifdef WINDOWS - system("taskkill /f /mi rsync.exe"); + system("taskkill /f /im rsync.exe"); #else system("pkill rsync"); #endif From 16f8c1f6280aa49e804a899ede70dae092cdd039 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 3 Nov 2023 17:17:43 +0800 Subject: [PATCH 06/20] fix:add windows support --- source/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 51c141c754..3593665de9 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -136,7 +136,7 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; // checkpoint backup char tsSnodeIp[TSDB_FQDN_LEN] = {0}; #ifdef WINDOWS -char tsCheckpointBackupDir[PATH_MAX] = "C:\TDengine\data\backup\checkpoint\"; +char tsCheckpointBackupDir[PATH_MAX] = "C:\\TDengine\\data\\backup\\checkpoint\\"; #else char tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/"; #endif From 5a6c3f2b048de3eaeb415be26b8f7e55aecf3fae Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 3 Nov 2023 17:45:37 +0800 Subject: [PATCH 07/20] fix:add windows support --- source/libs/stream/src/streamCheckpoint.c | 4 +-- source/util/src/rsync.c | 30 ++++++++++++++++++++--- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b27529d5b8..b30a3bdc50 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -467,7 +467,7 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { //} int uploadCheckpoint(char* id, char* path){ - if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0){ + if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ stError("uploadCheckpoint parameters invalid"); return -1; } @@ -480,7 +480,7 @@ int uploadCheckpoint(char* id, char* path){ } int downloadCheckpoint(char* id, char* path){ - if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0){ + if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ stError("downloadCheckpoint parameters invalid"); return -1; } diff --git a/source/util/src/rsync.c b/source/util/src/rsync.c index 6316be7f79..8c6bce66a0 100644 --- a/source/util/src/rsync.c +++ b/source/util/src/rsync.c @@ -152,13 +152,27 @@ void startRsync(){ } int uploadRsync(char* id, char* path){ +#ifdef WINDOWS + char pathTransform[PATH_MAX] = {0}; + changeDirFromWindowsToLinux(path, pathTransform); +#endif char command[PATH_MAX] = {0}; if(path[strlen(path) - 1] != '/'){ snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/", - path, tsSnodeIp, id); +#ifdef WINDOWS + pathTransform +#else + path +#endif + , tsSnodeIp, id); }else{ snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/", - path, tsSnodeIp, id); +#ifdef WINDOWS + pathTransform +#else + path +#endif + , tsSnodeIp, id); } int code = execCommand(command); @@ -171,9 +185,19 @@ int uploadRsync(char* id, char* path){ } int downloadRsync(char* id, char* path){ +#ifdef WINDOWS + char pathTransform[PATH_MAX] = {0}; + changeDirFromWindowsToLinux(path, pathTransform); +#endif char command[PATH_MAX] = {0}; snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s", - tsSnodeIp, id, path); + tsSnodeIp, id, +#ifdef WINDOWS + pathTransform +#else + path +#endif + ); int code = execCommand(command); if(code != 0){ From b1515dd069cebda55cb3b51b7e88c1d7722ef226 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 3 Nov 2023 17:56:34 +0800 Subject: [PATCH 08/20] fix:add windows support --- source/util/src/rsync.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/util/src/rsync.c b/source/util/src/rsync.c index 8c6bce66a0..cbb447a33e 100644 --- a/source/util/src/rsync.c +++ b/source/util/src/rsync.c @@ -157,7 +157,11 @@ int uploadRsync(char* id, char* path){ changeDirFromWindowsToLinux(path, pathTransform); #endif char command[PATH_MAX] = {0}; +#ifdef WINDOWS + if(pathTransform[strlen(pathTransform) - 1] != '/'){ +#else if(path[strlen(path) - 1] != '/'){ +#endif snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/", #ifdef WINDOWS pathTransform From dffaac55ea2a07c53bf0a7b05ed1b2a71da903f1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Mon, 6 Nov 2023 14:57:32 +0800 Subject: [PATCH 09/20] fix:move cos from util to common --- include/{util => common}/cos.h | 0 include/{util => common}/rsync.h | 0 source/common/CMakeLists.txt | 69 +++++++++++++++++++++++++++++ source/{util => common}/src/cos.c | 21 ++++----- source/{util => common}/src/rsync.c | 0 source/dnode/vnode/CMakeLists.txt | 69 ----------------------------- 6 files changed, 80 insertions(+), 79 deletions(-) rename include/{util => common}/cos.h (100%) rename include/{util => common}/rsync.h (100%) rename source/{util => common}/src/cos.c (98%) rename source/{util => common}/src/rsync.c (100%) diff --git a/include/util/cos.h b/include/common/cos.h similarity index 100% rename from include/util/cos.h rename to include/common/cos.h diff --git a/include/util/rsync.h b/include/common/rsync.h similarity index 100% rename from include/util/rsync.h rename to include/common/rsync.h diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index c35845f9df..c7831b2b6d 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -46,6 +46,75 @@ target_link_libraries( INTERFACE api ) +if(${BUILD_S3}) + + if(${BUILD_WITH_S3}) + target_include_directories( + common + + PUBLIC "$ENV{HOME}/.cos-local.2/include" + ) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) + find_library(S3_LIBRARY s3) + find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(XML2_LIBRARY xml2) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + target_link_libraries( + common + + # s3 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC ${XML2_LIBRARY} + ) + + add_definitions(-DUSE_S3) + endif() + + if(${BUILD_WITH_COS}) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) + find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) + find_library(MINIXML_LIBRARY mxml) + find_library(CURL_LIBRARY curl) + target_link_libraries( + common + + # s3 + PUBLIC cos_c_sdk_static + PUBLIC ${APR_UTIL_LIBRARY} + PUBLIC ${APR_LIBRARY} + PUBLIC ${MINIXML_LIBRARY} + PUBLIC ${CURL_LIBRARY} + ) + + # s3 + FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/) + IF (APR_CONFIG_BIN) + EXECUTE_PROCESS( + COMMAND ${APR_CONFIG_BIN} --includedir + OUTPUT_VARIABLE APR_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + ENDIF() + include_directories (${APR_INCLUDE_DIR}) + target_include_directories( + common + PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk" + PUBLIC "$ENV{HOME}/.cos-local.1/include" + ) + + add_definitions(-DUSE_COS) + endif(${BUILD_WITH_COS}) + +endif() + if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) endif(${BUILD_TEST}) diff --git a/source/util/src/cos.c b/source/common/src/cos.c similarity index 98% rename from source/util/src/cos.c rename to source/common/src/cos.c index 23b2d53990..67ddb4a748 100644 --- a/source/util/src/cos.c +++ b/source/common/src/cos.c @@ -13,6 +13,7 @@ extern int8_t tsS3Https; #if defined(USE_S3) #include "libs3.h" +#include "tarray.h" static int verifyPeerG = 0; static const char *awsRegionG = NULL; @@ -34,7 +35,7 @@ static int32_t s3Begin() { } if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { - vError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); + uError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); return -1; } @@ -65,9 +66,9 @@ static int should_retry() { static void s3PrintError(const char *func, S3Status status, char error_details[]) { if (status < S3StatusErrorAccessDenied) { - vError("%s: %s", __func__, S3_get_status_name(status)); + uError("%s: %s", __func__, S3_get_status_name(status)); } else { - vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); } } @@ -445,13 +446,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { data.noStatus = noStatus; if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { - vError("ERROR: %s Failed to stat file %s: ", __func__, file); + uError("ERROR: %s Failed to stat file %s: ", __func__, file); code = TAOS_SYSTEM_ERROR(errno); return code; } if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { - vError("ERROR: %s Failed to open file %s: ", __func__, file); + uError("ERROR: %s Failed to open file %s: ", __func__, file); code = TAOS_SYSTEM_ERROR(errno); return code; } @@ -486,7 +487,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { s3PrintError(__func__, data.status, data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { - vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, + uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data.contentLength); code = TAOS_SYSTEM_ERROR(EIO); } @@ -667,9 +668,9 @@ void s3DeleteObjectsByPrefix(const char *prefix) { const char *marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; list_bucket_callback_data data; - data.objectArray = taosArrayInit(32, POINTER_BYTES); + data.objectArray = taosArrayInit(32, sizeof(void*)); if (!data.objectArray) { - vError("%s: %s", __func__, "out of memoty"); + uError("%s: %s", __func__, "out of memoty"); return; } if (marker) { @@ -758,7 +759,7 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, } while (S3_status_is_retryable(cbd.status) && should_retry()); if (cbd.status != S3StatusOK) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); return TAOS_SYSTEM_ERROR(EIO); } @@ -782,7 +783,7 @@ long s3Size(const char *object_name) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); } size = cbd.content_length; diff --git a/source/util/src/rsync.c b/source/common/src/rsync.c similarity index 100% rename from source/util/src/rsync.c rename to source/common/src/rsync.c diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 27cb0f93f7..dc43da7fe7 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -160,75 +160,6 @@ target_link_libraries( PUBLIC index ) -if(${BUILD_S3}) - -if(${BUILD_WITH_S3}) - target_include_directories( - vnode - - PUBLIC "$ENV{HOME}/.cos-local.2/include" - ) - - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) - find_library(S3_LIBRARY s3) - find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) - find_library(XML2_LIBRARY xml2) - find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) - find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) - target_link_libraries( - vnode - - # s3 - PUBLIC ${S3_LIBRARY} - PUBLIC ${CURL_LIBRARY} - PUBLIC ${SSL_LIBRARY} - PUBLIC ${CRYPTO_LIBRARY} - PUBLIC ${XML2_LIBRARY} - ) - - add_definitions(-DUSE_S3) -endif() - -if(${BUILD_WITH_COS}) - -set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") -find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) -find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) -find_library(MINIXML_LIBRARY mxml) -find_library(CURL_LIBRARY curl) -target_link_libraries( - vnode - - # s3 - PUBLIC cos_c_sdk_static - PUBLIC ${APR_UTIL_LIBRARY} - PUBLIC ${APR_LIBRARY} - PUBLIC ${MINIXML_LIBRARY} - PUBLIC ${CURL_LIBRARY} -) - -# s3 -FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/) -IF (APR_CONFIG_BIN) - EXECUTE_PROCESS( - COMMAND ${APR_CONFIG_BIN} --includedir - OUTPUT_VARIABLE APR_INCLUDE_DIR - OUTPUT_STRIP_TRAILING_WHITESPACE - ) -ENDIF() -include_directories (${APR_INCLUDE_DIR}) -target_include_directories( - vnode - PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk" - PUBLIC "$ENV{HOME}/.cos-local.1/include" - ) - - add_definitions(-DUSE_COS) -endif(${BUILD_WITH_COS}) - -endif() - IF (TD_GRANT) TARGET_LINK_LIBRARIES(vnode PUBLIC grant) ENDIF () From e0b55cc61fc8a4f4208edccbb127ff45b1c210b4 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 7 Nov 2023 11:06:50 +0800 Subject: [PATCH 10/20] fix:test s3 interface in checkpoint --- include/common/tglobal.h | 3 +- source/common/src/rsync.c | 10 ++-- source/common/src/tglobal.c | 12 +++- source/libs/stream/inc/streamInt.h | 8 ++- source/libs/stream/src/streamCheckpoint.c | 70 ++++++++++++++++++---- source/libs/stream/test/checkpointTest.cpp | 24 ++++++-- 6 files changed, 100 insertions(+), 27 deletions(-) diff --git a/include/common/tglobal.h b/include/common/tglobal.h index b9bc046504..cc45e5ec68 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -83,7 +83,8 @@ extern int32_t tsHeartbeatTimeout; extern int64_t tsVndCommitMaxIntervalMs; // snode -extern char tsSnodeIp[]; +extern char tsSnodeAddress[]; //127.0.0.1:873 +extern int32_t tsRsyncPort; extern char tsCheckpointBackupDir[]; // mnode diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index cbb447a33e..ffab85761e 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -141,7 +141,7 @@ void startRsync(){ } char cmd[PATH_MAX] = {0}; - snprintf(cmd, PATH_MAX, "rsync --daemon --config=%s", confDir); + snprintf(cmd, PATH_MAX, "rsync --daemon --port=%d --config=%s", tsRsyncPort, confDir); // start rsync service to backup checkpoint code = system(cmd); if(code != 0){ @@ -168,7 +168,7 @@ int uploadRsync(char* id, char* path){ #else path #endif - , tsSnodeIp, id); + , tsSnodeAddress, id); }else{ snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/", #ifdef WINDOWS @@ -176,7 +176,7 @@ int uploadRsync(char* id, char* path){ #else path #endif - , tsSnodeIp, id); + , tsSnodeAddress, id); } int code = execCommand(command); @@ -195,7 +195,7 @@ int downloadRsync(char* id, char* path){ #endif char command[PATH_MAX] = {0}; snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s", - tsSnodeIp, id, + tsSnodeAddress, id, #ifdef WINDOWS pathTransform #else @@ -221,7 +221,7 @@ int deleteRsync(char* id){ } char command[PATH_MAX] = {0}; snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/", - tmp, tsSnodeIp, id); + tmp, tsSnodeAddress, id); code = execCommand(command); taosRemoveDir(tmp); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3593665de9..14036b3e74 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -134,7 +134,8 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; // int32_t tsSmlBatchSize = 10000; // checkpoint backup -char tsSnodeIp[TSDB_FQDN_LEN] = {0}; +char tsSnodeAddress[TSDB_FQDN_LEN] = {0}; +int32_t tsRsyncPort = 873; #ifdef WINDOWS char tsCheckpointBackupDir[PATH_MAX] = "C:\\TDengine\\data\\backup\\checkpoint\\"; #else @@ -661,7 +662,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "snodeIp", tsSnodeIp, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "snodeRsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "snodeRsyncAddress", tsSnodeAddress, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; @@ -1087,8 +1089,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval; tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; + tsRsyncPort = cfgGetItem(pCfg, "rsyncPort")->i32; tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); - tstrncpy(tsSnodeIp, cfgGetItem(pCfg, "snodeIp")->str, TSDB_FQDN_LEN); + tstrncpy(tsSnodeAddress, cfgGetItem(pCfg, "snodeAddress")->str, TSDB_FQDN_LEN); tstrncpy(tsCheckpointBackupDir, cfgGetItem(pCfg, "checkpointBackupDir")->str, PATH_MAX); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; @@ -1676,6 +1679,9 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile return -1; } + if (taosSetS3Cfg(tsCfg) != 0) return -1; + return 0; + if (tsc) { if (taosSetClientCfg(tsCfg)) return -1; } else { diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 86dd8ed542..fb47bf135b 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -148,10 +148,16 @@ int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); // char name[CHECKPOINT_PATH_LEN]; // char id[CHECKPOINT_PATH_LEN]; //} SChekpointDataHeader; - +typedef enum UPLOAD_TYPE{ + UPLOAD_DISABLE = -1, + UPLOAD_S3 = 0, + UPLOAD_RSYNC = 1, +} UPLOAD_TYPE; +UPLOAD_TYPE getUploadType(); int uploadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path); int deleteCheckpoint(char* id); +int deleteCheckpointFile(char* id, char* name); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b30a3bdc50..57f56f93f8 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -15,6 +15,7 @@ #include "streamInt.h" #include "rsync.h" +#include "cos.h" int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; @@ -466,15 +467,55 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { //} +static int uploadCheckpointToS3(char* id, char* path){ + TdDirPtr pDir = taosOpenDir(path); + if (pDir == NULL) return -1; + + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || + taosDirEntryIsDir(de)) continue; + + char filename[PATH_MAX] = {0}; + if(path[strlen(path - 1)] == '/'){ + snprintf(filename, sizeof(filename), "%s%s", path, name); + }else{ + snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name); + } + + char object[PATH_MAX] = {0}; + snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name); + + if(s3PutObjectFromFile2(filename, object) != 0){ + taosCloseDir(&pDir); + return -1; + } + } + taosCloseDir(&pDir); + + return 0; +} + +UPLOAD_TYPE getUploadType(){ + if(strlen(tsSnodeAddress) != 0){ + return UPLOAD_RSYNC; + }else if(tsS3StreamEnabled){ + return UPLOAD_S3; + }else{ + return UPLOAD_DISABLE; + } +} + int uploadCheckpoint(char* id, char* path){ if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ stError("uploadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ - uploadRsync(id, path); -// }else if(tsS3StreamEnabled){ - + if(strlen(tsSnodeAddress) != 0){ + return uploadRsync(id, path); + }else if(tsS3StreamEnabled){ + return uploadCheckpointToS3(id, path); } return 0; } @@ -484,9 +525,9 @@ int downloadCheckpoint(char* id, char* path){ stError("downloadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ - downloadRsync(id, path); -// }else if(tsS3StreamEnabled){ + if(strlen(tsSnodeAddress) != 0){ + return downloadRsync(id, path); + }else if(tsS3StreamEnabled){ } return 0; @@ -497,10 +538,17 @@ int deleteCheckpoint(char* id){ stError("deleteCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ - deleteRsync(id); -// }else if(tsS3StreamEnabled){ - + if(strlen(tsSnodeAddress) != 0){ + return deleteRsync(id); + }else if(tsS3StreamEnabled){ + s3DeleteObjectsByPrefix(id); } return 0; } + +int deleteCheckpointFile(char* id, char* name){ + char object[128] = {0}; + snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name); + s3DeleteObjects((const char**)&object, 1); + return 0; +} diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 6b53f13c71..3da30121a7 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -27,30 +27,42 @@ #include "rsync.h" #include "streamInt.h" +#include "cos.h" int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); - strcpy(tsSnodeIp, "127.0.0.1"); + if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { + printf("error"); + } + if (s3Init() < 0) { + return -1; + } +// strcpy(tsSnodeIp, "127.0.0.1"); return RUN_ALL_TESTS(); } TEST(testCase, checkpointUpload_Test) { - stopRsync(); - startRsync(); +// stopRsync(); +// startRsync(); taosSsleep(5); char* id = "2013892036"; - uploadCheckpoint(id, "/Users/mingmingwanng/rsync/"); + uploadCheckpoint(id, "/root/offset/"); } TEST(testCase, checkpointDownload_Test) { char* id = "2013892036"; - downloadRsync(id, "/Users/mingmingwanng/rsync/tmp"); + downloadCheckpoint(id, "/Users/mingmingwanng/rsync/tmp"); } TEST(testCase, checkpointDelete_Test) { char* id = "2013892036"; - deleteRsync(id); + deleteCheckpoint(id); +} + +TEST(testCase, checkpointDeleteFile_Test) { + char* id = "2013892036"; + deleteCheckpointFile(id, "offset-ver0"); } From f0b2ed567c50afb189c245e0f6a565f747675b68 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 7 Nov 2023 18:49:12 +0800 Subject: [PATCH 11/20] fix:upload checkpoint to s3 --- include/common/cos.h | 1 + include/common/tglobal.h | 3 +- source/common/src/cos.c | 76 ++++++++++++++++++++-- source/common/src/rsync.c | 10 +-- source/common/src/tglobal.c | 12 +++- source/libs/stream/inc/streamInt.h | 8 ++- source/libs/stream/src/streamCheckpoint.c | 73 +++++++++++++++++---- source/libs/stream/test/checkpointTest.cpp | 24 +++++-- 8 files changed, 172 insertions(+), 35 deletions(-) diff --git a/include/common/cos.h b/include/common/cos.h index 4d676d9ca7..0610c63f9e 100644 --- a/include/common/cos.h +++ b/include/common/cos.h @@ -39,6 +39,7 @@ void s3DeleteObjectsByPrefix(const char *prefix); void s3DeleteObjects(const char *object_name[], int nobject); bool s3Exists(const char *object_name); bool s3Get(const char *object_name, const char *path); +int32_t s3GetObjectsByPrefix(const char *prefix, const char* path); int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock); void s3EvictCache(const char *path, long object_size); long s3Size(const char *object_name); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index b9bc046504..cc45e5ec68 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -83,7 +83,8 @@ extern int32_t tsHeartbeatTimeout; extern int64_t tsVndCommitMaxIntervalMs; // snode -extern char tsSnodeIp[]; +extern char tsSnodeAddress[]; //127.0.0.1:873 +extern int32_t tsRsyncPort; extern char tsCheckpointBackupDir[]; // mnode diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 67ddb4a748..c209f1a063 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -72,6 +72,12 @@ static void s3PrintError(const char *func, S3Status status, char error_details[] } } +typedef struct { + char err_msg[128]; + S3Status status; + TdFilePtr file; +} TS3GetData; + typedef struct { char err_msg[128]; S3Status status; @@ -659,7 +665,7 @@ static void s3FreeObjectKey(void *pItem) { taosMemoryFree(key); } -void s3DeleteObjectsByPrefix(const char *prefix) { +static SArray* getListByPrefix(const char *prefix){ S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, @@ -671,7 +677,7 @@ void s3DeleteObjectsByPrefix(const char *prefix) { data.objectArray = taosArrayInit(32, sizeof(void*)); if (!data.objectArray) { uError("%s: %s", __func__, "out of memoty"); - return; + return NULL; } if (marker) { snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); @@ -694,18 +700,15 @@ void s3DeleteObjectsByPrefix(const char *prefix) { if (data.status == S3StatusOK) { if (data.keyCount > 0) { - // printListBucketHeader(allDetails); - s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); + return data.objectArray; } } else { s3PrintError(__func__, data.status, data.err_msg); } - - taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); + return NULL; } void s3DeleteObjects(const char *object_name[], int nobject) { - int status = 0; S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; @@ -722,6 +725,13 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } } +void s3DeleteObjectsByPrefix(const char *prefix) { + SArray* objectArray = getListByPrefix(prefix); + if(objectArray == NULL)return; + s3DeleteObjects(TARRAY_DATA(objectArray), TARRAY_SIZE(objectArray)); + taosArrayDestroyEx(objectArray, s3FreeObjectKey); +} + static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) { TS3SizeCBD *cbd = callbackData; if (cbd->content_length != bufferSize) { @@ -768,6 +778,58 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, return 0; } +static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) { + TS3GetData *cbd = (TS3GetData *) callbackData; + size_t wrote = taosWriteFile(cbd->file, buffer, bufferSize); + return ((wrote < (size_t) bufferSize) ? + S3StatusAbortedByCallback : S3StatusOK); +} + +int32_t s3GetObjectToFile(const char *object_name, char* fileName) { + int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; + const char *ifMatch = 0, *ifNotMatch = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; + S3GetObjectHandler getObjectHandler = {{NULL, &responseCompleteCallback}, + &getObjectCallback}; + + TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + uError("[s3] open file error, errno:%d, fileName:%s", errno, fileName); + return -1; + } + TS3GetData cbd = {0}; + cbd.file = pFile; + do { + S3_get_object(&bucketContext, object_name, &getConditions, 0, 0, 0, 0, &getObjectHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if (cbd.status != S3StatusOK) { + uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + taosCloseFile(&pFile); + return TAOS_SYSTEM_ERROR(EIO); + } + + taosCloseFile(&pFile); + return 0; +} + +int32_t s3GetObjectsByPrefix(const char *prefix, const char* path){ + SArray* objectArray = getListByPrefix(prefix); + if(objectArray == NULL) return -1; + + for(size_t i = 0; i < taosArrayGetSize(objectArray); i++){ + char* object = taosArrayGetP(objectArray, i); + char fileName[PATH_MAX] = {0}; + snprintf(fileName, PATH_MAX, "%s/%s", path, object); + s3GetObjectToFile(object, fileName); + } + taosArrayDestroyEx(objectArray, s3FreeObjectKey); + return 0; +} + long s3Size(const char *object_name) { long size = 0; int status = 0; diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c index cbb447a33e..ffab85761e 100644 --- a/source/common/src/rsync.c +++ b/source/common/src/rsync.c @@ -141,7 +141,7 @@ void startRsync(){ } char cmd[PATH_MAX] = {0}; - snprintf(cmd, PATH_MAX, "rsync --daemon --config=%s", confDir); + snprintf(cmd, PATH_MAX, "rsync --daemon --port=%d --config=%s", tsRsyncPort, confDir); // start rsync service to backup checkpoint code = system(cmd); if(code != 0){ @@ -168,7 +168,7 @@ int uploadRsync(char* id, char* path){ #else path #endif - , tsSnodeIp, id); + , tsSnodeAddress, id); }else{ snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/", #ifdef WINDOWS @@ -176,7 +176,7 @@ int uploadRsync(char* id, char* path){ #else path #endif - , tsSnodeIp, id); + , tsSnodeAddress, id); } int code = execCommand(command); @@ -195,7 +195,7 @@ int downloadRsync(char* id, char* path){ #endif char command[PATH_MAX] = {0}; snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s", - tsSnodeIp, id, + tsSnodeAddress, id, #ifdef WINDOWS pathTransform #else @@ -221,7 +221,7 @@ int deleteRsync(char* id){ } char command[PATH_MAX] = {0}; snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/", - tmp, tsSnodeIp, id); + tmp, tsSnodeAddress, id); code = execCommand(command); taosRemoveDir(tmp); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 3593665de9..14036b3e74 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -134,7 +134,8 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; // int32_t tsSmlBatchSize = 10000; // checkpoint backup -char tsSnodeIp[TSDB_FQDN_LEN] = {0}; +char tsSnodeAddress[TSDB_FQDN_LEN] = {0}; +int32_t tsRsyncPort = 873; #ifdef WINDOWS char tsCheckpointBackupDir[PATH_MAX] = "C:\\TDengine\\data\\backup\\checkpoint\\"; #else @@ -661,7 +662,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "snodeIp", tsSnodeIp, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "snodeRsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "snodeRsyncAddress", tsSnodeAddress, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; @@ -1087,8 +1089,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval; tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; + tsRsyncPort = cfgGetItem(pCfg, "rsyncPort")->i32; tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); - tstrncpy(tsSnodeIp, cfgGetItem(pCfg, "snodeIp")->str, TSDB_FQDN_LEN); + tstrncpy(tsSnodeAddress, cfgGetItem(pCfg, "snodeAddress")->str, TSDB_FQDN_LEN); tstrncpy(tsCheckpointBackupDir, cfgGetItem(pCfg, "checkpointBackupDir")->str, PATH_MAX); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; @@ -1676,6 +1679,9 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile return -1; } + if (taosSetS3Cfg(tsCfg) != 0) return -1; + return 0; + if (tsc) { if (taosSetClientCfg(tsCfg)) return -1; } else { diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index 86dd8ed542..fb47bf135b 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -148,10 +148,16 @@ int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); // char name[CHECKPOINT_PATH_LEN]; // char id[CHECKPOINT_PATH_LEN]; //} SChekpointDataHeader; - +typedef enum UPLOAD_TYPE{ + UPLOAD_DISABLE = -1, + UPLOAD_S3 = 0, + UPLOAD_RSYNC = 1, +} UPLOAD_TYPE; +UPLOAD_TYPE getUploadType(); int uploadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path); int deleteCheckpoint(char* id); +int deleteCheckpointFile(char* id, char* name); #ifdef __cplusplus } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b30a3bdc50..c3152aebb7 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -15,6 +15,7 @@ #include "streamInt.h" #include "rsync.h" +#include "cos.h" int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; @@ -466,15 +467,55 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { //} +static int uploadCheckpointToS3(char* id, char* path){ + TdDirPtr pDir = taosOpenDir(path); + if (pDir == NULL) return -1; + + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || + taosDirEntryIsDir(de)) continue; + + char filename[PATH_MAX] = {0}; + if(path[strlen(path - 1)] == '/'){ + snprintf(filename, sizeof(filename), "%s%s", path, name); + }else{ + snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name); + } + + char object[PATH_MAX] = {0}; + snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name); + + if(s3PutObjectFromFile2(filename, object) != 0){ + taosCloseDir(&pDir); + return -1; + } + } + taosCloseDir(&pDir); + + return 0; +} + +UPLOAD_TYPE getUploadType(){ + if(strlen(tsSnodeAddress) != 0){ + return UPLOAD_RSYNC; + }else if(tsS3StreamEnabled){ + return UPLOAD_S3; + }else{ + return UPLOAD_DISABLE; + } +} + int uploadCheckpoint(char* id, char* path){ if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ stError("uploadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ - uploadRsync(id, path); -// }else if(tsS3StreamEnabled){ - + if(strlen(tsSnodeAddress) != 0){ + return uploadRsync(id, path); + }else if(tsS3StreamEnabled){ + return uploadCheckpointToS3(id, path); } return 0; } @@ -484,10 +525,10 @@ int downloadCheckpoint(char* id, char* path){ stError("downloadCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ - downloadRsync(id, path); -// }else if(tsS3StreamEnabled){ - + if(strlen(tsSnodeAddress) != 0){ + return downloadRsync(id, path); + }else if(tsS3StreamEnabled){ + return s3GetObjectsByPrefix(id, path); } return 0; } @@ -497,10 +538,18 @@ int deleteCheckpoint(char* id){ stError("deleteCheckpoint parameters invalid"); return -1; } - if(strlen(tsSnodeIp) != 0){ - deleteRsync(id); -// }else if(tsS3StreamEnabled){ - + if(strlen(tsSnodeAddress) != 0){ + return deleteRsync(id); + }else if(tsS3StreamEnabled){ + s3DeleteObjectsByPrefix(id); } return 0; } + +int deleteCheckpointFile(char* id, char* name){ + char object[128] = {0}; + snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name); + char *tmp = object; + s3DeleteObjects((const char**)&tmp, 1); + return 0; +} diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 6b53f13c71..135431cfb7 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -27,30 +27,42 @@ #include "rsync.h" #include "streamInt.h" +#include "cos.h" int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); - strcpy(tsSnodeIp, "127.0.0.1"); + if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { + printf("error"); + } + if (s3Init() < 0) { + return -1; + } +// strcpy(tsSnodeIp, "127.0.0.1"); return RUN_ALL_TESTS(); } TEST(testCase, checkpointUpload_Test) { - stopRsync(); - startRsync(); +// stopRsync(); +// startRsync(); taosSsleep(5); char* id = "2013892036"; - uploadCheckpoint(id, "/Users/mingmingwanng/rsync/"); + uploadCheckpoint(id, "/root/offset/"); } TEST(testCase, checkpointDownload_Test) { char* id = "2013892036"; - downloadRsync(id, "/Users/mingmingwanng/rsync/tmp"); + downloadCheckpoint(id, "/root/offset/download/"); } TEST(testCase, checkpointDelete_Test) { char* id = "2013892036"; - deleteRsync(id); + deleteCheckpoint(id); +} + +TEST(testCase, checkpointDeleteFile_Test) { + char* id = "2013892036"; + deleteCheckpointFile(id, "offset-ver0"); } From 93cc50db16bf4341a8aa74800c13c88b09c48121 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 7 Nov 2023 20:01:44 +0800 Subject: [PATCH 12/20] fix:compile error --- source/common/src/tglobal.c | 882 +++++++++++++++++------------------- 1 file changed, 405 insertions(+), 477 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 14036b3e74..cab4325351 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -44,7 +44,6 @@ int32_t tsNumOfSupportVnodes = 256; // common int32_t tsMaxShellConns = 50000; int32_t tsShellActivityTimer = 3; // second -bool tsPrintAuth = false; // queue & threads int32_t tsNumOfRpcThreads = 1; @@ -73,9 +72,6 @@ int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; -// vnode -int64_t tsVndCommitMaxIntervalMs = 600 * 1000; - // mnode int64_t tsMndSdbWriteDelta = 200; int64_t tsMndLogRetention = 2000; @@ -99,8 +95,6 @@ bool tsMonitorComp = false; // audit bool tsEnableAudit = true; -char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; -uint16_t tsAuditPort = 6043; bool tsEnableAuditCreateTable = true; // telem @@ -176,22 +170,12 @@ int32_t tsTimeSeriesThreshold = 50; */ int32_t tsCompressMsgSize = -1; -/* denote if server needs to compress the retrieved column data before adding to the rpc response message body. - * 0: all data are compressed - * -1: all data are not compressed - * other values: if any retrieved column size is greater than the tsCompressColData, all data will be compressed. - */ -int32_t tsCompressColData = -1; - // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; // 1 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 1; -// the maxinum number of distict query result -int32_t tsMaxNumOfDistinctResults = 1000 * 10000; - // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; @@ -247,7 +231,7 @@ char tsCompressor[32] = "ZSTD_COMPRESSOR"; // ZSTD_COMPRESSOR or GZIP_COMPR #ifdef WINDOWS bool tsStartUdfd = false; #else -bool tsStartUdfd = true; +bool tsStartUdfd = true; #endif // wal @@ -359,9 +343,7 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { return 0; } -struct SConfig *taosGetCfg() { - return tsCfg; -} +struct SConfig *taosGetCfg() { return tsCfg; } static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *inputCfgDir, const char *envFile, char *apolloUrl) { @@ -412,39 +394,40 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } int32_t taosAddClientLogCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) + return -1; + if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; return 0; } static int32_t taosAddServerLogCfg(SConfig *pCfg) { - if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, 0) != CFG_SCOPE_SERVER) return -1; - if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; return 0; } @@ -455,62 +438,76 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { strcpy(defaultFqdn, "localhost"); } - if (cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) + if (cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1; - // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, CFG_SCOPE_CLIENT) != 0) return -1; - // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, - CFG_SCOPE_CLIENT) != 0) + if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != + 0) + return -1; + if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; + // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) + // return -1; + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT, + CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) + return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH) != 0) + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, + CFG_DYN_NONE) != 0) return -1; tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); if (tsNumOfTaskQueueThreads >= 10) { tsNumOfTaskQueueThreads = 10; } - if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) + return -1; return 0; } @@ -518,98 +515,106 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "assert", tsAssert, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "assert", tsAssert, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; #if !defined(_ALPINE) - if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; #endif - if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; return 0; } static int32_t taosAddServerCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; tsNumOfSupportVnodes = tsNumOfCores * 2; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, - CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER) != 0) return -1; + + if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) + return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH) != 0) return -1; - - tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) + return -1; tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; tsNumOfMnodeReadThreads = tsNumOfCores / 8; tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4); - if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; tsNumOfVnodeQueryThreads = tsNumOfCores * 2; tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; - if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER) != 0) + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 4; tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; tsNumOfVnodeRsmaThreads = tsNumOfCores; tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; tsNumOfQnodeQueryThreads = tsNumOfCores * 2; tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); @@ -617,112 +622,144 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSnodeStreamThreads = tsNumOfCores / 4; tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) return -1; tsNumOfSnodeWriteThreads = tsNumOfCores / 4; tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, - CFG_SCOPE_BOTH) != 0) + CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddInt32(pCfg, "snodeRsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "snodeRsyncAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + + if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "auditFqdn", tsAuditFqdn, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "auditPort", tsAuditPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH) != 0) return -1; - - if (cfgAddInt32(pCfg, "snodeRsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "snodeRsyncAddress", tsSnodeAddress, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, - CFG_SCOPE_SERVER) != 0) + CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, 0) != 0) return -1; + if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; - if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER) != 0) + if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) return -1; // min free disk space used to check if the disk is full [50MB, 1GB] if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, - CFG_SCOPE_SERVER) != 0) + CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; GRANT_CFG_ADD; return 0; @@ -772,13 +809,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } - pItem = cfgGetItem(tsCfg, "keepAliveIdle"); - if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 720000); - pItem->i32 = tsKeepAliveIdle; - pItem->stype = stype; - } - pItem = cfgGetItem(tsCfg, "numOfCommitThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { tsNumOfCommitThreads = numOfCores / 2; @@ -947,6 +977,7 @@ static int32_t taosSetSlowLogScope(char *pScope) { return -1; } +// for common configs static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; @@ -988,7 +1019,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; - tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; @@ -1038,6 +1068,7 @@ static void taosSetSystemCfg(SConfig *pCfg) { tsVersion = 30000000; } +// for server configs static int32_t taosSetServerCfg(SConfig *pCfg) { tsDataSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalDataDirGB")->fval) * 1024 * 1024 * 1024); tsNumOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; @@ -1045,17 +1076,13 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; - tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; - tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; - tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; - tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; @@ -1080,8 +1107,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; - tstrncpy(tsAuditFqdn, cfgGetItem(pCfg, "auditFqdn")->str, TSDB_FQDN_LEN); - tsAuditPort = (uint16_t)cfgGetItem(pCfg, "auditPort")->i32; tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; @@ -1113,8 +1138,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; - tsVndCommitMaxIntervalMs = cfgGetItem(pCfg, "vndCommitMaxInterval")->i64; - tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; @@ -1164,24 +1187,11 @@ static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } int32_t taosSetReleaseCfg(SConfig *pCfg); #endif -void taosLocalCfgForbiddenToChange(char *name, bool *forbidden) { - int32_t len = strlen(name); - char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; - strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - - if (strcasecmp("charset", name) == 0) { - *forbidden = true; - return; - } - GRANT_CFG_CHECK; - - *forbidden = false; -} - int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { int32_t len = strlen(name); char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); + bool matchItem = true; switch (lowcaseName[0]) { case 'a': { @@ -1189,25 +1199,22 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; } else if (strcasecmp("assert", name) == 0) { tsAssert = cfgGetItem(pCfg, "assert")->bval; + } else { + matchItem = false; } break; } case 'c': { - if (strcasecmp("charset", name) == 0) { - const char *locale = cfgGetItem(pCfg, "locale")->str; - const char *charset = cfgGetItem(pCfg, "charset")->str; - taosSetSystemLocale(locale, charset); - osSetSystemLocale(locale, charset); - } else if (strcasecmp("compressMsgSize", name) == 0) { + if (strcasecmp("compressMsgSize", name) == 0) { tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; - } else if (strcasecmp("compressColData", name) == 0) { - tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; } else if (strcasecmp("countAlwaysReturnValue", name) == 0) { tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; } else if (strcasecmp("cDebugFlag", name) == 0) { cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; } else if (strcasecmp("crashReporting", name) == 0) { tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; + } else { + matchItem = false; } break; } @@ -1217,6 +1224,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("debugFlag", name) == 0) { int32_t flag = cfgGetItem(pCfg, "debugFlag")->i32; taosSetAllDebugFlag(flag, true); + } else { + matchItem = false; } break; } @@ -1226,8 +1235,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { taosSetCoreDump(enableCore); } else if (strcasecmp("enableQueryHb", name) == 0) { tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; - } else if (strcasecmp("ttlChangeOnWrite", name) == 0) { - tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval; + } else { + matchItem = false; } break; } @@ -1260,24 +1269,34 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); } else if (strcasecmp("fsDebugFlag", name) == 0) { fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'i': { if (strcasecmp("idxDebugFlag", name) == 0) { idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'j': { if (strcasecmp("jniDebugFlag", name) == 0) { jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'k': { if (strcasecmp("keepColumnName", name) == 0) { tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; + } else if (strcasecmp("keepAliveIdle", name) == 0) { + tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->bval; + } else { + matchItem = false; } break; } @@ -1292,32 +1311,28 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); } else if (strcasecmp("logKeepDays", name) == 0) { tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + } else { + matchItem = false; } break; } case 'm': { switch (lowcaseName[1]) { case 'a': { - if (strcasecmp("maxShellConns", name) == 0) { - tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; - } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { - tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; - } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { + if (strcasecmp("maxInsertBatchRows", name) == 0) { tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; } else if (strcasecmp("maxRetryWaitTime", name) == 0) { tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; - } - break; - } - case 'd': { - if (strcasecmp("mDebugFlag", name) == 0) { - mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'e': { if (strcasecmp("metaCacheMaxSize", name) == 0) { atomic_store_32(&tsMetaCacheMaxSize, cfgGetItem(pCfg, "metaCacheMaxSize")->i32); + } else { + matchItem = false; } break; } @@ -1332,34 +1347,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; } else if (strcasecmp("minimalLogDirGB", name) == 0) { tsLogSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalLogDirGB")->fval) * 1024 * 1024 * 1024); - } - break; - } - case 'o': { - if (strcasecmp("monitor", name) == 0) { - tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; - } else if (strcasecmp("monitorInterval", name) == 0) { - tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; - } else if (strcasecmp("monitorFqdn", name) == 0) { - tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); - } else if (strcasecmp("monitorPort", name) == 0) { - tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; - } else if (strcasecmp("monitorMaxLogs", name) == 0) { - tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; - } else if (strcasecmp("monitorComp", name) == 0) { - tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; - } - break; - } - case 'q': { - if (strcasecmp("mqRebalanceInterval", name) == 0) { - tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; - } - break; - } - case 'u': { - if (strcasecmp("udfDebugFlag", name) == 0) { - udfDebugFlag = cfgGetItem(pCfg, "udfDebugFlag")->i32; + } else { + matchItem = false; } break; } @@ -1370,55 +1359,18 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { break; } case 'n': { - if (strcasecmp("numOfTaskQueueThreads", name) == 0) { - tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; - } else if (strcasecmp("numOfRpcThreads", name) == 0) { - tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; - } else if (strcasecmp("numOfRpcSessions", name) == 0) { - tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; - } else if (strcasecmp("numOfCommitThreads", name) == 0) { - tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; - } else if (strcasecmp("numOfMnodeReadThreads", name) == 0) { - tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; - } else if (strcasecmp("numOfVnodeQueryThreads", name) == 0) { - tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; - /* - } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) { - tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; - */ - } else if (strcasecmp("numOfVnodeRsmaThreads", name) == 0) { - tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; - } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { - tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; - /* - } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { - tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; - */ - } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { - tsNumOfSnodeStreamThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; - } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { - tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; - } else if (strcasecmp("numOfLogLines", name) == 0) { + if (strcasecmp("numOfLogLines", name) == 0) { tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; - } - break; - } - case 'p': { - if (strcasecmp("printAuth", name) == 0) { - tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; + } else { + matchItem = false; } break; } case 'q': { - if (strcasecmp("queryPolicy", name) == 0) { - tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; - } else if (strcasecmp("querySmaOptimize", name) == 0) { + if (strcasecmp("querySmaOptimize", name) == 0) { tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; - } else if (strcasecmp("queryBufferSize", name) == 0) { - tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; - } + } else if (strcasecmp("queryPolicy", name) == 0) { + tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; } else if (strcasecmp("qDebugFlag", name) == 0) { qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; } else if (strcasecmp("queryPlannerTrace", name) == 0) { @@ -1427,16 +1379,16 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; } else if (strcasecmp("queryUseNodeAllocator", name) == 0) { tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; - } else if (strcasecmp("queryRsmaTolerance", name) == 0) { - tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; + } else { + matchItem = false; } break; } case 'r': { - if (strcasecmp("rpcQueueMemoryAllowed", name) == 0) { - tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; - } else if (strcasecmp("rpcDebugFlag", name) == 0) { + if (strcasecmp("rpcDebugFlag", name) == 0) { rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + } else { + matchItem = false; } break; } @@ -1464,10 +1416,6 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsSmlDot2Underline = cfgGetItem(pCfg, "smlDot2Underline")->bval; } else if (strcasecmp("shellActivityTimer", name) == 0) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; - } else if (strcasecmp("supportVnodes", name) == 0) { - tsNumOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; - } else if (strcasecmp("statusInterval", name) == 0) { - tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; } else if (strcasecmp("serverPort", name) == 0) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; @@ -1481,8 +1429,6 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); - } else if (strcasecmp("sDebugFlag", name) == 0) { - sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; } else if (strcasecmp("smaDebugFlag", name) == 0) { smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; } else if (strcasecmp("slowLogThreshold", name) == 0) { @@ -1491,6 +1437,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; } + } else { + matchItem = false; } break; } @@ -1507,58 +1455,22 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); return -1; } - } else if (strcasecmp("tdbDebugFlag", name) == 0) { - tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32; - } else if (strcasecmp("telemetryReporting", name) == 0) { - tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; - } else if (strcasecmp("telemetryInterval", name) == 0) { - tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; } else if (strcasecmp("telemetryServer", name) == 0) { tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); - } else if (strcasecmp("telemetryPort", name) == 0) { - tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; - } else if (strcasecmp("transPullupInterval", name) == 0) { - tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; - } else if (strcasecmp("ttlUnit", name) == 0) { - tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; - } else if (strcasecmp("ttlPushInterval", name) == 0) { - tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32; - } else if (strcasecmp("ttlBatchDropNum", name) == 0) { - tsTtlBatchDropNum = cfgGetItem(pCfg, "ttlBatchDropNum")->i32; - } else if (strcasecmp("trimVDbIntervalSec", name) == 0) { - tsTrimVDbIntervalSec = cfgGetItem(pCfg, "trimVDbIntervalSec")->i32; } else if (strcasecmp("tmrDebugFlag", name) == 0) { tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; - } else if (strcasecmp("tsdbDebugFlag", name) == 0) { - tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; - } else if (strcasecmp("tqDebugFlag", name) == 0) { - tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; - } else if (strcasecmp("ttlFlushThreshold", name) == 0) { - tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32; - } else if (strcasecmp("timeseriesThreshold", name) == 0) { - tsTimeSeriesThreshold = cfgGetItem(pCfg, "timeseriesThreshold")->i32; + } else { + matchItem = false; } break; } case 'u': { - if (strcasecmp("udf", name) == 0) { - tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; - } else if (strcasecmp("uDebugFlag", name) == 0) { + if (strcasecmp("uDebugFlag", name) == 0) { uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; } else if (strcasecmp("useAdapter", name) == 0) { tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; - } - break; - } - case 'v': { - if (strcasecmp("vDebugFlag", name) == 0) { - vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; - } - break; - } - case 'w': { - if (strcasecmp("wDebugFlag", name) == 0) { - wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + } else { + matchItem = false; } break; } @@ -1567,7 +1479,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { return -1; } - return 0; + if (!matchItem) terrno = TSDB_CODE_CFG_NOT_FOUND; + return matchItem ? 0 : -1; } int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, @@ -1710,6 +1623,10 @@ void taosCleanupCfg() { tsCfg = NULL; } } +typedef struct { + const char *optionName; + void *optionVar; +} OptionNameAndVar; void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { @@ -1724,95 +1641,106 @@ void taosCfgDynamicOptions(const char *option, const char *value) { return; } - if (strcasecmp(option, "monitor") == 0) { - int32_t monitor = atoi(value); - uInfo("monitor set from %d to %d", tsEnableMonitor, monitor); - tsEnableMonitor = monitor; - SConfigItem *pItem = cfgGetItem(tsCfg, "monitor"); - if (pItem != NULL) { - pItem->bval = tsEnableMonitor; + { // 'bool/int32_t/int64_t' variables with general modification function + const int32_t nDebugFlag = 20; + static OptionNameAndVar options[] = { + {"dDebugFlag", &dDebugFlag}, + {"vDebugFlag", &vDebugFlag}, + {"mDebugFlag", &mDebugFlag}, + {"wDebugFlag", &wDebugFlag}, + {"sDebugFlag", &sDebugFlag}, + {"tsdbDebugFlag", &tsdbDebugFlag}, + {"tqDebugFlag", &tqDebugFlag}, + {"fsDebugFlag", &fsDebugFlag}, + {"udfDebugFlag", &udfDebugFlag}, + {"smaDebugFlag", &smaDebugFlag}, + {"idxDebugFlag", &idxDebugFlag}, + {"tdbDebugFlag", &tdbDebugFlag}, + {"tmrDebugFlag", &tmrDebugFlag}, + {"uDebugFlag", &uDebugFlag}, + {"smaDebugFlag", &smaDebugFlag}, + {"rpcDebugFlag", &rpcDebugFlag}, + {"qDebugFlag", &qDebugFlag}, + {"metaDebugFlag", &metaDebugFlag}, + {"jniDebugFlag", &jniDebugFlag}, + {"stDebugFlag", &stDebugFlag}, + + {"audit", &tsEnableAudit}, + {"asynclog", &tsAsyncLog}, + {"disableStream", &tsDisableStream}, + {"enableWhiteList", &tsEnableWhiteList}, + {"telemetryReporting", &tsEnableTelem}, + {"monitor", &tsEnableMonitor}, + + {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, + {"minDiskFreeSize", &tsMinDiskFreeSize}, + + {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, + {"checkpointInterval", &tsStreamCheckpointInterval}, + {"keepAliveIdle", &tsKeepAliveIdle}, + {"logKeepDays", &tsLogKeepDays}, + {"maxStreamBackendCache", &tsMaxStreamBackendCache}, + {"mqRebalanceInterval", &tsMqRebalanceInterval}, + {"numOfLogLines", &tsNumOfLogLines}, + {"queryRspPolicy", &tsQueryRspPolicy}, + {"timeseriesThreshold", &tsTimeSeriesThreshold}, + {"tmqMaxTopicNum", &tmqMaxTopicNum}, + {"transPullupInterval", &tsTransPullupInterval}, + {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, + {"ttlBatchDropNum", &tsTtlBatchDropNum}, + {"ttlFlushThreshold", &tsTtlFlushThreshold}, + {"ttlPushInterval", &tsTtlPushIntervalSec}, + {"s3BlockCacheSize", &tsS3BlockCacheSize}, + {"s3PageCacheSize", &tsS3PageCacheSize}, + {"s3UploadDelaySec", &tsS3UploadDelaySec}, + {"supportVnodes", &tsNumOfSupportVnodes}, + }; + + int32_t optionSize = tListLen(options); + for (int32_t d = 0; d < optionSize; ++d) { + const char *optName = options[d].optionName; + int32_t optLen = strlen(optName); + if (strncasecmp(option, optName, optLen) != 0) continue; + + SConfig *pCfg = taosGetCfg(); + SConfigItem *pItem = NULL; + + pItem = cfgGetItem(pCfg, optName); + if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { + uError("failed to config:%s, not support", optName); + break; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + int32_t flag = atoi(value); + bool *pVar = options[d].optionVar; + uInfo("%s set from %d to %d", optName, *pVar, flag); + *pVar = flag; + } break; + case CFG_DTYPE_INT32: { + int32_t flag = atoi(value); + int32_t *pVar = options[d].optionVar; + uInfo("%s set from %d to %d", optName, *pVar, flag); + *pVar = flag; + + if (d < nDebugFlag) { + // debug flags + taosSetDebugFlag(options[d].optionVar, optName, flag, true); + } + } break; + case CFG_DTYPE_INT64: { + int64_t flag = atoll(value); + int64_t *pVar = options[d].optionVar; + uInfo("%s set from %" PRId64 " to %" PRId64, optName, *pVar, flag); + *pVar = flag; + } break; + default: + break; + } + + return; } - return; - } - /* cannot alter s3BlockSize - if (strcasecmp(option, "s3BlockSize") == 0) { - int32_t newS3BlockSize = atoi(value); - uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize); - tsS3BlockSize = newS3BlockSize; - return; - } - */ - if (strcasecmp(option, "s3BlockCacheSize") == 0) { - int32_t newS3BlockCacheSize = atoi(value); - uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize); - tsS3BlockCacheSize = newS3BlockCacheSize; - return; - } - - if (strcasecmp(option, "s3PageCacheSize") == 0) { - int32_t newS3PageCacheSize = atoi(value); - uInfo("s3PageCacheSize set from %d to %d", tsS3PageCacheSize, newS3PageCacheSize); - tsS3PageCacheSize = newS3PageCacheSize; - return; - } - - if (strcasecmp(option, "s3UploadDelaySec") == 0) { - int32_t newS3UploadDelaysec = atoi(value); - uInfo("s3UploadDelaySec set from %d to %d", tsS3UploadDelaySec, newS3UploadDelaysec); - tsS3UploadDelaySec = newS3UploadDelaysec; - return; - } - - if (strcasecmp(option, "ttlPushInterval") == 0) { - int32_t newTtlPushInterval = atoi(value); - uInfo("ttlPushInterval set from %d to %d", tsTtlPushIntervalSec, newTtlPushInterval); - tsTtlPushIntervalSec = newTtlPushInterval; - return; - } - - if (strcasecmp(option, "ttlBatchDropNum") == 0) { - int32_t newTtlBatchDropNum = atoi(value); - uInfo("ttlBatchDropNum set from %d to %d", tsTtlBatchDropNum, newTtlBatchDropNum); - tsTtlBatchDropNum = newTtlBatchDropNum; - return; - } - - if (strcasecmp(option, "supportVnodes") == 0) { - int32_t newSupportVnodes = atoi(value); - uInfo("supportVnodes set from %d to %d", tsNumOfSupportVnodes, newSupportVnodes); - tsNumOfSupportVnodes = newSupportVnodes; - return; - } - - if (strcasecmp(option, "asynclog") == 0) { - int32_t newAsynclog = atoi(value); - uInfo("asynclog set from %d to %d", tsAsyncLog, newAsynclog); - tsAsyncLog = newAsynclog; - return; - } - - const char *options[] = { - "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", "tqDebugFlag", - "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag", "tmrDebugFlag", "uDebugFlag", - "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", "metaDebugFlag", "jniDebugFlag", "stDebugFlag", - }; - int32_t *optionVars[] = { - &dDebugFlag, &vDebugFlag, &mDebugFlag, &wDebugFlag, &sDebugFlag, &tsdbDebugFlag, &tqDebugFlag, - &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tdbDebugFlag, &tmrDebugFlag, &uDebugFlag, - &smaDebugFlag, &rpcDebugFlag, &qDebugFlag, &metaDebugFlag, &jniDebugFlag, &stDebugFlag, - }; - - int32_t optionSize = tListLen(options); - for (int32_t d = 0; d < optionSize; ++d) { - const char *optName = options[d]; - int32_t optLen = strlen(optName); - if (strncasecmp(option, optName, optLen) != 0) continue; - - int32_t flag = atoi(value); - uInfo("%s set from %d to %d", optName, *optionVars[d], flag); - *optionVars[d] = flag; - taosSetDebugFlag(optionVars[d], optName, flag, true); - return; } uError("failed to cfg dynamic option:%s value:%s", option, value); From 885acd1a4281eb6b7b79960010cb342ef198560f Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 7 Nov 2023 20:29:56 +0800 Subject: [PATCH 13/20] fix:download checkpoint from s3 --- source/common/src/cos.c | 13 +++++++++++-- source/common/src/tglobal.c | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index c209f1a063..4b079a17cb 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -822,9 +822,18 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char* path){ for(size_t i = 0; i < taosArrayGetSize(objectArray); i++){ char* object = taosArrayGetP(objectArray, i); + const char* tmp = strchr(object, '/'); + tmp = (tmp == NULL) ? object : tmp + 1; char fileName[PATH_MAX] = {0}; - snprintf(fileName, PATH_MAX, "%s/%s", path, object); - s3GetObjectToFile(object, fileName); + if(path[strlen(path) - 1] != '/'){ + snprintf(fileName, PATH_MAX, "%s/%s", path, tmp); + }else{ + snprintf(fileName, PATH_MAX, "%s%s", path, tmp); + } + if(s3GetObjectToFile(object, fileName) != 0){ + taosArrayDestroyEx(objectArray, s3FreeObjectKey); + return -1; + } } taosArrayDestroyEx(objectArray, s3FreeObjectKey); return 0; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index c844261007..cab4325351 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -672,9 +672,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "snodeRsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "snodeRsyncAddress", tsSnodeAddress, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "snodeRsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "snodeRsyncAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; From 07aa749fcd352b9b9ebb6e23a6dc0d9f852af103 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Nov 2023 10:28:45 +0800 Subject: [PATCH 14/20] fix:adapt to windows --- include/os/osDef.h | 6 ++++++ source/common/src/cos.c | 4 ++-- source/libs/stream/src/streamCheckpoint.c | 5 +++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/include/os/osDef.h b/include/os/osDef.h index bad4536fa6..1a831f2e86 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -237,6 +237,12 @@ void syslog(int unused, const char *format, ...); #define TD_DIRSEP "/" #endif +#if defined(_WIN32) +#define TD_DIRSEP_CHAR '\\' +#else +#define TD_DIRSEP_CHAR '/' +#endif + #define TD_LOCALE_LEN 64 #define TD_CHARSET_LEN 64 #define TD_TIMEZONE_LEN 96 diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 4b079a17cb..423b7c1243 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -825,8 +825,8 @@ int32_t s3GetObjectsByPrefix(const char *prefix, const char* path){ const char* tmp = strchr(object, '/'); tmp = (tmp == NULL) ? object : tmp + 1; char fileName[PATH_MAX] = {0}; - if(path[strlen(path) - 1] != '/'){ - snprintf(fileName, PATH_MAX, "%s/%s", path, tmp); + if(path[strlen(path) - 1] != TD_DIRSEP_CHAR){ + snprintf(fileName, PATH_MAX, "%s%s%s", path, TD_DIRSEP, tmp); }else{ snprintf(fileName, PATH_MAX, "%s%s", path, tmp); } diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index c3152aebb7..b4a9dd3b43 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -478,7 +478,7 @@ static int uploadCheckpointToS3(char* id, char* path){ taosDirEntryIsDir(de)) continue; char filename[PATH_MAX] = {0}; - if(path[strlen(path - 1)] == '/'){ + if(path[strlen(path - 1)] == TD_DIRSEP_CHAR){ snprintf(filename, sizeof(filename), "%s%s", path, name); }else{ snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name); @@ -491,6 +491,7 @@ static int uploadCheckpointToS3(char* id, char* path){ taosCloseDir(&pDir); return -1; } + stDebug("[s3] upload checkpoint:%s", filename); } taosCloseDir(&pDir); @@ -548,7 +549,7 @@ int deleteCheckpoint(char* id){ int deleteCheckpointFile(char* id, char* name){ char object[128] = {0}; - snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name); + snprintf(object, sizeof(object), "%s/%s", id, name); char *tmp = object; s3DeleteObjects((const char**)&tmp, 1); return 0; From 047eb14f673aeb0e1ec10a56dee9c00fd9660e19 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Nov 2023 11:57:49 +0800 Subject: [PATCH 15/20] fix:error --- source/common/src/cos.c | 10 +++++----- source/libs/stream/src/streamCheckpoint.c | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 423b7c1243..3a775816f7 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -298,7 +298,7 @@ S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { } S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { - responsePropertiesCallback(properties, callbackData); +// responsePropertiesCallback(properties, callbackData); MultipartPartData *data = (MultipartPartData *)callbackData; int seq = data->seq; @@ -476,7 +476,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { metaProperties, useServerSideEncryption}; if (contentLength <= MULTIPART_CHUNK_SIZE) { - S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + S3PutObjectHandler putObjectHandler = {{NULL, &responseCompleteCallback}, &putObjectDataCallback}; do { @@ -513,14 +513,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { memset(&partData, 0, sizeof(MultipartPartData)); int partContentLength = 0; - S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback}, + S3MultipartInitialHandler handler = {{NULL, &responseCompleteCallback}, &initial_multipart_callback}; S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback}, &putObjectDataCallback}; S3MultipartCommitHandler commit_handler = { - {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; + {NULL, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); manager.next_etags_pos = 0; @@ -668,7 +668,7 @@ static void s3FreeObjectKey(void *pItem) { static SArray* getListByPrefix(const char *prefix){ S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; - S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + S3ListBucketHandler listBucketHandler = {{NULL, &responseCompleteCallback}, &listBucketCallback}; const char *marker = 0, *delimiter = 0; diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index b4a9dd3b43..3cfdd65a77 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -478,7 +478,7 @@ static int uploadCheckpointToS3(char* id, char* path){ taosDirEntryIsDir(de)) continue; char filename[PATH_MAX] = {0}; - if(path[strlen(path - 1)] == TD_DIRSEP_CHAR){ + if(path[strlen(path) - 1] == TD_DIRSEP_CHAR){ snprintf(filename, sizeof(filename), "%s%s", path, name); }else{ snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name); From 5ed7b8aab3e9326a597587d397d341f649807d55 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Nov 2023 14:09:04 +0800 Subject: [PATCH 16/20] fix:error --- source/common/src/cos.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index 3a775816f7..ebe3a6d7cd 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -85,6 +85,11 @@ typedef struct { char *buf; } TS3SizeCBD; +static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *properties, void *callbackData) { +// (void)callbackData; + return S3StatusOK; +} + static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { //(void)callbackData; TS3SizeCBD *cbd = callbackData; @@ -298,7 +303,7 @@ S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { } S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { -// responsePropertiesCallback(properties, callbackData); + responsePropertiesCallbackNull(properties, callbackData); MultipartPartData *data = (MultipartPartData *)callbackData; int seq = data->seq; @@ -397,7 +402,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; - S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback}; + S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &listPartsCallback}; list_parts_callback_data data; @@ -476,7 +481,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { metaProperties, useServerSideEncryption}; if (contentLength <= MULTIPART_CHUNK_SIZE) { - S3PutObjectHandler putObjectHandler = {{NULL, &responseCompleteCallback}, + S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &putObjectDataCallback}; do { @@ -513,14 +518,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { memset(&partData, 0, sizeof(MultipartPartData)); int partContentLength = 0; - S3MultipartInitialHandler handler = {{NULL, &responseCompleteCallback}, + S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &initial_multipart_callback}; S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback}, &putObjectDataCallback}; S3MultipartCommitHandler commit_handler = { - {NULL, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; + {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); manager.next_etags_pos = 0; @@ -668,7 +673,7 @@ static void s3FreeObjectKey(void *pItem) { static SArray* getListByPrefix(const char *prefix){ S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; - S3ListBucketHandler listBucketHandler = {{NULL, &responseCompleteCallback}, + S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &listBucketCallback}; const char *marker = 0, *delimiter = 0; @@ -792,7 +797,7 @@ int32_t s3GetObjectToFile(const char *object_name, char* fileName) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; - S3GetObjectHandler getObjectHandler = {{NULL, &responseCompleteCallback}, + S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &getObjectCallback}; TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); From 2e7d041e0a0585d967ed056bff516d9ce2f394a4 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 14:48:28 +0800 Subject: [PATCH 17/20] fix stream transfer err --- source/libs/stream/src/streamSnapshot.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 3de5de9967..618b392861 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -19,7 +19,6 @@ #include "streamBackendRocksdb.h" #include "streamInt.h" #include "tcommon.h" -#include "streamInt.h" enum SBackendFileType { ROCKSDB_OPTIONS_TYPE = 1, @@ -52,6 +51,7 @@ struct SStreamSnapHandle { int8_t filetype; SArray* pFileList; int32_t currFileIdx; + int8_t delFlag; // 0 : not del, 1: del }; struct SStreamSnapBlockHdr { int8_t type; @@ -274,7 +274,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { if (handle->checkpointId == 0) { // del tmp dir if (pFile && taosIsDir(pFile->path)) { - taosRemoveDir(pFile->path); + if (handle->delFlag) taosRemoveDir(pFile->path); } } else { streamBackendDelInUseChkp(handle->handle, handle->checkpointId); @@ -308,6 +308,7 @@ int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* pa taosMemoryFree(pReader); return -1; } + pReader->handle.delFlag = 1; *ppReader = pReader; @@ -344,10 +345,10 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si stDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", STREAM_STATE_TRANSFER, item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); - if(buf == NULL){ + if (buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); + int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); if (nread == -1) { taosMemoryFree(buf); code = TAOS_SYSTEM_ERROR(terrno); @@ -423,6 +424,7 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path pHandle->pFileList = list; pHandle->currFileIdx = 0; pHandle->offset = 0; + pHandle->delFlag = 0; *ppWriter = pWriter; return 0; From 7e03fa6664e0cf46671aef5a1d6872e752d79f1e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 8 Nov 2023 14:57:20 +0800 Subject: [PATCH 18/20] fix stream transfer err --- source/libs/stream/src/streamSnapshot.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 618b392861..757164739a 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -148,6 +148,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk taosMemoryFree(tdir); return code; } + pHandle->delFlag = 1; chkpId = 0; } @@ -308,7 +309,6 @@ int32_t streamSnapReaderOpen(void* pMeta, int64_t sver, int64_t chkpId, char* pa taosMemoryFree(pReader); return -1; } - pReader->handle.delFlag = 1; *ppReader = pReader; From c4c1423243cf797f8c360c3e1da021a9b86866cc Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Nov 2023 15:01:04 +0800 Subject: [PATCH 19/20] fix:error --- source/libs/stream/inc/streamInt.h | 7 +- source/libs/stream/src/streamCheckpoint.c | 92 ---------------------- source/libs/stream/test/checkpointTest.cpp | 6 +- 3 files changed, 4 insertions(+), 101 deletions(-) diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index f1904850d7..4efba478f3 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -140,17 +140,12 @@ void* streamQueueNextItem(SStreamQueue* pQueue); void streamFreeQitem(SStreamQueueItem* data); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); -//#define CHECKPOINT_PATH_LEN 128 -//typedef struct SChekpointDataHeader{ -// int64_t size; -// char name[CHECKPOINT_PATH_LEN]; -// char id[CHECKPOINT_PATH_LEN]; -//} SChekpointDataHeader; typedef enum UPLOAD_TYPE{ UPLOAD_DISABLE = -1, UPLOAD_S3 = 0, UPLOAD_RSYNC = 1, } UPLOAD_TYPE; + UPLOAD_TYPE getUploadType(); int uploadCheckpoint(char* id, char* path); int downloadCheckpoint(char* id, char* path); diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 3cfdd65a77..5479a2dab2 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -375,98 +375,6 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { return code; } - -//static int64_t kBlockSize = 64 * 1024; -//static int sendCheckpointToS3(char* id, SArray* fileList){ -// code = s3PutObjectFromFile2(from->fname, object_name); -// return 0; -//} -//static int sendCheckpointToSnode(char* id, SArray* fileList){ -// if(strlen(id) >= CHECKPOINT_PATH_LEN){ -// tqError("uploadCheckpoint id name too long, name:%s", id); -// return -1; -// } -// uint8_t* buf = taosMemoryCalloc(1, sizeof(SChekpointDataHeader) + kBlockSize); -// if(buf == NULL){ -// tqError("uploadCheckpoint malloc failed"); -// return -1; -// } -// -// SChekpointDataHeader* pHdr = (SChekpointDataHeader*)buf; -// strcpy(pHdr->id, id); -// -// TdFilePtr fd = NULL; -// for(int i = 0; i < taosArrayGetSize(fileList); i++){ -// char* name = (char*)taosArrayGetP(fileList, i); -// if(strlen(name) >= CHECKPOINT_PATH_LEN){ -// tqError("uploadCheckpoint file name too long, name:%s", name); -// return -1; -// } -// int64_t offset = 0; -// -// fd = taosOpenFile(name, TD_FILE_READ); -// tqDebug("uploadCheckpoint open file %s, file index: %d", name, i); -// -// while(1){ -// int64_t nread = taosPReadFile(fd, buf + sizeof(SChekpointDataHeader), kBlockSize, offset); -// if (nread == -1) { -// taosCloseFile(&fd); -// taosMemoryFree(buf); -// tqError("uploadCheckpoint failed to read file name:%s,reason:%d", name, errno); -// return -1; -// } else if (nread == 0){ -// tqDebug("uploadCheckpoint no data read, close file:%s, move to next file, open and read", name); -// taosCloseFile(&fd); -// break; -// } else if (nread == kBlockSize){ -// offset += nread; -// } else { -// taosCloseFile(&fd); -// offset = 0; -// } -// tqDebug("uploadCheckpoint read file %s, size:%" PRId64 ", current offset:%" PRId64, name, nread, offset); -// -// -// pHdr->size = nread; -// strcpy(pHdr->name, name); -// -// SRpcMsg rpcMsg = {0}; -// int32_t bytes = sizeof(SChekpointDataHeader) + nread; -// rpcMsg.pCont = rpcMallocCont(bytes); -// rpcMsg.msgType = TDMT_SYNC_SNAPSHOT_SEND; -// rpcMsg.contLen = bytes; -// if (rpcMsg.pCont == NULL) { -// tqError("uploadCheckpoint malloc failed"); -// taosCloseFile(&fd); -// taosMemoryFree(buf); -// return -1; -// } -// memcpy(rpcMsg.pCont, buf, bytes); -// int try = 3; -// int32_t code = 0; -// while(try-- > 0){ -// code = tmsgSendReq(pEpSet, &rpcMsg); -// if(code == 0) -// break; -// taosMsleep(10); -// } -// if(code != 0){ -// tqError("uploadCheckpoint send request failed code:%d", code); -// taosCloseFile(&fd); -// taosMemoryFree(buf); -// return -1; -// } -// -// if(offset == 0){ -// break; -// } -// } -// } -// -// taosMemoryFree(buf); - -//} - static int uploadCheckpointToS3(char* id, char* path){ TdDirPtr pDir = taosOpenDir(path); if (pDir == NULL) return -1; diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp index 135431cfb7..dca2e97c28 100644 --- a/source/libs/stream/test/checkpointTest.cpp +++ b/source/libs/stream/test/checkpointTest.cpp @@ -38,13 +38,13 @@ int main(int argc, char **argv) { if (s3Init() < 0) { return -1; } -// strcpy(tsSnodeIp, "127.0.0.1"); + strcpy(tsSnodeAddress, "127.0.0.1"); return RUN_ALL_TESTS(); } TEST(testCase, checkpointUpload_Test) { -// stopRsync(); -// startRsync(); + stopRsync(); + startRsync(); taosSsleep(5); char* id = "2013892036"; From e7bf159174516f66e18f7339a98294a5ce5c61ca Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 8 Nov 2023 16:56:05 +0800 Subject: [PATCH 20/20] fix:compile error --- source/common/src/cos.c | 1 + source/common/src/tglobal.c | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/source/common/src/cos.c b/source/common/src/cos.c index ebe3a6d7cd..daeb81b8a3 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -1314,6 +1314,7 @@ void s3DeleteObjectsByPrefix(const char *prefix) {} void s3DeleteObjects(const char *object_name[], int nobject) {} bool s3Exists(const char *object_name) { return false; } bool s3Get(const char *object_name, const char *path) { return false; } +int32_t s3GetObjectsByPrefix(const char *prefix, const char* path) { return 0; } int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { return 0; } void s3EvictCache(const char *path, long object_size) {} long s3Size(const char *object_name) { return 0; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2b587b147e..d441b22aa3 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -672,8 +672,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "snodeRsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "snodeRsyncAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) @@ -1592,9 +1592,6 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile return -1; } - if (taosSetS3Cfg(tsCfg) != 0) return -1; - return 0; - if (tsc) { if (taosSetClientCfg(tsCfg)) return -1; } else {