diff --git a/Jenkinsfile2 b/Jenkinsfile2 index f308db759d..a46736d7f3 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -316,7 +316,7 @@ def pre_test_build_win() { python -m pip uninstall taospy -y python -m pip install taospy==2.7.12 python -m pip uninstall taos-ws-py -y - python -m pip install taos-ws-py==0.2.9 + python -m pip install taos-ws-py==0.3.1 xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\lib\\taos.dll C:\\Windows\\System32 ''' return 1 diff --git a/cmake/curl_CMakeLists.txt.in b/cmake/curl_CMakeLists.txt.in index 458a518092..af7f005dda 100644 --- a/cmake/curl_CMakeLists.txt.in +++ b/cmake/curl_CMakeLists.txt.in @@ -11,7 +11,7 @@ ExternalProject_Add(curl2 BUILD_IN_SOURCE TRUE BUILD_ALWAYS 1 UPDATE_COMMAND "" - CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 #--enable-debug + CONFIGURE_COMMAND ./configure --prefix=$ENV{HOME}/.cos-local.2 --with-ssl=$ENV{HOME}/.cos-local.2 --enable-shared=no --disable-ldap --disable-ldaps --without-brotli --without-zstd --without-libidn2 --without-nghttp2 #--enable-debug BUILD_COMMAND make -j INSTALL_COMMAND make install TEST_COMMAND "" diff --git a/cmake/ssl_CMakeLists.txt.in b/cmake/ssl_CMakeLists.txt.in index c9f836bade..fe176498a5 100644 --- a/cmake/ssl_CMakeLists.txt.in +++ b/cmake/ssl_CMakeLists.txt.in @@ -8,7 +8,7 @@ ExternalProject_Add(openssl BUILD_IN_SOURCE TRUE #BUILD_ALWAYS 1 #UPDATE_COMMAND "" - CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 -static #--no-shared + CONFIGURE_COMMAND ./Configure --prefix=$ENV{HOME}/.cos-local.2 no-shared BUILD_COMMAND make -j INSTALL_COMMAND make install_sw -j TEST_COMMAND "" diff --git a/deps/x86/rocksdb_static/librocksdb.a b/deps/x86/rocksdb_static/librocksdb.a index 0c788dd674..0461fce8e4 100644 Binary files a/deps/x86/rocksdb_static/librocksdb.a and b/deps/x86/rocksdb_static/librocksdb.a differ diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 2ea144c56a..4f8ccc418b 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -565,7 +565,7 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) - The uppercase or lowercase of `MONTH`, `MON`, `DAY`, `DY` and formtas that output digits have same effect when used in `to_timestamp`, like `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month` can be replaced by `MONTH`, or `month`. The cases are ignored. - If multi times are specified for one component, the previous will be overwritten. Like `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, the output year will be `2022`. - To avoid unexpected time zone used during the convertion, it's recommended to put time zone in the ts string, e.g. '2023-10-10 10:10:10+08'. If time zone not specified, default will be that in server or client. -- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with the timezone specified or default to local timezone. +- The default timestamp if some components are not specified will be: `1970-01-01 00:00:00` with the timezone specified or default to local timezone. Only `DDD` is specified without `DD` is not supported currently, e.g. format 'yyyy-mm-ddd' is not supported, but 'yyyy-mm-dd' is supported. - If `AM` or `PM` is specified in formats, the Hour must between `1-12`. - In some cases, `to_timestamp` can convert correctly even the format and the timestamp string are not totally matched. Like `to_timetamp('200101/2', 'yyyyMM1/dd')`, the digit `1` in format string are ignored, and the output timestsamp is `2001-01-02 00:00:00`. Spaces and tabs in formats and tiemstamp string are also ignored automatically. - The precision of the output timestamp will be the same as the table in SELECT stmt, millisecond will be used if no table is specified. The output of `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')` will be truncated to millisecond precision. If a nano precision table is specified, no truncation will be applied. Like `select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`. diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md index 7f0b8c7769..502b7562f3 100644 --- a/docs/en/12-taos-sql/12-distinguished.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -38,11 +38,16 @@ Aggregation by time window is supported in TDengine. For example, in the case wh window_clause: { SESSION(ts_col, tol_val) | STATE_WINDOW(col) - | INTERVAL(interval [, offset]) [SLIDING sliding] [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] + | INTERVAL(interval_val [, offset]) [SLIDING (sliding_value)] [FILL({NONE | VALUE | PREV | NULL | LINEAR | NEXT})] | EVENT_WINDOW START WITH start_trigger_condition END WITH end_trigger_condition } ``` +Both interval_val and sliding_value are time durations which have 3 forms of representation. +- INTERVAL(1s, 500a) SLIDING(1s), the unit char should be any one of a (millisecond), b (nanosecond), d (day), h (hour), m (minute), n (month), s (second), u (microsecond), w (week), y (year). +- INTERVAL(1000, 500) SLIDING(1000), the unit will the same as the queried database, if there are more than one databases, higher precision will be used. +- INTERVAL('1s', '500a') SLIDING('1s'), unit must be specified, no spaces allowed. + The following restrictions apply: ### Other Rules diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index c1dc6a6363..26313390a6 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -565,7 +565,7 @@ TO_TIMESTAMP(ts_str_literal, format_str_literal) - `MONTH`, `MON`, `DAY`, `DY` 以及其他输出为数字的格式的大小写意义相同, 如 `to_timestamp('2023-JANUARY-01', 'YYYY-month-dd')`, `month`可以被替换为`MONTH` 或者`Month`. - 如果同一字段被指定了多次, 那么前面的指定将会被覆盖. 如 `to_timestamp('2023-22-10-10', 'yyyy-yy-MM-dd')`, 输出年份是`2022`. - 为避免转换时使用了非预期的时区，推荐在时间中携带时区信息，例如'2023-10-10 10:10:10+08'，如果未指定时区则默认时区为服务端或客户端指定的时区。 -- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分. +- 如果没有指定完整的时间，那么默认时间值为指定或默认时区的 `1970-01-01 00:00:00`, 未指定部分使用该默认值中的对应部分. 暂不支持只指定年日而不指定月日的格式, 如'yyyy-mm-DDD', 支持'yyyy-mm-DD'. - 如果格式串中有`AM`, `PM`等, 那么小时必须是12小时制, 范围必须是01-12. - `to_timestamp`转换具有一定的容错机制, 在格式串和时间戳串不完全对应时, 有时也可转换, 如: `to_timestamp('200101/2', 'yyyyMM1/dd')`, 格式串中多出来的1会被丢弃. 格式串与时间戳串中多余的空格字符(空格, tab等)也会被 自动忽略. 如`to_timestamp(' 23 年 - 1 月 - 01 日 ', 'yy 年-MM月-dd日')` 可以被成功转换. 虽然`MM`等字段需要两个数字对应(只有一位时前面补0), 在`to_timestamp`时, 一个数字也可以成功转换. - 输出时间戳的精度与查询表的精度相同, 若查询未指定表, 则输出精度为毫秒. 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns')`的输出将会把微妙和纳秒进行截断. 如果指定一张纳秒表, 那么就不会发生截断, 如`select to_timestamp('2023-08-1 10:10:10.123456789', 'yyyy-mm-dd hh:mi:ss.ns') from db_ns.table_ns limit 1`. diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md index 62888cc5f7..98bfd3567a 100755 --- a/docs/zh/12-taos-sql/12-distinguished.md +++ b/docs/zh/12-taos-sql/12-distinguished.md @@ -44,7 +44,11 @@ window_clause: { } ``` -在上述语法中的具体限制如下 +其中，interval_val 和 sliding_val 都表示时间段, 语法上支持三种方式，举例说明如下: + - INTERVAL(1s, 500a) SLIDING(1s), 自带时间单位的形式，其中的时间单位是单字符表示, 分别为: a (毫秒), b (纳秒), d (天), h (小时), m (分钟), n (月), s (秒), u (微妙), w (周), y (年). + - INTERVAL(1000, 500) SLIDING(1000), 不带时间单位的形式，将使用查询库的时间精度作为默认时间单位，当存在多个库时默认采用精度更高的库. + - INTERVAL('1s', '500a') SLIDING('1s'), 自带时间单位的字符串形式，字符串内部不能有任何空格等其它字符. + ### 窗口子句的规则 diff --git a/source/dnode/vnode/src/inc/vndCos.h b/include/common/cos.h similarity index 90% rename from source/dnode/vnode/src/inc/vndCos.h rename to include/common/cos.h index 8581b039f8..21b645f604 100644 --- a/source/dnode/vnode/src/inc/vndCos.h +++ b/include/common/cos.h @@ -16,7 +16,7 @@ #ifndef _TD_VND_COS_H_ #define _TD_VND_COS_H_ -#include "vnd.h" +#include "os.h" #ifdef __cplusplus extern "C" { @@ -24,6 +24,7 @@ extern "C" { #define S3_BLOCK_CACHE +extern int8_t tsS3StreamEnabled; extern int8_t tsS3Enabled; extern int32_t tsS3BlockSize; extern int32_t tsS3BlockCacheSize; @@ -38,7 +39,8 @@ void s3DeleteObjectsByPrefix(const char *prefix); void s3DeleteObjects(const char *object_name[], int nobject); bool s3Exists(const char *object_name); bool s3Get(const char *object_name, const char *path); -int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock); +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock); +int32_t s3GetObjectsByPrefix(const char *prefix, const char *path); void s3EvictCache(const char *path, long object_size); long s3Size(const char *object_name); diff --git a/include/common/rsync.h b/include/common/rsync.h new file mode 100644 index 0000000000..6cce645d1e --- /dev/null +++ b/include/common/rsync.h @@ -0,0 +1,24 @@ +// +// Created by mingming wanng on 2023/11/2. +// + +#ifndef TDENGINE_RSYNC_H +#define TDENGINE_RSYNC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tarray.h" + +void stopRsync(); +void startRsync(); +int uploadRsync(char* id, char* path); +int downloadRsync(char* id, char* path); +int deleteRsync(char* id); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_RSYNC_H diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 72aab9adf0..e072eaa831 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -249,6 +249,7 @@ typedef struct SQueryTableDataCond { SColumnInfo* colList; int32_t* pSlotList; // the column output destation slot, and it may be null int32_t type; // data block load type: + bool skipRollup; STimeWindow twindows; int64_t startVersion; int64_t endVersion; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 1d0da9e150..8688e07932 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -44,10 +44,6 @@ extern int32_t tsNumOfSupportVnodes; extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; extern int32_t tsCompressMsgSize; -extern int32_t tsCompressColData; -extern int32_t tsMaxNumOfDistinctResults; -extern int32_t tsCompatibleModel; -extern bool tsPrintAuth; extern int64_t tsTickPerMin[3]; extern int64_t tsTickPerHour[3]; extern int32_t tsCountAlwaysReturnValue; @@ -79,8 +75,12 @@ extern int32_t tsElectInterval; extern int32_t tsHeartbeatInterval; extern int32_t tsHeartbeatTimeout; -// vnode -extern int64_t tsVndCommitMaxIntervalMs; +// snode +extern int32_t tsRsyncPort; +extern char tsCheckpointBackupDir[]; + +// vnode checkpoint +extern char tsSnodeAddress[]; //127.0.0.1:873 // mnode extern int64_t tsMndSdbWriteDelta; @@ -105,8 +105,6 @@ extern bool tsMonitorComp; // audit extern bool tsEnableAudit; -extern char tsAuditFqdn[]; -extern uint16_t tsAuditPort; extern bool tsEnableAuditCreateTable; // telem diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 5990ae1c9c..f78b7a3126 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -49,6 +49,7 @@ typedef struct { uint64_t checkpointId; bool initTableReader; bool initTqReader; + bool skipRollup; int32_t numOfVgroups; void* sContext; // SSnapContext* void* pStateBackend; @@ -184,11 +185,7 @@ void qDestroyTask(qTaskInfo_t tinfo); void qProcessRspMsg(void* parent, struct SRpcMsg* pMsg, struct SEpSet* pEpSet); -int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList /*,int32_t* resNum, SExplainExecInfo** pRes*/); - -int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); - -int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); +int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList); void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order); void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery); @@ -217,7 +214,7 @@ int32_t qStreamSourceScanParamForHistoryScanStep1(qTaskInfo_t tinfo, SVersionRan int32_t qStreamSourceScanParamForHistoryScanStep2(qTaskInfo_t tinfo, SVersionRange *pVerRange, STimeWindow* pWindow); int32_t qStreamRecoverFinish(qTaskInfo_t tinfo); int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo); -bool qStreamRecoverScanFinished(qTaskInfo_t tinfo); +bool qStreamScanhistoryFinished(qTaskInfo_t tinfo); int32_t qStreamInfoResetTimewindowFilter(qTaskInfo_t tinfo); void resetTaskInfo(qTaskInfo_t tinfo); diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 2e3cd670d7..49435a6317 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -168,7 +168,6 @@ typedef struct { struct SStreamFileState *pFileState; int32_t number; SSHashObj *parNameMap; - int64_t checkPointId; int32_t taskId; int64_t streamId; int64_t streamBackendRid; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index dbbe1d92dc..4ffcb616dd 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -388,6 +388,7 @@ typedef struct SLastRowScanPhysiNode { SNodeList* pGroupTags; bool groupSort; bool ignoreNull; + SNodeList* pTargets; } SLastRowScanPhysiNode; typedef SLastRowScanPhysiNode STableCountScanPhysiNode; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2e4204ab34..6c991543e3 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -241,6 +241,24 @@ typedef struct { SEpSet epset; } SDownstreamTaskEpset; +typedef enum { + TASK_SCANHISTORY_CONT = 0x1, + TASK_SCANHISTORY_QUIT = 0x2, + TASK_SCANHISTORY_REXEC = 0x3, +} EScanHistoryRet; + +typedef struct { + EScanHistoryRet ret; + int32_t idleTime; +} SScanhistoryDataInfo; + +typedef struct { + int32_t idleDuration; // idle time before use time slice the continue execute scan-history + int32_t numOfTicks; + tmr_h pTimer; + int32_t execCount; +} SScanhistorySchedInfo; + typedef struct { int64_t stbUid; char stbFullName[TSDB_TABLE_FNAME_LEN]; @@ -354,7 +372,9 @@ typedef struct STaskExecStatisInfo { int64_t init; int64_t start; int64_t step1Start; + double step1El; int64_t step2Start; + double step2El; int32_t updateCount; int64_t latestUpdateTs; int32_t processDataBlocks; @@ -378,9 +398,10 @@ typedef struct STaskOutputInfo { union { STaskDispatcherFixed fixedDispatcher; STaskDispatcherShuffle shuffleDispatcher; - STaskSinkTb tbSink; - STaskSinkSma smaSink; - STaskSinkFetch fetchSink; + + STaskSinkTb tbSink; + STaskSinkSma smaSink; + STaskSinkFetch fetchSink; }; int8_t type; STokenBucket* pTokenBucket; @@ -414,7 +435,10 @@ struct SStreamTask { SStreamState* pState; // state backend SArray* pRspMsgList; SUpstreamInfo upstreamInfo; + // the followings attributes don't be serialized + SScanhistorySchedInfo schedHistoryInfo; + int32_t notReadyTasks; int32_t numOfWaitingUpstream; int64_t checkReqId; @@ -432,8 +456,10 @@ struct SStreamTask { typedef struct STaskStartInfo { int64_t startTs; int64_t readyTs; - int32_t startAllTasksFlag; - SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing + int32_t tasksWillRestart; + int32_t taskStarting; // restart flag, sentinel to guard the restart procedure. + SHashObj* pReadyTaskSet; // tasks that are all ready for running stream processing + SHashObj* pFailedTaskSet; // tasks that are done the check downstream process, may be successful or failed int32_t elapsedTime; } STaskStartInfo; @@ -732,8 +758,6 @@ void initRpcMsg(SRpcMsg* pMsg, int32_t msgType, void* pCont, int32_t contLen) // recover and fill history void streamTaskCheckDownstream(SStreamTask* pTask); -int32_t onNormalTaskReady(SStreamTask* pTask); -int32_t onScanhistoryTaskReady(SStreamTask* pTask); int32_t streamTaskCheckStatus(SStreamTask* pTask, int32_t upstreamTaskId, int32_t vgId, int64_t stage); int32_t streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); @@ -755,7 +779,9 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs int32_t streamLaunchFillHistoryTask(SStreamTask* pTask); int32_t streamTaskScanHistoryDataComplete(SStreamTask* pTask); int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated); +int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration); bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer); + int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue); // common @@ -778,12 +804,11 @@ void streamTaskStatusCopy(STaskStatusEntry* pDst, const STaskStatusEntry* pSrc); // source level int32_t streamSetParamForStreamScannerStep1(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); int32_t streamSetParamForStreamScannerStep2(SStreamTask* pTask, SVersionRange* pVerRange, STimeWindow* pWindow); -int32_t streamScanHistoryData(SStreamTask* pTask); +SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st); int32_t streamDispatchScanHistoryFinishMsg(SStreamTask* pTask); // agg level -int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, - SRpcHandleInfo* pRpcInfo); +int32_t streamProcessScanHistoryFinishReq(SStreamTask* pTask, SStreamScanHistoryFinishReq* pReq, SRpcHandleInfo* pInfo); int32_t streamProcessScanHistoryFinishRsp(SStreamTask* pTask); // stream task meta @@ -801,19 +826,22 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); int32_t streamMetaReopen(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); +int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); void streamMetaNotifyClose(SStreamMeta* pMeta); void streamMetaStartHb(SStreamMeta* pMeta); void streamMetaInitForSnode(SStreamMeta* pMeta); bool streamMetaTaskInTimer(SStreamMeta* pMeta); -int32_t streamMetaUpdateTaskReadyInfo(SStreamTask* pTask); +int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool succ); void streamMetaRLock(SStreamMeta* pMeta); void streamMetaRUnLock(SStreamMeta* pMeta); void streamMetaWLock(SStreamMeta* pMeta); void streamMetaWUnLock(SStreamMeta* pMeta); +void streamMetaResetStartInfo(STaskStartInfo* pMeta); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask); +int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); void streamTaskClearCheckInfo(SStreamTask* pTask); int32_t streamAlignTransferState(SStreamTask* pTask); int32_t streamBuildAndSendDropTaskMsg(SMsgCb* pMsgCb, int32_t vgId, SStreamTaskId* pTaskId); diff --git a/include/os/osDef.h b/include/os/osDef.h index bad4536fa6..1a831f2e86 100644 --- a/include/os/osDef.h +++ b/include/os/osDef.h @@ -237,6 +237,12 @@ void syslog(int unused, const char *format, ...); #define TD_DIRSEP "/" #endif +#if defined(_WIN32) +#define TD_DIRSEP_CHAR '\\' +#else +#define TD_DIRSEP_CHAR '/' +#endif + #define TD_LOCALE_LEN 64 #define TD_CHARSET_LEN 64 #define TD_TIMEZONE_LEN 96 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 980d5b9177..7665550153 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -754,6 +754,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_DUP_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x2805) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR TAOS_DEF_ERROR_CODE(0, 0x2806) #define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR TAOS_DEF_ERROR_CODE(0, 0x2807) +#define TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2808) //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 34798f3816..07abdd223d 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -57,11 +57,28 @@ typedef enum { CFG_SCOPE_BOTH } ECfgScopeType; +typedef enum { + CFG_DYN_NONE = 0, + CFG_DYN_SERVER = 1, + CFG_DYN_CLIENT = 2, + CFG_DYN_BOTH = 3, +#ifdef TD_ENTERPRISE + CFG_DYN_ENT_SERVER = CFG_DYN_SERVER, + CFG_DYN_ENT_CLIENT = CFG_DYN_CLIENT, + CFG_DYN_ENT_BOTH = CFG_DYN_BOTH, +#else + CFG_DYN_ENT_SERVER = CFG_DYN_NONE, + CFG_DYN_ENT_CLIENT = CFG_DYN_NONE, + CFG_DYN_ENT_BOTH = CFG_DYN_NONE, +#endif +} ECfgDynType; + typedef struct SConfigItem { ECfgSrcType stype; ECfgDataType dtype; int8_t scope; - char *name; + int8_t dynScope; + char *name; union { bool bval; float fval; @@ -99,15 +116,20 @@ int32_t cfgGetSize(SConfig *pCfg); SConfigItem *cfgGetItem(SConfig *pCfg, const char *name); int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcType stype); -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope); -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope); -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope); -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope); +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer); + +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope); +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, + int8_t dynScope); +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, + int8_t dynScope); +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, + int8_t dynScope); +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 395a396d89..4462bd9c9d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1052,6 +1052,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue } } + pRequest->body.execMode = pQuery->execMode; switch (pQuery->execMode) { case QUERY_EXEC_MODE_LOCAL: if (!pRequest->validateOnly) { diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index c35845f9df..c7831b2b6d 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -46,6 +46,75 @@ target_link_libraries( INTERFACE api ) +if(${BUILD_S3}) + + if(${BUILD_WITH_S3}) + target_include_directories( + common + + PUBLIC "$ENV{HOME}/.cos-local.2/include" + ) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) + find_library(S3_LIBRARY s3) + find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) + find_library(XML2_LIBRARY xml2) + find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) + target_link_libraries( + common + + # s3 + PUBLIC ${S3_LIBRARY} + PUBLIC ${CURL_LIBRARY} + PUBLIC ${SSL_LIBRARY} + PUBLIC ${CRYPTO_LIBRARY} + PUBLIC ${XML2_LIBRARY} + ) + + add_definitions(-DUSE_S3) + endif() + + if(${BUILD_WITH_COS}) + + set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") + find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) + find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) + find_library(MINIXML_LIBRARY mxml) + find_library(CURL_LIBRARY curl) + target_link_libraries( + common + + # s3 + PUBLIC cos_c_sdk_static + PUBLIC ${APR_UTIL_LIBRARY} + PUBLIC ${APR_LIBRARY} + PUBLIC ${MINIXML_LIBRARY} + PUBLIC ${CURL_LIBRARY} + ) + + # s3 + FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/) + IF (APR_CONFIG_BIN) + EXECUTE_PROCESS( + COMMAND ${APR_CONFIG_BIN} --includedir + OUTPUT_VARIABLE APR_INCLUDE_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + ENDIF() + include_directories (${APR_INCLUDE_DIR}) + target_include_directories( + common + PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk" + PUBLIC "$ENV{HOME}/.cos-local.1/include" + ) + + add_definitions(-DUSE_COS) + endif(${BUILD_WITH_COS}) + +endif() + if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/vnd/vnodeCos.c b/source/common/src/cos.c similarity index 87% rename from source/dnode/vnode/src/vnd/vnodeCos.c rename to source/common/src/cos.c index 6e36739f5a..0b6b0db885 100644 --- a/source/dnode/vnode/src/vnd/vnodeCos.c +++ b/source/common/src/cos.c @@ -1,6 +1,6 @@ #define ALLOW_FORBID_FUNC -#include "vndCos.h" +#include "cos.h" extern char tsS3Endpoint[]; extern char tsS3AccessKeyId[]; @@ -13,6 +13,7 @@ extern int8_t tsS3Https; #if defined(USE_S3) #include "libs3.h" +#include "tarray.h" static int verifyPeerG = 0; static const char *awsRegionG = NULL; @@ -24,7 +25,7 @@ static S3UriStyle uriStyleG = S3UriStylePath; static int retriesG = 5; static int timeoutMsG = 0; -static int32_t s3Begin() { +int32_t s3Begin() { S3Status status; const char *hostname = tsS3Hostname; const char *env_hn = getenv("S3_HOSTNAME"); @@ -34,7 +35,7 @@ static int32_t s3Begin() { } if ((status = S3_initialize("s3", verifyPeerG | S3_INIT_ALL, hostname)) != S3StatusOK) { - vError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); + uError("Failed to initialize libs3: %s\n", S3_get_status_name(status)); return -1; } @@ -43,10 +44,12 @@ static int32_t s3Begin() { return 0; } -static void s3End() { S3_deinitialize(); } -int32_t s3Init() { return s3Begin(); } +void s3End() { S3_deinitialize(); } -void s3CleanUp() { s3End(); } +int32_t s3Init() { return 0; /*s3Begin();*/ } + +void s3CleanUp() { /*s3End();*/ +} static int should_retry() { /* @@ -65,19 +68,32 @@ static int should_retry() { static void s3PrintError(const char *func, S3Status status, char error_details[]) { if (status < S3StatusErrorAccessDenied) { - vError("%s: %s", __func__, S3_get_status_name(status)); + uError("%s: %s", __func__, S3_get_status_name(status)); } else { - vError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); + uError("%s: %s, %s", __func__, S3_get_status_name(status), error_details); } } +typedef struct { + char err_msg[512]; + S3Status status; + uint64_t content_length; + TdFilePtr file; +} TS3GetData; + typedef struct { char err_msg[128]; S3Status status; uint64_t content_length; char *buf; + int64_t buf_pos; } TS3SizeCBD; +static S3Status responsePropertiesCallbackNull(const S3ResponseProperties *properties, void *callbackData) { + // (void)callbackData; + return S3StatusOK; +} + static S3Status responsePropertiesCallback(const S3ResponseProperties *properties, void *callbackData) { //(void)callbackData; TS3SizeCBD *cbd = callbackData; @@ -97,20 +113,22 @@ static void responseCompleteCallback(S3Status status, const S3ErrorDetails *erro int len = 0; const int elen = sizeof(cbd->err_msg); if (error) { - if (error->message) { + if (error->message && elen - len > 0) { len += snprintf(&(cbd->err_msg[len]), elen - len, " Message: %s\n", error->message); } - if (error->resource) { + if (error->resource && elen - len > 0) { len += snprintf(&(cbd->err_msg[len]), elen - len, " Resource: %s\n", error->resource); } - if (error->furtherDetails) { + if (error->furtherDetails && elen - len > 0) { len += snprintf(&(cbd->err_msg[len]), elen - len, " Further Details: %s\n", error->furtherDetails); } - if (error->extraDetailsCount) { + if (error->extraDetailsCount && elen - len > 0) { len += snprintf(&(cbd->err_msg[len]), elen - len, "%s", " Extra Details:\n"); for (int i = 0; i < error->extraDetailsCount; i++) { - len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, - error->extraDetails[i].value); + if (elen - len > 0) { + len += snprintf(&(cbd->err_msg[len]), elen - len, " %s: %s\n", error->extraDetails[i].name, + error->extraDetails[i].value); + } } } } @@ -201,8 +219,9 @@ static void growbuffer_destroy(growbuffer *gb) { } typedef struct put_object_callback_data { - char err_msg[128]; + char err_msg[512]; S3Status status; + uint64_t content_length; // FILE *infile; TdFilePtr infileFD; growbuffer *gb; @@ -214,8 +233,9 @@ typedef struct put_object_callback_data { #define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct UploadManager { - char err_msg[128]; + char err_msg[512]; S3Status status; + uint64_t content_length; // used for initial multipart char *upload_id; @@ -229,8 +249,9 @@ typedef struct UploadManager { } UploadManager; typedef struct list_parts_callback_data { - char err_msg[128]; + char err_msg[512]; S3Status status; + uint64_t content_length; int isTruncated; char nextPartNumberMarker[24]; char initiatorId[256]; @@ -246,7 +267,7 @@ typedef struct list_parts_callback_data { } list_parts_callback_data; typedef struct MultipartPartData { - char err_msg[128]; + char err_msg[512]; S3Status status; put_object_callback_data put_object_data; int seq; @@ -291,7 +312,7 @@ S3Status initial_multipart_callback(const char *upload_id, void *callbackData) { } S3Status MultipartResponseProperiesCallback(const S3ResponseProperties *properties, void *callbackData) { - responsePropertiesCallback(properties, callbackData); + responsePropertiesCallbackNull(properties, callbackData); MultipartPartData *data = (MultipartPartData *)callbackData; int seq = data->seq; @@ -390,7 +411,8 @@ static int try_get_parts_info(const char *bucketName, const char *key, UploadMan S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; - S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, &listPartsCallback}; + S3ListPartsHandler listPartsHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, +&listPartsCallback}; list_parts_callback_data data; @@ -445,13 +467,13 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { data.noStatus = noStatus; if (taosStatFile(file, &contentLength, NULL, NULL) < 0) { - vError("ERROR: %s Failed to stat file %s: ", __func__, file); + uError("ERROR: %s Failed to stat file %s: ", __func__, file); code = TAOS_SYSTEM_ERROR(errno); return code; } if (!(data.infileFD = taosOpenFile(file, TD_FILE_READ))) { - vError("ERROR: %s Failed to open file %s: ", __func__, file); + uError("ERROR: %s Failed to open file %s: ", __func__, file); code = TAOS_SYSTEM_ERROR(errno); return code; } @@ -469,7 +491,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { metaProperties, useServerSideEncryption}; if (contentLength <= MULTIPART_CHUNK_SIZE) { - S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + S3PutObjectHandler putObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &putObjectDataCallback}; do { @@ -486,7 +508,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { s3PrintError(__func__, data.status, data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); } else if (data.contentLength) { - vError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, + uError("ERROR: %s Failed to read remaining %llu bytes from input", __func__, (unsigned long long)data.contentLength); code = TAOS_SYSTEM_ERROR(EIO); } @@ -506,14 +528,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { memset(&partData, 0, sizeof(MultipartPartData)); int partContentLength = 0; - S3MultipartInitialHandler handler = {{&responsePropertiesCallback, &responseCompleteCallback}, + S3MultipartInitialHandler handler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &initial_multipart_callback}; S3PutObjectHandler putObjectHandler = {{&MultipartResponseProperiesCallback, &responseCompleteCallback}, &putObjectDataCallback}; S3MultipartCommitHandler commit_handler = { - {&responsePropertiesCallback, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; + {&responsePropertiesCallbackNull, &responseCompleteCallback}, &multipartPutXmlCallback, 0}; manager.etags = (char **)taosMemoryMalloc(sizeof(char *) * totalSeq); manager.next_etags_pos = 0; @@ -609,7 +631,7 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { } typedef struct list_bucket_callback_data { - char err_msg[128]; + char err_msg[512]; S3Status status; int isTruncated; char nextMarker[1024]; @@ -658,19 +680,19 @@ static void s3FreeObjectKey(void *pItem) { taosMemoryFree(key); } -void s3DeleteObjectsByPrefix(const char *prefix) { +static SArray *getListByPrefix(const char *prefix) { S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; - S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallback, &responseCompleteCallback}, + S3ListBucketHandler listBucketHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, &listBucketCallback}; const char *marker = 0, *delimiter = 0; int maxkeys = 0, allDetails = 0; list_bucket_callback_data data; - data.objectArray = taosArrayInit(32, POINTER_BYTES); + data.objectArray = taosArrayInit(32, sizeof(void *)); if (!data.objectArray) { - vError("%s: %s", __func__, "out of memoty"); - return; + uError("%s: %s", __func__, "out of memoty"); + return NULL; } if (marker) { snprintf(data.nextMarker, sizeof(data.nextMarker), "%s", marker); @@ -693,18 +715,15 @@ void s3DeleteObjectsByPrefix(const char *prefix) { if (data.status == S3StatusOK) { if (data.keyCount > 0) { - // printListBucketHeader(allDetails); - s3DeleteObjects(TARRAY_DATA(data.objectArray), TARRAY_SIZE(data.objectArray)); + return data.objectArray; } } else { s3PrintError(__func__, data.status, data.err_msg); } - - taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); + return NULL; } void s3DeleteObjects(const char *object_name[], int nobject) { - int status = 0; S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, 0, awsRegionG}; S3ResponseHandler responseHandler = {0, &responseCompleteCallback}; @@ -721,17 +740,28 @@ void s3DeleteObjects(const char *object_name[], int nobject) { } } +void s3DeleteObjectsByPrefix(const char *prefix) { + SArray *objectArray = getListByPrefix(prefix); + if (objectArray == NULL) return; + s3DeleteObjects(TARRAY_DATA(objectArray), TARRAY_SIZE(objectArray)); + taosArrayDestroyEx(objectArray, s3FreeObjectKey); +} + static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void *callbackData) { TS3SizeCBD *cbd = callbackData; + /* if (cbd->content_length != bufferSize) { cbd->status = S3StatusAbortedByCallback; return S3StatusAbortedByCallback; } + */ + if (!cbd->buf) { + cbd->buf = taosMemoryCalloc(1, cbd->content_length); + } - char *buf = taosMemoryCalloc(1, bufferSize); - if (buf) { - memcpy(buf, buffer, bufferSize); - cbd->buf = buf; + if (cbd->buf) { + memcpy(cbd->buf + cbd->buf_pos, buffer, bufferSize); + cbd->buf_pos += bufferSize; cbd->status = S3StatusOK; return S3StatusOK; } else { @@ -740,7 +770,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, void * } } -int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { int status = 0; int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; const char *ifMatch = 0, *ifNotMatch = 0; @@ -753,12 +783,18 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, TS3SizeCBD cbd = {0}; cbd.content_length = size; + cbd.buf_pos = 0; do { S3_get_object(&bucketContext, object_name, &getConditions, offset, size, 0, 0, &getObjectHandler, &cbd); } while (S3_status_is_retryable(cbd.status) && should_retry()); if (cbd.status != S3StatusOK) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + return TAOS_SYSTEM_ERROR(EIO); + } + + if (check && cbd.buf_pos != size) { + uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); return TAOS_SYSTEM_ERROR(EIO); } @@ -767,6 +803,66 @@ int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, return 0; } +static S3Status getObjectCallback(int bufferSize, const char *buffer, void *callbackData) { + TS3GetData *cbd = (TS3GetData *)callbackData; + size_t wrote = taosWriteFile(cbd->file, buffer, bufferSize); + return ((wrote < (size_t)bufferSize) ? S3StatusAbortedByCallback : S3StatusOK); +} + +int32_t s3GetObjectToFile(const char *object_name, char *fileName) { + int64_t ifModifiedSince = -1, ifNotModifiedSince = -1; + const char *ifMatch = 0, *ifNotMatch = 0; + + S3BucketContext bucketContext = {0, tsS3BucketName, protocolG, uriStyleG, tsS3AccessKeyId, tsS3AccessKeySecret, + 0, awsRegionG}; + S3GetConditions getConditions = {ifModifiedSince, ifNotModifiedSince, ifMatch, ifNotMatch}; + S3GetObjectHandler getObjectHandler = {{&responsePropertiesCallbackNull, &responseCompleteCallback}, + &getObjectCallback}; + + TdFilePtr pFile = taosOpenFile(fileName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + uError("[s3] open file error, errno:%d, fileName:%s", errno, fileName); + return -1; + } + TS3GetData cbd = {0}; + cbd.file = pFile; + do { + S3_get_object(&bucketContext, object_name, &getConditions, 0, 0, 0, 0, &getObjectHandler, &cbd); + } while (S3_status_is_retryable(cbd.status) && should_retry()); + + if (cbd.status != S3StatusOK) { + uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + taosCloseFile(&pFile); + return TAOS_SYSTEM_ERROR(EIO); + } + + taosCloseFile(&pFile); + return 0; +} + +int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { + SArray *objectArray = getListByPrefix(prefix); + if (objectArray == NULL) return -1; + + for (size_t i = 0; i < taosArrayGetSize(objectArray); i++) { + char *object = taosArrayGetP(objectArray, i); + const char *tmp = strchr(object, '/'); + tmp = (tmp == NULL) ? object : tmp + 1; + char fileName[PATH_MAX] = {0}; + if (path[strlen(path) - 1] != TD_DIRSEP_CHAR) { + snprintf(fileName, PATH_MAX, "%s%s%s", path, TD_DIRSEP, tmp); + } else { + snprintf(fileName, PATH_MAX, "%s%s", path, tmp); + } + if (s3GetObjectToFile(object, fileName) != 0) { + taosArrayDestroyEx(objectArray, s3FreeObjectKey); + return -1; + } + } + taosArrayDestroyEx(objectArray, s3FreeObjectKey); + return 0; +} + long s3Size(const char *object_name) { long size = 0; int status = 0; @@ -782,7 +878,7 @@ long s3Size(const char *object_name) { } while (S3_status_is_retryable(cbd.status) && should_retry()); if ((cbd.status != S3StatusOK) && (cbd.status != S3StatusErrorPreconditionFailed)) { - vError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); + uError("%s: %d(%s)", __func__, cbd.status, cbd.err_msg); } size = cbd.content_length; @@ -1045,7 +1141,8 @@ bool s3Get(const char *object_name, const char *path) { return ret; } -int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, uint8_t **ppBlock) { +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t block_size, bool check, uint8_t **ppBlock) { + (void)check; int32_t code = 0; cos_pool_t *p = NULL; int is_cname = 0; @@ -1237,8 +1334,11 @@ void s3DeleteObjectsByPrefix(const char *prefix) {} void s3DeleteObjects(const char *object_name[], int nobject) {} bool s3Exists(const char *object_name) { return false; } bool s3Get(const char *object_name, const char *path) { return false; } -int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, uint8_t **ppBlock) { return 0; } +int32_t s3GetObjectBlock(const char *object_name, int64_t offset, int64_t size, bool check, uint8_t **ppBlock) { + return 0; +} void s3EvictCache(const char *path, long object_size) {} long s3Size(const char *object_name) { return 0; } +int32_t s3GetObjectsByPrefix(const char *prefix, const char *path) { return 0; } #endif diff --git a/source/common/src/rsync.c b/source/common/src/rsync.c new file mode 100644 index 0000000000..ffab85761e --- /dev/null +++ b/source/common/src/rsync.c @@ -0,0 +1,235 @@ +// +// Created by mingming wanng on 2023/11/2. +// +#include "rsync.h" +#include +#include "tglobal.h" + +#define ERRNO_ERR_FORMAT "errno:%d,msg:%s" +#define ERRNO_ERR_DATA errno,strerror(errno) + +// deleteRsync function produce empty directories, traverse base directory to remove them +static void removeEmptyDir(){ + TdDirPtr pDir = taosOpenDir(tsCheckpointBackupDir); + if (pDir == NULL) return; + + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + if (!taosDirEntryIsDir(de)) { + continue; + } + + if (strcmp(taosGetDirEntryName(de), ".") == 0 || strcmp(taosGetDirEntryName(de), "..") == 0) continue; + + char filename[PATH_MAX] = {0}; + snprintf(filename, sizeof(filename), "%s%s", tsCheckpointBackupDir, taosGetDirEntryName(de)); + + TdDirPtr pDirTmp = taosOpenDir(filename); + TdDirEntryPtr deTmp = NULL; + bool empty = true; + while ((deTmp = taosReadDir(pDirTmp)) != NULL){ + if (strcmp(taosGetDirEntryName(deTmp), ".") == 0 || strcmp(taosGetDirEntryName(deTmp), "..") == 0) continue; + empty = false; + } + if(empty) taosRemoveDir(filename); + taosCloseDir(&pDirTmp); + } + + taosCloseDir(&pDir); +} + +#ifdef WINDOWS +// C:\TDengine\data\backup\checkpoint\ -> /c/TDengine/data/backup/checkpoint/ +static void changeDirFromWindowsToLinux(char* from, char* to){ + to[0] = '/'; + to[1] = from[0]; + for(int i = 2; i < strlen(from); i++) { + if (from[i] == '\\') { + to[i] = '/'; + } else { + to[i] = from[i]; + } + } +} +#endif + +static int generateConfigFile(char* confDir){ + TdFilePtr pFile = taosOpenFile(confDir, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + uError("[rsync] open conf file error, dir:%s,"ERRNO_ERR_FORMAT, confDir, ERRNO_ERR_DATA); + return -1; + } + +#ifdef WINDOWS + char path[PATH_MAX] = {0}; + changeDirFromWindowsToLinux(tsCheckpointBackupDir, path); +#endif + + char confContent[PATH_MAX*4] = {0}; + snprintf(confContent, PATH_MAX*4, +#ifndef WINDOWS + "uid = root\n" + "gid = root\n" +#endif + "use chroot = false\n" + "max connections = 200\n" + "timeout = 100\n" + "lock file = %srsync.lock\n" + "log file = %srsync.log\n" + "ignore errors = true\n" + "read only = false\n" + "list = false\n" + "[checkpoint]\n" + "path = %s", tsCheckpointBackupDir, tsCheckpointBackupDir, +#ifdef WINDOWS + path +#else + tsCheckpointBackupDir +#endif + ); + uDebug("[rsync] conf:%s", confContent); + if (taosWriteFile(pFile, confContent, strlen(confContent)) <= 0){ + uError("[rsync] write conf file error,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + taosCloseFile(&pFile); + return -1; + } + + taosCloseFile(&pFile); + return 0; +} + +static int execCommand(char* command){ + int try = 3; + int32_t code = 0; + while(try-- > 0) { + code = system(command); + if (code == 0) { + break; + } + taosMsleep(10); + } + return code; +} + +void stopRsync(){ + int code = +#ifdef WINDOWS + system("taskkill /f /im rsync.exe"); +#else + system("pkill rsync"); +#endif + if(code != 0){ + uError("[rsync] stop rsync server failed,"ERRNO_ERR_FORMAT, ERRNO_ERR_DATA); + return; + } + uDebug("[rsync] stop rsync server successful"); +} + +void startRsync(){ + if(taosMulMkDir(tsCheckpointBackupDir) != 0){ + uError("[rsync] build checkpoint backup dir failed, dir:%s,"ERRNO_ERR_FORMAT, tsCheckpointBackupDir, ERRNO_ERR_DATA); + return; + } + removeEmptyDir(); + + char confDir[PATH_MAX] = {0}; + snprintf(confDir, PATH_MAX, "%srsync.conf", tsCheckpointBackupDir); + + int code = generateConfigFile(confDir); + if(code != 0){ + return; + } + + char cmd[PATH_MAX] = {0}; + snprintf(cmd, PATH_MAX, "rsync --daemon --port=%d --config=%s", tsRsyncPort, confDir); + // start rsync service to backup checkpoint + code = system(cmd); + if(code != 0){ + uError("[rsync] start server failed, code:%d,"ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return; + } + uDebug("[rsync] start server successful"); +} + +int uploadRsync(char* id, char* path){ +#ifdef WINDOWS + char pathTransform[PATH_MAX] = {0}; + changeDirFromWindowsToLinux(path, pathTransform); +#endif + char command[PATH_MAX] = {0}; +#ifdef WINDOWS + if(pathTransform[strlen(pathTransform) - 1] != '/'){ +#else + if(path[strlen(path) - 1] != '/'){ +#endif + snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s/ rsync://%s/checkpoint/%s/", +#ifdef WINDOWS + pathTransform +#else + path +#endif + , tsSnodeAddress, id); + }else{ + snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 --bwlimit=100000 %s rsync://%s/checkpoint/%s/", +#ifdef WINDOWS + pathTransform +#else + path +#endif + , tsSnodeAddress, id); + } + + int code = execCommand(command); + if(code != 0){ + uError("[rsync] send failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + uDebug("[rsync] upload data:%s successful", id); + return 0; +} + +int downloadRsync(char* id, char* path){ +#ifdef WINDOWS + char pathTransform[PATH_MAX] = {0}; + changeDirFromWindowsToLinux(path, pathTransform); +#endif + char command[PATH_MAX] = {0}; + snprintf(command, PATH_MAX, "rsync -av --timeout=10 --bwlimit=100000 rsync://%s/checkpoint/%s/ %s", + tsSnodeAddress, id, +#ifdef WINDOWS + pathTransform +#else + path +#endif + ); + + int code = execCommand(command); + if(code != 0){ + uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + uDebug("[rsync] down data:%s successful", id); + return 0; +} + +int deleteRsync(char* id){ + char* tmp = "./tmp_empty/"; + int code = taosMkDir(tmp); + if(code != 0){ + uError("[rsync] make tmp dir failed. code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + char command[PATH_MAX] = {0}; + snprintf(command, PATH_MAX, "rsync -av --delete --timeout=10 %s rsync://%s/checkpoint/%s/", + tmp, tsSnodeAddress, id); + + code = execCommand(command); + taosRemoveDir(tmp); + if(code != 0){ + uError("[rsync] get failed code:%d," ERRNO_ERR_FORMAT, code, ERRNO_ERR_DATA); + return -1; + } + uDebug("[rsync] delete data:%s successful", id); + + return 0; +} \ No newline at end of file diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 8e824b4d67..495aee1ece 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -44,7 +44,6 @@ int32_t tsNumOfSupportVnodes = 256; // common int32_t tsMaxShellConns = 50000; int32_t tsShellActivityTimer = 3; // second -bool tsPrintAuth = false; // queue & threads int32_t tsNumOfRpcThreads = 1; @@ -73,9 +72,6 @@ int32_t tsElectInterval = 25 * 1000; int32_t tsHeartbeatInterval = 1000; int32_t tsHeartbeatTimeout = 20 * 1000; -// vnode -int64_t tsVndCommitMaxIntervalMs = 600 * 1000; - // mnode int64_t tsMndSdbWriteDelta = 200; int64_t tsMndLogRetention = 2000; @@ -98,10 +94,8 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; // audit -bool tsEnableAudit = true; -char tsAuditFqdn[TSDB_FQDN_LEN] = {0}; -uint16_t tsAuditPort = 6043; -bool tsEnableAuditCreateTable = true; +bool tsEnableAudit = true; +bool tsEnableAuditCreateTable = true; // telem #ifdef TD_ENTERPRISE @@ -133,6 +127,15 @@ char tsSmlAutoChildTableNameDelimiter[TSDB_TABLE_NAME_LEN] = ""; // bool tsSmlDataFormat = false; // int32_t tsSmlBatchSize = 10000; +// checkpoint backup +char tsSnodeAddress[TSDB_FQDN_LEN] = {0}; +int32_t tsRsyncPort = 873; +#ifdef WINDOWS +char tsCheckpointBackupDir[PATH_MAX] = "C:\\TDengine\\data\\backup\\checkpoint\\"; +#else +char tsCheckpointBackupDir[PATH_MAX] = "/var/lib/taos/backup/checkpoint/"; +#endif + // tmq int32_t tmqMaxTopicNum = 20; // query @@ -167,22 +170,12 @@ int32_t tsTimeSeriesThreshold = 50; */ int32_t tsCompressMsgSize = -1; -/* denote if server needs to compress the retrieved column data before adding to the rpc response message body. - * 0: all data are compressed - * -1: all data are not compressed - * other values: if any retrieved column size is greater than the tsCompressColData, all data will be compressed. - */ -int32_t tsCompressColData = -1; - // count/hyperloglog function always return values in case of all NULL data or Empty data set. int32_t tsCountAlwaysReturnValue = 1; // 1 ms for sliding time, the value will changed in case of time precision changed int32_t tsMinSlidingTime = 1; -// the maxinum number of distict query result -int32_t tsMaxNumOfDistinctResults = 1000 * 10000; - // 1 database precision unit for interval time range, changed accordingly int32_t tsMinIntervalTime = 1; @@ -276,6 +269,7 @@ char tsS3AccessKeySecret[TSDB_FQDN_LEN] = ""; char tsS3BucketName[TSDB_FQDN_LEN] = ""; char tsS3AppId[TSDB_FQDN_LEN] = ""; int8_t tsS3Enabled = false; +int8_t tsS3StreamEnabled = false; int8_t tsS3Https = true; char tsS3Hostname[TSDB_FQDN_LEN] = ""; @@ -339,9 +333,10 @@ int32_t taosSetS3Cfg(SConfig *pCfg) { tstrncpy(tsS3AppId, appid + 1, TSDB_FQDN_LEN); } } - if (tsS3BucketName[0] != '<' && tsDiskCfgNum > 1) { + if (tsS3BucketName[0] != '<') { #if defined(USE_COS) || defined(USE_S3) - tsS3Enabled = true; + if (tsDiskCfgNum > 1) tsS3Enabled = true; + tsS3StreamEnabled = true; #endif } @@ -401,39 +396,40 @@ static int32_t taosLoadCfg(SConfig *pCfg, const char **envCmd, const char *input } int32_t taosAddClientLogCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "configDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddDir(pCfg, "scriptDir", configDir, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddDir(pCfg, "logDir", tsLogDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalLogDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfLogLines", tsNumOfLogLines, 1000, 2000000000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) + return -1; + if (cfgAddBool(pCfg, "asyncLog", tsAsyncLog, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "logKeepDays", 0, -365000, 365000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "debugFlag", 0, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "simDebugFlag", 143, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "tmrDebugFlag", tmrDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "uDebugFlag", uDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "rpcDebugFlag", rpcDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "jniDebugFlag", jniDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "qDebugFlag", qDebugFlag, 0, 255, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "cDebugFlag", cDebugFlag, 0, 255, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; return 0; } static int32_t taosAddServerLogCfg(SConfig *pCfg) { - if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, 0) != CFG_SCOPE_SERVER) return -1; - if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "dDebugFlag", dDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "vDebugFlag", vDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "mDebugFlag", mDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "wDebugFlag", wDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "sDebugFlag", sDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tsdbDebugFlag", tsdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tqDebugFlag", tqDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "fsDebugFlag", fsDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "udfDebugFlag", udfDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "smaDebugFlag", smaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "idxDebugFlag", idxDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tdbDebugFlag", tdbDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "metaDebugFlag", metaDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "stDebugFlag", stDebugFlag, 0, 255, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; return 0; } @@ -444,62 +440,77 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { strcpy(defaultFqdn, "localhost"); } - if (cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT) != 0) + if (cfgAddString(pCfg, "firstEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "secondEp", "", CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "fqdn", defaultFqdn, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "serverPort", defaultServerPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddDir(pCfg, "tempDir", tsTempDir, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalTmpDirGB", 1.0f, 0.001f, 10000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "shellActivityTimer", tsShellActivityTimer, 1, 120, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT) != 0) return -1; - // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, CFG_SCOPE_CLIENT) != 0) return -1; - // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, - CFG_SCOPE_CLIENT) != 0) + if (cfgAddInt32(pCfg, "compressMsgSize", tsCompressMsgSize, -1, 100000000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, CFG_SCOPE_CLIENT, CFG_DYN_ENT_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableScience", tsEnableScience, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddBool(pCfg, "queryUseNodeAllocator", tsQueryUseNodeAllocator, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddBool(pCfg, "keepColumnName", tsKeepColumnName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlChildTableName", tsSmlChildTableName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlAutoChildTableNameDelimiter", tsSmlAutoChildTableNameDelimiter, CFG_SCOPE_CLIENT, + CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddString(pCfg, "smlTagName", tsSmlTagName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddString(pCfg, "smlTsDefaultName", tsSmlTsDefaultName, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "smlDot2Underline", tsSmlDot2Underline, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + // if (cfgAddBool(pCfg, "smlDataFormat", tsSmlDataFormat, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; + // if (cfgAddInt32(pCfg, "smlBatchSize", tsSmlBatchSize, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) + // return -1; + if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "maxInsertBatchRows", tsMaxInsertBatchRows, 1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != + 0) + return -1; + if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_SERVER, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, CFG_SCOPE_CLIENT, + CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "metaCacheMaxSize", tsMetaCacheMaxSize, -1, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddString(pCfg, "slowLogScope", "", CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 100000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) + return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 10000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH) != 0) + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsTimeToGetAvailableConn, 20, 1000000, CFG_SCOPE_BOTH, + CFG_DYN_NONE) != 0) return -1; tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH, CFG_DYN_ENT_BOTH) != 0) return -1; tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); if (tsNumOfTaskQueueThreads >= 10) { tsNumOfTaskQueueThreads = 10; } - if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) + return -1; return 0; } @@ -507,98 +518,107 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { static int32_t taosAddSystemCfg(SConfig *pCfg) { SysNameInfo info = taosGetSysNameInfo(); - if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "assert", tsAssert, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddTimezone(pCfg, "timezone", tsTimezoneStr, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddLocale(pCfg, "locale", tsLocale, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddCharset(pCfg, "charset", tsCharset, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "assert", tsAssert, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddBool(pCfg, "enableCoreFile", 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) return -1; + if (cfgAddFloat(pCfg, "numOfCores", tsNumOfCores, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddBool(pCfg, "ssd42", tsSSE42Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "avx", tsAVXEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "avx2", tsAVX2Enable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "fma", tsFMAEnable, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "simdEnable", tsSIMDBuiltins, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "tagFilterCache", tsTagFilterCache, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt64(pCfg, "openMax", tsOpenMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; #if !defined(_ALPINE) - if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt64(pCfg, "streamMax", tsStreamMax, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; #endif - if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "pageSizeKB", tsPageSizeKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt64(pCfg, "totalMemoryKB", tsTotalMemoryKB, 0, INT64_MAX, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os sysname", info.sysname, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os nodename", info.nodename, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os release", info.release, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os version", info.version, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "os machine", info.machine, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddString(pCfg, "version", version, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "compatible_version", compatible_version, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "gitinfo", gitinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "buildinfo", buildinfo, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; return 0; } static int32_t taosAddServerCfg(SConfig *pCfg) { - if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddDir(pCfg, "dataDir", tsDataDir, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "minimalDataDirGB", 2.0f, 0.001f, 10000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; tsNumOfSupportVnodes = tsNumOfCores * 2; tsNumOfSupportVnodes = TMAX(tsNumOfSupportVnodes, 2); - if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "supportVnodes", tsNumOfSupportVnodes, 0, 4096, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "maxShellConns", tsMaxShellConns, 10, 50000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "maxNumOfDistinctRes", tsMaxNumOfDistinctResults, 10 * 10000, 10000 * 10000, - CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "statusInterval", tsStatusInterval, 1, 30, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "minSlidingTime", tsMinSlidingTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1; - if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "printAuth", tsPrintAuth, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) + return -1; + + if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0) + return -1; + if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); - if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcThreads", tsNumOfRpcThreads, 1, 1024, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; tsNumOfRpcSessions = TRANGE(tsNumOfRpcSessions, 100, 10000); - if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfRpcSessions", tsNumOfRpcSessions, 1, 100000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) + return -1; tsTimeToGetAvailableConn = TRANGE(tsTimeToGetAvailableConn, 20, 1000000); - if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH) != 0) return -1; - - tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 72000); - if (cfgAddInt32(pCfg, "keepAliveIdle", tsKeepAliveIdle, 1, 7200000, CFG_SCOPE_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "timeToGetAvailableConn", tsNumOfRpcSessions, 20, 1000000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) + return -1; tsNumOfCommitThreads = tsNumOfCores / 2; tsNumOfCommitThreads = TRANGE(tsNumOfCommitThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfCommitThreads", tsNumOfCommitThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; tsNumOfMnodeReadThreads = tsNumOfCores / 8; tsNumOfMnodeReadThreads = TRANGE(tsNumOfMnodeReadThreads, 1, 4); - if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfMnodeReadThreads", tsNumOfMnodeReadThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; tsNumOfVnodeQueryThreads = tsNumOfCores * 2; tsNumOfVnodeQueryThreads = TMAX(tsNumOfVnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeQueryThreads", tsNumOfVnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; - if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER) != 0) + if (cfgAddFloat(pCfg, "ratioOfVnodeStreamThreads", tsRatioOfVnodeStreamThreads, 0.01, 100, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) return -1; tsNumOfVnodeFetchThreads = tsNumOfCores / 4; tsNumOfVnodeFetchThreads = TMAX(tsNumOfVnodeFetchThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeFetchThreads", tsNumOfVnodeFetchThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; - tsNumOfVnodeRsmaThreads = tsNumOfCores; + tsNumOfVnodeRsmaThreads = tsNumOfCores / 2; tsNumOfVnodeRsmaThreads = TMAX(tsNumOfVnodeRsmaThreads, 4); - if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfVnodeRsmaThreads", tsNumOfVnodeRsmaThreads, 1, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; tsNumOfQnodeQueryThreads = tsNumOfCores * 2; tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); - if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 4, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; // tsNumOfQnodeFetchThreads = tsNumOfCores / 2; // tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); @@ -606,108 +626,146 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfSnodeStreamThreads = tsNumOfCores / 4; tsNumOfSnodeStreamThreads = TRANGE(tsNumOfSnodeStreamThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "numOfSnodeSharedThreads", tsNumOfSnodeStreamThreads, 2, 1024, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) return -1; tsNumOfSnodeWriteThreads = tsNumOfCores / 4; tsNumOfSnodeWriteThreads = TRANGE(tsNumOfSnodeWriteThreads, 2, 4); - if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "numOfSnodeUniqueThreads", tsNumOfSnodeWriteThreads, 2, 1024, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) + return -1; tsRpcQueueMemoryAllowed = tsTotalMemoryKB * 1024 * 0.1; tsRpcQueueMemoryAllowed = TRANGE(tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10LL, TSDB_MAX_MSG_SIZE * 10000LL); if (cfgAddInt64(pCfg, "rpcQueueMemoryAllowed", tsRpcQueueMemoryAllowed, TSDB_MAX_MSG_SIZE * 10L, INT64_MAX, - CFG_SCOPE_BOTH) != 0) + CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "syncElectInterval", tsElectInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, CFG_DYN_NONE) != + 0) return -1; - if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "syncHeartbeatInterval", tsHeartbeatInterval, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "syncHeartbeatTimeout", tsHeartbeatTimeout, 10, 1000 * 60 * 24 * 2, CFG_SCOPE_SERVER, + CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "vndCommitMaxInterval", tsVndCommitMaxIntervalMs, 1000, 1000 * 60 * 60, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH, CFG_DYN_BOTH) != 0) return -1; + if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1; + + if (cfgAddInt32(pCfg, "rsyncPort", tsRsyncPort, 1, 65535, CFG_SCOPE_BOTH, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "snodeAddress", tsSnodeAddress, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "checkpointBackupDir", tsCheckpointBackupDir, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "mndSdbWriteDelta", tsMndSdbWriteDelta, 20, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "mndLogRetention", tsMndLogRetention, 500, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "grantMode", tsMndGrantMode, 0, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "skipGrant", tsMndSkipGrant, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; - if (cfgAddBool(pCfg, "monitor", tsEnableMonitor, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "monitorFqdn", tsMonitorFqdn, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorPort", tsMonitorPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "monitorMaxLogs", tsMonitorMaxLogs, 1, 1000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "monitorComp", tsMonitorComp, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddBool(pCfg, "audit", tsEnableAudit, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "auditFqdn", tsAuditFqdn, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "auditPort", tsAuditPort, 1, 65056, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "auditCreateTable", tsEnableAuditCreateTable, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddBool(pCfg, "telemetryReporting", tsEnableTelem, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryInterval", tsTelemInterval, 1, 200000, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddString(pCfg, "telemetryServer", tsTelemServer, CFG_SCOPE_BOTH) != 0) return -1; - if (cfgAddInt32(pCfg, "telemetryPort", tsTelemPort, 1, 65056, CFG_SCOPE_BOTH) != 0) return -1; - - if (cfgAddInt32(pCfg, "tmqMaxTopicNum", tmqMaxTopicNum, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; - - if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "ttlBatchDropNum", tsTtlBatchDropNum, 0, INT32_MAX, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddBool(pCfg, "ttlChangeOnWrite", tsTtlChangeOnWrite, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlFlushThreshold", tsTtlFlushThreshold, -1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "trimVDbIntervalSec", tsTrimVDbIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "uptimeInterval", tsUptimeInterval, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "queryRsmaTolerance", tsQueryRsmaTolerance, 0, 900000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "timeseriesThreshold", tsTimeSeriesThreshold, 0, 2000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; if (cfgAddInt64(pCfg, "walFsyncDataSizeLimit", tsWalFsyncDataSizeLimit, 100 * 1024 * 1024, INT64_MAX, - CFG_SCOPE_SERVER) != 0) + CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "udf", tsStartUdfd, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "udfdResFuncs", tsUdfdResFuncs, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "udfdLdLibPath", tsUdfdLdLibPath, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER) != 0) + if (cfgAddInt32(pCfg, "cacheLazyLoadThreshold", tsCacheLazyLoadThreshold, 0, 100000, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddString(pCfg, "lossyColumns", tsLossyColumns, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "fPrecision", tsFPrecision, 0.0f, 100000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddFloat(pCfg, "dPrecision", tsDPrecision, 0.0f, 1000000.0f, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "maxRange", tsMaxRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "curRange", tsCurRange, 0, 65536, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddBool(pCfg, "ifAdtFse", tsIfAdtFse, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "compressor", tsCompressor, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, 0) != 0) return -1; + if (cfgAddBool(pCfg, "filterScalarMode", tsFilterScalarMode, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "maxStreamBackendCache", tsMaxStreamBackendCache, 16, 1024, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "pqSortMemThreshold", tsPQSortMemThreshold, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; + if (cfgAddInt32(pCfg, "resolveFQDNRetryTime", tsResolveFQDNRetryTime, 1, 10240, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) + return -1; - if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -100, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER) != 0) return -1; - if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER) != 0) + if (cfgAddString(pCfg, "s3Accesskey", tsS3AccessKey, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "s3Endpoint", tsS3Endpoint, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; + if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != + 0) + return -1; + if (cfgAddInt32(pCfg, "s3PageCacheSize", tsS3PageCacheSize, 4, 1024 * 1024 * 1024, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) + return -1; + if (cfgAddInt32(pCfg, "s3UploadDelaySec", tsS3UploadDelaySec, 60 * 10, 60 * 60 * 24 * 30, CFG_SCOPE_SERVER, + CFG_DYN_ENT_SERVER) != 0) return -1; // min free disk space used to check if the disk is full [50MB, 1GB] if (cfgAddInt64(pCfg, "minDiskFreeSize", tsMinDiskFreeSize, TFS_MIN_DISK_FREE_SIZE, 1024 * 1024 * 1024, - CFG_SCOPE_SERVER) != 0) + CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; - if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER) != 0) return -1; + if (cfgAddBool(pCfg, "enableWhiteList", tsEnableWhiteList, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; GRANT_CFG_ADD; return 0; @@ -757,13 +815,6 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } - pItem = cfgGetItem(tsCfg, "keepAliveIdle"); - if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { - tsKeepAliveIdle = TRANGE(tsKeepAliveIdle, 1, 720000); - pItem->i32 = tsKeepAliveIdle; - pItem->stype = stype; - } - pItem = cfgGetItem(tsCfg, "numOfCommitThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { tsNumOfCommitThreads = numOfCores / 2; @@ -932,6 +983,7 @@ static int32_t taosSetSlowLogScope(char *pScope) { return -1; } +// for common configs static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; @@ -973,7 +1025,6 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; - tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; @@ -1023,6 +1074,7 @@ static void taosSetSystemCfg(SConfig *pCfg) { tsVersion = 30000000; } +// for server configs static int32_t taosSetServerCfg(SConfig *pCfg) { tsDataSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalDataDirGB")->fval) * 1024 * 1024 * 1024); tsNumOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; @@ -1030,17 +1082,13 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; - tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; - tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; tsTimeToGetAvailableConn = cfgGetItem(pCfg, "timeToGetAvailableConn")->i32; - tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; - tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; @@ -1065,8 +1113,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsQueryRspPolicy = cfgGetItem(pCfg, "queryRspPolicy")->i32; tsEnableAudit = cfgGetItem(pCfg, "audit")->bval; - tstrncpy(tsAuditFqdn, cfgGetItem(pCfg, "auditFqdn")->str, TSDB_FQDN_LEN); - tsAuditPort = (uint16_t)cfgGetItem(pCfg, "auditPort")->i32; tsEnableAuditCreateTable = cfgGetItem(pCfg, "auditCreateTable")->bval; tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; @@ -1074,7 +1120,10 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval; tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32; tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; + tsRsyncPort = cfgGetItem(pCfg, "rsyncPort")->i32; tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); + tstrncpy(tsSnodeAddress, cfgGetItem(pCfg, "snodeAddress")->str, TSDB_FQDN_LEN); + tstrncpy(tsCheckpointBackupDir, cfgGetItem(pCfg, "checkpointBackupDir")->str, PATH_MAX); tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32; @@ -1095,8 +1144,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsHeartbeatInterval = cfgGetItem(pCfg, "syncHeartbeatInterval")->i32; tsHeartbeatTimeout = cfgGetItem(pCfg, "syncHeartbeatTimeout")->i32; - tsVndCommitMaxIntervalMs = cfgGetItem(pCfg, "vndCommitMaxInterval")->i64; - tsMndSdbWriteDelta = cfgGetItem(pCfg, "mndSdbWriteDelta")->i64; tsMndLogRetention = cfgGetItem(pCfg, "mndLogRetention")->i64; tsMndSkipGrant = cfgGetItem(pCfg, "skipGrant")->bval; @@ -1146,24 +1193,11 @@ static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } int32_t taosSetReleaseCfg(SConfig *pCfg); #endif -void taosLocalCfgForbiddenToChange(char *name, bool *forbidden) { - int32_t len = strlen(name); - char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; - strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - - if (strcasecmp("charset", name) == 0) { - *forbidden = true; - return; - } - GRANT_CFG_CHECK; - - *forbidden = false; -} - int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { int32_t len = strlen(name); char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); + bool matchItem = true; switch (lowcaseName[0]) { case 'a': { @@ -1171,25 +1205,22 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; } else if (strcasecmp("assert", name) == 0) { tsAssert = cfgGetItem(pCfg, "assert")->bval; + } else { + matchItem = false; } break; } case 'c': { - if (strcasecmp("charset", name) == 0) { - const char *locale = cfgGetItem(pCfg, "locale")->str; - const char *charset = cfgGetItem(pCfg, "charset")->str; - taosSetSystemLocale(locale, charset); - osSetSystemLocale(locale, charset); - } else if (strcasecmp("compressMsgSize", name) == 0) { + if (strcasecmp("compressMsgSize", name) == 0) { tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; - } else if (strcasecmp("compressColData", name) == 0) { - tsCompressColData = cfgGetItem(pCfg, "compressColData")->i32; } else if (strcasecmp("countAlwaysReturnValue", name) == 0) { tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; } else if (strcasecmp("cDebugFlag", name) == 0) { cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; } else if (strcasecmp("crashReporting", name) == 0) { tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; + } else { + matchItem = false; } break; } @@ -1199,6 +1230,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { } else if (strcasecmp("debugFlag", name) == 0) { int32_t flag = cfgGetItem(pCfg, "debugFlag")->i32; taosSetAllDebugFlag(flag, true); + } else { + matchItem = false; } break; } @@ -1208,8 +1241,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { taosSetCoreDump(enableCore); } else if (strcasecmp("enableQueryHb", name) == 0) { tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; - } else if (strcasecmp("ttlChangeOnWrite", name) == 0) { - tsTtlChangeOnWrite = cfgGetItem(pCfg, "ttlChangeOnWrite")->bval; + } else { + matchItem = false; } break; } @@ -1242,24 +1275,34 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); } else if (strcasecmp("fsDebugFlag", name) == 0) { fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'i': { if (strcasecmp("idxDebugFlag", name) == 0) { idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'j': { if (strcasecmp("jniDebugFlag", name) == 0) { jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'k': { if (strcasecmp("keepColumnName", name) == 0) { tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; + } else if (strcasecmp("keepAliveIdle", name) == 0) { + tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; + } else { + matchItem = false; } break; } @@ -1274,32 +1317,28 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); } else if (strcasecmp("logKeepDays", name) == 0) { tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; + } else { + matchItem = false; } break; } case 'm': { switch (lowcaseName[1]) { case 'a': { - if (strcasecmp("maxShellConns", name) == 0) { - tsMaxShellConns = cfgGetItem(pCfg, "maxShellConns")->i32; - } else if (strcasecmp("maxNumOfDistinctRes", name) == 0) { - tsMaxNumOfDistinctResults = cfgGetItem(pCfg, "maxNumOfDistinctRes")->i32; - } else if (strcasecmp("maxMemUsedByInsert", name) == 0) { + if (strcasecmp("maxInsertBatchRows", name) == 0) { tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; } else if (strcasecmp("maxRetryWaitTime", name) == 0) { tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; - } - break; - } - case 'd': { - if (strcasecmp("mDebugFlag", name) == 0) { - mDebugFlag = cfgGetItem(pCfg, "mDebugFlag")->i32; + } else { + matchItem = false; } break; } case 'e': { if (strcasecmp("metaCacheMaxSize", name) == 0) { atomic_store_32(&tsMetaCacheMaxSize, cfgGetItem(pCfg, "metaCacheMaxSize")->i32); + } else { + matchItem = false; } break; } @@ -1314,34 +1353,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; } else if (strcasecmp("minimalLogDirGB", name) == 0) { tsLogSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalLogDirGB")->fval) * 1024 * 1024 * 1024); - } - break; - } - case 'o': { - if (strcasecmp("monitor", name) == 0) { - tsEnableMonitor = cfgGetItem(pCfg, "monitor")->bval; - } else if (strcasecmp("monitorInterval", name) == 0) { - tsMonitorInterval = cfgGetItem(pCfg, "monitorInterval")->i32; - } else if (strcasecmp("monitorFqdn", name) == 0) { - tstrncpy(tsMonitorFqdn, cfgGetItem(pCfg, "monitorFqdn")->str, TSDB_FQDN_LEN); - } else if (strcasecmp("monitorPort", name) == 0) { - tsMonitorPort = (uint16_t)cfgGetItem(pCfg, "monitorPort")->i32; - } else if (strcasecmp("monitorMaxLogs", name) == 0) { - tsMonitorMaxLogs = cfgGetItem(pCfg, "monitorMaxLogs")->i32; - } else if (strcasecmp("monitorComp", name) == 0) { - tsMonitorComp = cfgGetItem(pCfg, "monitorComp")->bval; - } - break; - } - case 'q': { - if (strcasecmp("mqRebalanceInterval", name) == 0) { - tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; - } - break; - } - case 'u': { - if (strcasecmp("udfDebugFlag", name) == 0) { - udfDebugFlag = cfgGetItem(pCfg, "udfDebugFlag")->i32; + } else { + matchItem = false; } break; } @@ -1352,55 +1365,18 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { break; } case 'n': { - if (strcasecmp("numOfTaskQueueThreads", name) == 0) { - tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; - } else if (strcasecmp("numOfRpcThreads", name) == 0) { - tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32; - } else if (strcasecmp("numOfRpcSessions", name) == 0) { - tsNumOfRpcSessions = cfgGetItem(pCfg, "numOfRpcSessions")->i32; - } else if (strcasecmp("numOfCommitThreads", name) == 0) { - tsNumOfCommitThreads = cfgGetItem(pCfg, "numOfCommitThreads")->i32; - } else if (strcasecmp("numOfMnodeReadThreads", name) == 0) { - tsNumOfMnodeReadThreads = cfgGetItem(pCfg, "numOfMnodeReadThreads")->i32; - } else if (strcasecmp("numOfVnodeQueryThreads", name) == 0) { - tsNumOfVnodeQueryThreads = cfgGetItem(pCfg, "numOfVnodeQueryThreads")->i32; - /* - } else if (strcasecmp("numOfVnodeFetchThreads", name) == 0) { - tsNumOfVnodeFetchThreads = cfgGetItem(pCfg, "numOfVnodeFetchThreads")->i32; - */ - } else if (strcasecmp("numOfVnodeRsmaThreads", name) == 0) { - tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; - } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { - tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; - /* - } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { - tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; - */ - } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { - tsNumOfSnodeStreamThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; - } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { - tsNumOfSnodeWriteThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; - } else if (strcasecmp("numOfLogLines", name) == 0) { + if (strcasecmp("numOfLogLines", name) == 0) { tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; - } - break; - } - case 'p': { - if (strcasecmp("printAuth", name) == 0) { - tsPrintAuth = cfgGetItem(pCfg, "printAuth")->bval; + } else { + matchItem = false; } break; } case 'q': { - if (strcasecmp("queryPolicy", name) == 0) { - tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; - } else if (strcasecmp("querySmaOptimize", name) == 0) { + if (strcasecmp("querySmaOptimize", name) == 0) { tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; - } else if (strcasecmp("queryBufferSize", name) == 0) { - tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32; - if (tsQueryBufferSize >= 0) { - tsQueryBufferSizeBytes = tsQueryBufferSize * 1048576UL; - } + } else if (strcasecmp("queryPolicy", name) == 0) { + tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; } else if (strcasecmp("qDebugFlag", name) == 0) { qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; } else if (strcasecmp("queryPlannerTrace", name) == 0) { @@ -1409,16 +1385,16 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; } else if (strcasecmp("queryUseNodeAllocator", name) == 0) { tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; - } else if (strcasecmp("queryRsmaTolerance", name) == 0) { - tsQueryRsmaTolerance = cfgGetItem(pCfg, "queryRsmaTolerance")->i32; + } else { + matchItem = false; } break; } case 'r': { - if (strcasecmp("rpcQueueMemoryAllowed", name) == 0) { - tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; - } else if (strcasecmp("rpcDebugFlag", name) == 0) { + if (strcasecmp("rpcDebugFlag", name) == 0) { rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; + } else { + matchItem = false; } break; } @@ -1446,10 +1422,6 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { tsSmlDot2Underline = cfgGetItem(pCfg, "smlDot2Underline")->bval; } else if (strcasecmp("shellActivityTimer", name) == 0) { tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; - } else if (strcasecmp("supportVnodes", name) == 0) { - tsNumOfSupportVnodes = cfgGetItem(pCfg, "supportVnodes")->i32; - } else if (strcasecmp("statusInterval", name) == 0) { - tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32; } else if (strcasecmp("serverPort", name) == 0) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; @@ -1463,8 +1435,6 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); - } else if (strcasecmp("sDebugFlag", name) == 0) { - sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; } else if (strcasecmp("smaDebugFlag", name) == 0) { smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; } else if (strcasecmp("slowLogThreshold", name) == 0) { @@ -1473,6 +1443,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { return -1; } + } else { + matchItem = false; } break; } @@ -1489,58 +1461,22 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); return -1; } - } else if (strcasecmp("tdbDebugFlag", name) == 0) { - tdbDebugFlag = cfgGetItem(pCfg, "tdbDebugFlag")->i32; - } else if (strcasecmp("telemetryReporting", name) == 0) { - tsEnableTelem = cfgGetItem(pCfg, "telemetryReporting")->bval; - } else if (strcasecmp("telemetryInterval", name) == 0) { - tsTelemInterval = cfgGetItem(pCfg, "telemetryInterval")->i32; } else if (strcasecmp("telemetryServer", name) == 0) { tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); - } else if (strcasecmp("telemetryPort", name) == 0) { - tsTelemPort = (uint16_t)cfgGetItem(pCfg, "telemetryPort")->i32; - } else if (strcasecmp("transPullupInterval", name) == 0) { - tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; - } else if (strcasecmp("ttlUnit", name) == 0) { - tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; - } else if (strcasecmp("ttlPushInterval", name) == 0) { - tsTtlPushIntervalSec = cfgGetItem(pCfg, "ttlPushInterval")->i32; - } else if (strcasecmp("ttlBatchDropNum", name) == 0) { - tsTtlBatchDropNum = cfgGetItem(pCfg, "ttlBatchDropNum")->i32; - } else if (strcasecmp("trimVDbIntervalSec", name) == 0) { - tsTrimVDbIntervalSec = cfgGetItem(pCfg, "trimVDbIntervalSec")->i32; } else if (strcasecmp("tmrDebugFlag", name) == 0) { tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; - } else if (strcasecmp("tsdbDebugFlag", name) == 0) { - tsdbDebugFlag = cfgGetItem(pCfg, "tsdbDebugFlag")->i32; - } else if (strcasecmp("tqDebugFlag", name) == 0) { - tqDebugFlag = cfgGetItem(pCfg, "tqDebugFlag")->i32; - } else if (strcasecmp("ttlFlushThreshold", name) == 0) { - tsTtlFlushThreshold = cfgGetItem(pCfg, "ttlFlushThreshold")->i32; - } else if (strcasecmp("timeseriesThreshold", name) == 0) { - tsTimeSeriesThreshold = cfgGetItem(pCfg, "timeseriesThreshold")->i32; + } else { + matchItem = false; } break; } case 'u': { - if (strcasecmp("udf", name) == 0) { - tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; - } else if (strcasecmp("uDebugFlag", name) == 0) { + if (strcasecmp("uDebugFlag", name) == 0) { uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; } else if (strcasecmp("useAdapter", name) == 0) { tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; - } - break; - } - case 'v': { - if (strcasecmp("vDebugFlag", name) == 0) { - vDebugFlag = cfgGetItem(pCfg, "vDebugFlag")->i32; - } - break; - } - case 'w': { - if (strcasecmp("wDebugFlag", name) == 0) { - wDebugFlag = cfgGetItem(pCfg, "wDebugFlag")->i32; + } else { + matchItem = false; } break; } @@ -1549,7 +1485,8 @@ int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { return -1; } - return 0; + if (!matchItem) terrno = TSDB_CODE_CFG_NOT_FOUND; + return matchItem ? 0 : -1; } int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, @@ -1689,6 +1626,10 @@ void taosCleanupCfg() { tsCfg = NULL; } } +typedef struct { + const char *optionName; + void *optionVar; +} OptionNameAndVar; void taosCfgDynamicOptions(const char *option, const char *value) { if (strncasecmp(option, "debugFlag", 9) == 0) { @@ -1703,95 +1644,107 @@ void taosCfgDynamicOptions(const char *option, const char *value) { return; } - if (strcasecmp(option, "monitor") == 0) { - int32_t monitor = atoi(value); - uInfo("monitor set from %d to %d", tsEnableMonitor, monitor); - tsEnableMonitor = monitor; - SConfigItem *pItem = cfgGetItem(tsCfg, "monitor"); - if (pItem != NULL) { - pItem->bval = tsEnableMonitor; + { // 'bool/int32_t/int64_t' variables with general modification function + const int32_t nDebugFlag = 20; + static OptionNameAndVar options[] = { + {"dDebugFlag", &dDebugFlag}, + {"vDebugFlag", &vDebugFlag}, + {"mDebugFlag", &mDebugFlag}, + {"wDebugFlag", &wDebugFlag}, + {"sDebugFlag", &sDebugFlag}, + {"tsdbDebugFlag", &tsdbDebugFlag}, + {"tqDebugFlag", &tqDebugFlag}, + {"fsDebugFlag", &fsDebugFlag}, + {"udfDebugFlag", &udfDebugFlag}, + {"smaDebugFlag", &smaDebugFlag}, + {"idxDebugFlag", &idxDebugFlag}, + {"tdbDebugFlag", &tdbDebugFlag}, + {"tmrDebugFlag", &tmrDebugFlag}, + {"uDebugFlag", &uDebugFlag}, + {"smaDebugFlag", &smaDebugFlag}, + {"rpcDebugFlag", &rpcDebugFlag}, + {"qDebugFlag", &qDebugFlag}, + {"metaDebugFlag", &metaDebugFlag}, + {"jniDebugFlag", &jniDebugFlag}, + {"stDebugFlag", &stDebugFlag}, + + {"audit", &tsEnableAudit}, + {"asynclog", &tsAsyncLog}, + {"disableStream", &tsDisableStream}, + {"enableWhiteList", &tsEnableWhiteList}, + {"telemetryReporting", &tsEnableTelem}, + {"monitor", &tsEnableMonitor}, + + {"mndSdbWriteDelta", &tsMndSdbWriteDelta}, + {"minDiskFreeSize", &tsMinDiskFreeSize}, + + {"cacheLazyLoadThreshold", &tsCacheLazyLoadThreshold}, + {"checkpointInterval", &tsStreamCheckpointInterval}, + {"keepAliveIdle", &tsKeepAliveIdle}, + {"logKeepDays", &tsLogKeepDays}, + {"maxStreamBackendCache", &tsMaxStreamBackendCache}, + {"mqRebalanceInterval", &tsMqRebalanceInterval}, + {"numOfLogLines", &tsNumOfLogLines}, + {"queryRspPolicy", &tsQueryRspPolicy}, + {"timeseriesThreshold", &tsTimeSeriesThreshold}, + {"tmqMaxTopicNum", &tmqMaxTopicNum}, + {"transPullupInterval", &tsTransPullupInterval}, + {"trimVDbIntervalSec", &tsTrimVDbIntervalSec}, + {"ttlBatchDropNum", &tsTtlBatchDropNum}, + {"ttlFlushThreshold", &tsTtlFlushThreshold}, + {"ttlPushInterval", &tsTtlPushIntervalSec}, + //{"s3BlockSize", &tsS3BlockSize}, + {"s3BlockCacheSize", &tsS3BlockCacheSize}, + {"s3PageCacheSize", &tsS3PageCacheSize}, + {"s3UploadDelaySec", &tsS3UploadDelaySec}, + {"supportVnodes", &tsNumOfSupportVnodes}, + }; + + int32_t optionSize = tListLen(options); + for (int32_t d = 0; d < optionSize; ++d) { + const char *optName = options[d].optionName; + int32_t optLen = strlen(optName); + if (strncasecmp(option, optName, optLen) != 0) continue; + + SConfig *pCfg = taosGetCfg(); + SConfigItem *pItem = NULL; + + pItem = cfgGetItem(pCfg, optName); + if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { + uError("failed to config:%s, not support", optName); + break; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + int32_t flag = atoi(value); + bool *pVar = options[d].optionVar; + uInfo("%s set from %d to %d", optName, *pVar, flag); + *pVar = flag; + } break; + case CFG_DTYPE_INT32: { + int32_t flag = atoi(value); + int32_t *pVar = options[d].optionVar; + uInfo("%s set from %d to %d", optName, *pVar, flag); + *pVar = flag; + + if (d < nDebugFlag) { + // debug flags + taosSetDebugFlag(options[d].optionVar, optName, flag, true); + } + } break; + case CFG_DTYPE_INT64: { + int64_t flag = atoll(value); + int64_t *pVar = options[d].optionVar; + uInfo("%s set from %" PRId64 " to %" PRId64, optName, *pVar, flag); + *pVar = flag; + } break; + default: + break; + } + + return; } - return; - } - /* cannot alter s3BlockSize - if (strcasecmp(option, "s3BlockSize") == 0) { - int32_t newS3BlockSize = atoi(value); - uInfo("s3BlockSize set from %d to %d", tsS3BlockSize, newS3BlockSize); - tsS3BlockSize = newS3BlockSize; - return; - } - */ - if (strcasecmp(option, "s3BlockCacheSize") == 0) { - int32_t newS3BlockCacheSize = atoi(value); - uInfo("s3BlockCacheSize set from %d to %d", tsS3BlockCacheSize, newS3BlockCacheSize); - tsS3BlockCacheSize = newS3BlockCacheSize; - return; - } - - if (strcasecmp(option, "s3PageCacheSize") == 0) { - int32_t newS3PageCacheSize = atoi(value); - uInfo("s3PageCacheSize set from %d to %d", tsS3PageCacheSize, newS3PageCacheSize); - tsS3PageCacheSize = newS3PageCacheSize; - return; - } - - if (strcasecmp(option, "s3UploadDelaySec") == 0) { - int32_t newS3UploadDelaysec = atoi(value); - uInfo("s3UploadDelaySec set from %d to %d", tsS3UploadDelaySec, newS3UploadDelaysec); - tsS3UploadDelaySec = newS3UploadDelaysec; - return; - } - - if (strcasecmp(option, "ttlPushInterval") == 0) { - int32_t newTtlPushInterval = atoi(value); - uInfo("ttlPushInterval set from %d to %d", tsTtlPushIntervalSec, newTtlPushInterval); - tsTtlPushIntervalSec = newTtlPushInterval; - return; - } - - if (strcasecmp(option, "ttlBatchDropNum") == 0) { - int32_t newTtlBatchDropNum = atoi(value); - uInfo("ttlBatchDropNum set from %d to %d", tsTtlBatchDropNum, newTtlBatchDropNum); - tsTtlBatchDropNum = newTtlBatchDropNum; - return; - } - - if (strcasecmp(option, "supportVnodes") == 0) { - int32_t newSupportVnodes = atoi(value); - uInfo("supportVnodes set from %d to %d", tsNumOfSupportVnodes, newSupportVnodes); - tsNumOfSupportVnodes = newSupportVnodes; - return; - } - - if (strcasecmp(option, "asynclog") == 0) { - int32_t newAsynclog = atoi(value); - uInfo("asynclog set from %d to %d", tsAsyncLog, newAsynclog); - tsAsyncLog = newAsynclog; - return; - } - - const char *options[] = { - "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", "tqDebugFlag", - "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag", "tmrDebugFlag", "uDebugFlag", - "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", "metaDebugFlag", "jniDebugFlag", "stDebugFlag", - }; - int32_t *optionVars[] = { - &dDebugFlag, &vDebugFlag, &mDebugFlag, &wDebugFlag, &sDebugFlag, &tsdbDebugFlag, &tqDebugFlag, - &fsDebugFlag, &udfDebugFlag, &smaDebugFlag, &idxDebugFlag, &tdbDebugFlag, &tmrDebugFlag, &uDebugFlag, - &smaDebugFlag, &rpcDebugFlag, &qDebugFlag, &metaDebugFlag, &jniDebugFlag, &stDebugFlag, - }; - - int32_t optionSize = tListLen(options); - for (int32_t d = 0; d < optionSize; ++d) { - const char *optName = options[d]; - int32_t optLen = strlen(optName); - if (strncasecmp(option, optName, optLen) != 0) continue; - - int32_t flag = atoi(value); - uInfo("%s set from %d to %d", optName, *optionVars[d], flag); - *optionVars[d] = flag; - taosSetDebugFlag(optionVars[d], optName, flag, true); - return; } uError("failed to cfg dynamic option:%s value:%s", option, value); diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 4b0848e5e9..a701c88a24 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -1580,6 +1580,7 @@ static bool needMoreDigits(SArray* formats, int32_t curIdx) { /// @retval 0 for success /// @retval -1 for format and s mismatch error /// @retval -2 if datetime err, like 2023-13-32 25:61:69 +/// @retval -3 if not supported static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t precision, const char** sErrPos, int32_t* fErrIdx) { int32_t size = taosArrayGetSize(formats); @@ -1589,6 +1590,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec int32_t hour = 0, min = 0, sec = 0, us = 0, ms = 0, ns = 0; int32_t tzSign = 1, tz = tsTimezone; int32_t err = 0; + bool withYD = false, withMD = false; for (int32_t i = 0; i < size && *s != '\0'; ++i) { while (isspace(*s) && *s != '\0') { @@ -1782,6 +1784,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec } else { s = newPos; } + withYD = true; } break; case TSFKW_DD: { const char* newPos = tsFormatStr2Int32(&md, s, 2, needMoreDigits(formats, i)); @@ -1790,6 +1793,7 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec } else { s = newPos; } + withMD = true; } break; case TSFKW_D: { const char* newPos = tsFormatStr2Int32(&wd, s, 1, needMoreDigits(formats, i)); @@ -1843,6 +1847,10 @@ static int32_t char2ts(const char* s, SArray* formats, int64_t* ts, int32_t prec return err; } } + if (!withMD) { + // yyyy-mm-DDD, currently, the c api can't convert to correct timestamp, return not supported + if (withYD) return -3; + } struct STm tm = {0}; tm.tm.tm_year = year - 1900; tm.tm.tm_mon = mon; @@ -1892,8 +1900,13 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int TSFormatNode* fNode = (taosArrayGet(*formats, fErrIdx)); snprintf(errMsg, errMsgLen, "mismatch format for: %s and %s", sErrPos, fErrIdx < taosArrayGetSize(*formats) ? ((TSFormatNode*)taosArrayGet(*formats, fErrIdx))->key->name : ""); + code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR; } else if (code == -2) { snprintf(errMsg, errMsgLen, "timestamp format error: %s -> %s", tsStr, format); + code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR; + } else if (code == -3) { + snprintf(errMsg, errMsgLen, "timestamp format not supported"); + code = TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED; } return code; } diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index c65d8761b7..107276d7f9 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -489,6 +489,7 @@ TEST(timeTest, char2ts) { ASSERT_EQ(ts, 0); ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a a a 1/1+0")); ASSERT_EQ(0, TEST_char2ts("yyyy年 a a a a a a a a a a a a a a a MM/ddTZH", &ts, TSDB_TIME_PRECISION_MICRO, "1970年 a ")); + ASSERT_EQ(-3, TEST_char2ts("yyyy-mm-DDD", &ts, TSDB_TIME_PRECISION_MILLI, "1970-01-001")); } #pragma GCC diagnostic pop diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index d2093ff77c..21b791eb4d 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -144,7 +144,7 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) char path[TSDB_FILENAME_LEN] = {0}; bool atExit = true; - if (vnodeIsLeader(pVnode->pImpl)) { + if (pVnode->pImpl && vnodeIsLeader(pVnode->pImpl)) { vnodeProposeCommitOnNeed(pVnode->pImpl, atExit); } @@ -153,6 +153,10 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) taosThreadRwlockUnlock(&pMgmt->lock); vmReleaseVnode(pMgmt, pVnode); + if (pVnode->failed) { + ASSERT(pVnode->pImpl == NULL); + goto _closed; + } dInfo("vgId:%d, pre close", pVnode->vgId); vnodePreClose(pVnode->pImpl); @@ -202,6 +206,8 @@ void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) vnodeClose(pVnode->pImpl); pVnode->pImpl = NULL; + +_closed: dInfo("vgId:%d, vnode is closed", pVnode->vgId); if (commitAndRemoveWal) { @@ -386,7 +392,6 @@ static void *vmCloseVnodeInThread(void *param) { for (int32_t v = 0; v < pThread->vnodeNum; ++v) { SVnodeObj *pVnode = pThread->ppVnodes[v]; - if (pVnode->failed) continue; char stepDesc[TSDB_STEP_DESC_LEN] = {0}; snprintf(stepDesc, TSDB_STEP_DESC_LEN, "vgId:%d, start to close, %d of %d have been closed", pVnode->vgId, diff --git a/source/dnode/mgmt/node_mgmt/CMakeLists.txt b/source/dnode/mgmt/node_mgmt/CMakeLists.txt index f1be20289a..6b875db860 100644 --- a/source/dnode/mgmt/node_mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/node_mgmt/CMakeLists.txt @@ -3,6 +3,17 @@ add_library(dnode STATIC ${IMPLEMENT_SRC}) target_link_libraries( dnode mgmt_mnode mgmt_qnode mgmt_snode mgmt_vnode mgmt_dnode ) + +IF (TD_STORAGE) + + IF(${BUILD_WITH_S3}) + add_definitions(-DUSE_S3) + ELSEIF(${BUILD_WITH_COS}) + add_definitions(-DUSE_COS) + ENDIF() + +ENDIF () + target_include_directories( dnode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index d560ba1644..6f13abcebc 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -18,17 +18,17 @@ #include "audit.h" #include "libs/function/tudf.h" -#define DM_INIT_AUDIT() \ - do { \ - auditCfg.port = tsMonitorPort; \ - auditCfg.server = tsMonitorFqdn; \ - auditCfg.comp = tsMonitorComp; \ - if (auditInit(&auditCfg) != 0) { \ - return -1; \ - } \ +#define DM_INIT_AUDIT() \ + do { \ + auditCfg.port = tsMonitorPort; \ + auditCfg.server = tsMonitorFqdn; \ + auditCfg.comp = tsMonitorComp; \ + if (auditInit(&auditCfg) != 0) { \ + return -1; \ + } \ } while (0) -static SDnode globalDnode = {0}; +static SDnode globalDnode = {0}; SDnode *dmInstance() { return &globalDnode; } @@ -146,6 +146,13 @@ static bool dmCheckDataDirVersion() { return true; } +#if defined(USE_S3) + +extern int32_t s3Begin(); +extern void s3End(); + +#endif + int32_t dmInit() { dInfo("start to init dnode env"); if (dmDiskInit() != 0) return -1; @@ -156,6 +163,9 @@ int32_t dmInit() { if (dmInitMonitor() != 0) return -1; if (dmInitAudit() != 0) return -1; if (dmInitDnode(dmInstance()) != 0) return -1; +#if defined(USE_S3) + if (s3Begin() != 0) return -1; +#endif dInfo("dnode env is initialized"); return 0; @@ -181,6 +191,9 @@ void dmCleanup() { udfStopUdfd(); taosStopCacheRefreshWorker(); dmDiskClose(); +#if defined(USE_S3) + s3End(); +#endif dInfo("dnode env is cleaned up"); taosCleanupCfg(); @@ -265,19 +278,19 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { pWrapper = &pDnode->wrappers[ntype]; - if(pWrapper->func.nodeRoleFp != NULL){ + if (pWrapper->func.nodeRoleFp != NULL) { ESyncRole role = (*pWrapper->func.nodeRoleFp)(pWrapper->pMgmt); dInfo("node:%s, checking node role:%d", pWrapper->name, role); - if(role == TAOS_SYNC_ROLE_VOTER){ + if (role == TAOS_SYNC_ROLE_VOTER) { dError("node:%s, failed to alter node type since node already is role:%d", pWrapper->name, role); terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER; return -1; } } - if(pWrapper->func.isCatchUpFp != NULL){ + if (pWrapper->func.isCatchUpFp != NULL) { dInfo("node:%s, checking node catch up", pWrapper->name); - if((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1){ + if ((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1) { terrno = TSDB_CODE_MNODE_NOT_CATCH_UP; return -1; } @@ -394,7 +407,4 @@ void dmReportStartup(const char *pName, const char *pDesc) { dDebug("step:%s, %s", pStartup->name, pStartup->desc); } -int64_t dmGetClusterId() { - return globalDnode.data.clusterId; -} - +int64_t dmGetClusterId() { return globalDnode.data.clusterId; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index c32625009e..f4108b52c6 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -79,7 +79,7 @@ static void mndCancelGetNextConfig(SMnode *pMnode, void *pIter); static int32_t mndRetrieveDnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter); -static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t opLen, int32_t *pOutValue); +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pInMCfgReq, int32_t optLen, int32_t *pOutValue); #ifdef _GRANT int32_t mndUpdClusterInfo(SRpcMsg *pReq); @@ -1182,15 +1182,72 @@ _OVER: return code; } -static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - const char *options[] = { - "debugFlag", "dDebugFlag", "vDebugFlag", "mDebugFlag", "wDebugFlag", "sDebugFlag", "tsdbDebugFlag", - "tqDebugFlag", "fsDebugFlag", "udfDebugFlag", "smaDebugFlag", "idxDebugFlag", "tdbDebugFlag", "tmrDebugFlag", - "uDebugFlag", "smaDebugFlag", "rpcDebugFlag", "qDebugFlag", "metaDebugFlag", "stDebugFlag", - }; - int32_t optionSize = tListLen(options); +static int32_t mndMCfg2DCfg(SMCfgDnodeReq *pMCfgReq, SDCfgDnodeReq *pDCfgReq) { + terrno = 0; + char *p = pMCfgReq->config; + while (*p) { + if (*p == ' ') { + break; + } + p++; + } + size_t optLen = p - pMCfgReq->config; + strncpy(pDCfgReq->config, pMCfgReq->config, optLen); + pDCfgReq->config[optLen] = 0; + + if (' ' == pMCfgReq->config[optLen]) { + // 'key value' + if (strlen(pMCfgReq->value) != 0) goto _err; + strcpy(pDCfgReq->value, p + 1); + } else { + // 'key' 'value' + if (strlen(pMCfgReq->value) == 0) goto _err; + strcpy(pDCfgReq->value, pMCfgReq->value); + } + + return 0; + +_err: + mError("dnode:%d, failed to config since invalid conf:%s", pMCfgReq->dnodeId, pMCfgReq->config); + terrno = TSDB_CODE_INVALID_CFG; + return -1; +} + +static int32_t mndSendCfgDnodeReq(SMnode *pMnode, int32_t dnodeId, SDCfgDnodeReq *pDcfgReq) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SDnodeObj *pDnode = NULL; + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); + if (pIter == NULL) break; + + if (pDnode->id == dnodeId || dnodeId == -1 || dnodeId == 0) { + SEpSet epSet = mndGetDnodeEpset(pDnode); + int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, pDcfgReq); + void *pBuf = rpcMallocCont(bufLen); + + if (pBuf != NULL) { + tSerializeSDCfgDnodeReq(pBuf, bufLen, pDcfgReq); + mInfo("dnode:%d, send config req to dnode, config:%s value:%s", dnodeId, pDcfgReq->config, pDcfgReq->value); + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; + tmsgSendReq(&epSet, &rpcMsg); + code = 0; + } + } + + sdbRelease(pSdb, pDnode); + } + + if (code == -1) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + } + return code; +} + +static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; SMCfgDnodeReq cfgReq = {0}; if (tDeserializeSMCfgDnodeReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -1206,139 +1263,6 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { SDCfgDnodeReq dcfgReq = {0}; if (strcasecmp(cfgReq.config, "resetlog") == 0) { strcpy(dcfgReq.config, "resetlog"); - } else if (strncasecmp(cfgReq.config, "monitor", 7) == 0) { - if (' ' != cfgReq.config[7] && 0 != cfgReq.config[7]) { - mError("dnode:%d, failed to config monitor since invalid conf:%s", cfgReq.dnodeId, cfgReq.config); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - const char *value = cfgReq.value; - int32_t flag = atoi(value); - if (flag <= 0) { - flag = atoi(cfgReq.config + 8); - } - if (flag < 0 || flag > 2) { - mError("dnode:%d, failed to config monitor since value:%d", cfgReq.dnodeId, flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "monitor"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { - int32_t optLen = strlen("s3blocksize"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag > 1024 * 1024) { - mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId, - flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "s3blocksize"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "s3blockcachesize", 16) == 0) { - int32_t optLen = strlen("s3blockcachesize"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 4 || flag > 1024 * 1024) { - mError("dnode:%d, failed to config s3BlockCacheSize since value:%d. Valid range: [4, 1024 * 1024]", - cfgReq.dnodeId, flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "s3blockcachesize"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "s3pagecachesize", 16) == 0) { - int32_t optLen = strlen("s3pagecachesize"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 4 || flag > 1024 * 1024 * 1024) { - mError("dnode:%d, failed to config s3PageCacheSize since value:%d. Valid range: [4, 1024 * 1024]", cfgReq.dnodeId, - flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "s3pagecachesize"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "s3uploaddelaysec", 16) == 0) { - int32_t optLen = strlen("s3uploaddelaysec"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 600 || flag > 60 * 60 * 24 * 30) { - mError("dnode:%d, failed to config s3UploadDelaySec since value:%d. Valid range: [600, 60 * 60 * 24 * 30]", - cfgReq.dnodeId, flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "s3uploaddelaysec"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) { - int32_t optLen = strlen("ttlpushinterval"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 0 || flag > 100000) { - mError("dnode:%d, failed to config ttlPushInterval since value:%d. Valid range: [0, 100000]", cfgReq.dnodeId, - flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "ttlpushinterval"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "ttlbatchdropnum", 15) == 0) { - int32_t optLen = strlen("ttlbatchdropnum"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 0) { - mError("dnode:%d, failed to config ttlBatchDropNum since value:%d. Valid range: [0, %d]", cfgReq.dnodeId, flag, - INT32_MAX); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "ttlbatchdropnum"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - } else if (strncasecmp(cfgReq.config, "asynclog", 8) == 0) { - int32_t optLen = strlen("asynclog"); - int32_t flag = -1; - int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); - if (code < 0) return code; - - if (flag < 0 || flag > 1) { - mError("dnode:%d, failed to config asynclog since value:%d. Valid range: [0, 1]", cfgReq.dnodeId, flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - strcpy(dcfgReq.config, "asynclog"); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); #ifdef TD_ENTERPRISE } else if (strncasecmp(cfgReq.config, "supportvnodes", 13) == 0) { int32_t optLen = strlen("supportvnodes"); @@ -1348,9 +1272,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (flag < 0 || flag > 4096) { mError("dnode:%d, failed to config supportVnodes since value:%d. Valid range: [0, 4096]", cfgReq.dnodeId, flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; + terrno = TSDB_CODE_OUT_OF_RANGE; + goto _err_out; } if (flag == 0) { flag = tsNumOfCores * 2; @@ -1365,8 +1288,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (' ' != cfgReq.config[index] && 0 != cfgReq.config[index]) { mError("dnode:%d, failed to config activeCode since invalid conf:%s", cfgReq.dnodeId, cfgReq.config); terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; + goto _err_out; } int32_t vlen = strlen(cfgReq.value); if (vlen > 0 && ((opt == DND_ACTIVE_CODE && vlen != (TSDB_ACTIVE_KEY_LEN - 1)) || @@ -1374,9 +1296,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { (vlen > (TSDB_CONN_ACTIVE_KEY_LEN - 1) || vlen < (TSDB_ACTIVE_KEY_LEN - 1))))) { mError("dnode:%d, failed to config activeCode since invalid vlen:%d. conf:%s, val:%s", cfgReq.dnodeId, vlen, cfgReq.config, cfgReq.value); - terrno = TSDB_CODE_INVALID_OPTION; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; + terrno = TSDB_CODE_INVALID_CFG; + goto _err_out; } strcpy(dcfgReq.config, opt == DND_ACTIVE_CODE ? "activeCode" : "cActiveCode"); @@ -1384,88 +1305,53 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) { mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr()); - tFreeSMCfgDnodeReq(&cfgReq); - return -1; + terrno = TSDB_CODE_INVALID_CFG; + goto _err_out; } tFreeSMCfgDnodeReq(&cfgReq); return 0; -#endif - } else { - bool findOpt = false; - for (int32_t d = 0; d < optionSize; ++d) { - const char *optName = options[d]; - int32_t optLen = strlen(optName); - if (strncasecmp(cfgReq.config, optName, optLen) != 0) continue; + } else if (strncasecmp(cfgReq.config, "s3blocksize", 11) == 0) { + int32_t optLen = strlen("s3blocksize"); + int32_t flag = -1; + int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); + if (code < 0) return code; - if (' ' != cfgReq.config[optLen] && 0 != cfgReq.config[optLen]) { - mError("dnode:%d, failed to config since invalid conf:%s", cfgReq.dnodeId, cfgReq.config); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - const char *value = cfgReq.value; - int32_t flag = atoi(value); - if (flag <= 0) { - flag = atoi(cfgReq.config + optLen + 1); - } - if (flag < 0 || flag > 255) { - mError("dnode:%d, failed to config %s since value:%d", cfgReq.dnodeId, optName, flag); - terrno = TSDB_CODE_INVALID_CFG; - tFreeSMCfgDnodeReq(&cfgReq); - return -1; - } - - tstrncpy(dcfgReq.config, optName, optLen + 1); - snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); - findOpt = true; - } - - if (!findOpt) { + if (flag > 1024 * 1024 || (flag > -1 && flag < 4) || flag < -1) { + mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [4, 1024 * 1024]", + cfgReq.dnodeId, flag); terrno = TSDB_CODE_INVALID_CFG; - mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr()); tFreeSMCfgDnodeReq(&cfgReq); return -1; } + + strcpy(dcfgReq.config, "s3blocksize"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); +#endif + } else { + mndMCfg2DCfg(&cfgReq, &dcfgReq); + if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { + mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); + terrno = TSDB_CODE_INVALID_CFG; + goto _err_out; + } + + if (cfgCheckRangeForDynUpdate(taosGetCfg(), dcfgReq.config, dcfgReq.value, true) != 0) goto _err_out; } - char obj[50] = {0}; - sprintf(obj, "%d", cfgReq.dnodeId); + { // audit + char obj[50] = {0}; + sprintf(obj, "%d", cfgReq.dnodeId); - auditRecord(pReq, pMnode->clusterId, "alterDnode", "", obj, cfgReq.sql, cfgReq.sqlLen); + auditRecord(pReq, pMnode->clusterId, "alterDnode", obj, "", cfgReq.sql, cfgReq.sqlLen); + } tFreeSMCfgDnodeReq(&cfgReq); - int32_t code = -1; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDnodeObj *pDnode = NULL; - pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); - if (pIter == NULL) break; + return mndSendCfgDnodeReq(pMnode, cfgReq.dnodeId, &dcfgReq); - if (pDnode->id == cfgReq.dnodeId || cfgReq.dnodeId == -1 || cfgReq.dnodeId == 0) { - SEpSet epSet = mndGetDnodeEpset(pDnode); - int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq); - void *pBuf = rpcMallocCont(bufLen); - - if (pBuf != NULL) { - tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); - mInfo("dnode:%d, send config req to dnode, app:%p config:%s value:%s", cfgReq.dnodeId, pReq->info.ahandle, - dcfgReq.config, dcfgReq.value); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; - tmsgSendReq(&epSet, &rpcMsg); - code = 0; - } - } - - sdbRelease(pSdb, pDnode); - } - - if (code == -1) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - } - return code; +_err_out: + tFreeSMCfgDnodeReq(&cfgReq); + return -1; } static int32_t mndProcessConfigDnodeRsp(SRpcMsg *pRsp) { @@ -1606,16 +1492,16 @@ static void mndCancelGetNextDnode(SMnode *pMnode, void *pIter) { } // get int32_t value from 'SMCfgDnodeReq' -static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t opLen, int32_t *pOutValue) { +static int32_t mndMCfgGetValInt32(SMCfgDnodeReq *pMCfgReq, int32_t optLen, int32_t *pOutValue) { terrno = 0; - if (' ' != pMCfgReq->config[opLen] && 0 != pMCfgReq->config[opLen]) { + if (' ' != pMCfgReq->config[optLen] && 0 != pMCfgReq->config[optLen]) { goto _err; } - if (' ' == pMCfgReq->config[opLen]) { + if (' ' == pMCfgReq->config[optLen]) { // 'key value' if (strlen(pMCfgReq->value) != 0) goto _err; - *pOutValue = atoi(pMCfgReq->config + opLen + 1); + *pOutValue = atoi(pMCfgReq->config + optLen + 1); } else { // 'key' 'value' if (strlen(pMCfgReq->value) == 0) goto _err; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fd0c349dd2..135aab285b 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -37,23 +37,23 @@ typedef struct SNodeEntry { int32_t nodeId; - bool stageUpdated; // the stage has been updated due to the leader/follower change or node reboot. - SEpSet epset; // compare the epset to identify the vgroup tranferring between different dnodes. - int64_t hbTimestamp; // second + bool stageUpdated; // the stage has been updated due to the leader/follower change or node reboot. + SEpSet epset; // compare the epset to identify the vgroup tranferring between different dnodes. + int64_t hbTimestamp; // second } SNodeEntry; typedef struct SStreamExecInfo { - SArray *pNodeEntryList; + SArray * pNodeEntryList; int64_t ts; // snapshot ts int64_t activeCheckpoint; // active check point id - SHashObj *pTaskMap; - SArray *pTaskList; + SHashObj * pTaskMap; + SArray * pTaskList; TdThreadMutex lock; } SStreamExecInfo; typedef struct SVgroupChangeInfo { SHashObj *pDBMap; - SArray *pUpdateNodeList; // SArray + SArray * pUpdateNodeList; // SArray } SVgroupChangeInfo; static int32_t mndNodeCheckSentinel = 0; @@ -78,7 +78,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in static int32_t mndProcessNodeCheck(SRpcMsg *pReq); static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg); static SArray *extractNodeListFromStream(SMnode *pMnode); -static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady); +static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady); static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pPrevNodeList, const SArray *pNodeList); @@ -91,7 +91,7 @@ static void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExe static void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode); static int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot); static int32_t doKillActiveCheckpointTrans(SMnode *pMnode); -static int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList); +static int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList); int32_t mndInitStream(SMnode *pMnode) { SSdbTable table = { @@ -193,9 +193,9 @@ STREAM_ENCODE_OVER: SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRow *pRow = NULL; + SSdbRow * pRow = NULL; SStreamObj *pStream = NULL; - void *buf = NULL; + void * buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) { @@ -272,7 +272,7 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream } SStreamObj *mndAcquireStream(SMnode *pMnode, char *streamName) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = sdbAcquire(pSdb, SDB_STREAM, streamName); if (pStream == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; @@ -325,7 +325,7 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 return TSDB_CODE_SUCCESS; } - SNode *pAst = NULL; + SNode * pAst = NULL; int32_t code = nodesStringToNode(ast, &pAst); SQueryPlan *pPlan = NULL; @@ -350,7 +350,7 @@ static int32_t mndStreamGetPlanString(const char *ast, int8_t triggerType, int64 } static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj, SCMCreateStreamReq *pCreate) { - SNode *pAst = NULL; + SNode * pAst = NULL; SQueryPlan *pPlan = NULL; mInfo("stream:%s to create", pCreate->name); @@ -589,7 +589,7 @@ int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStr static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStreamObj *pStream, const char *user) { SStbObj *pStb = NULL; - SDbObj *pDb = NULL; + SDbObj * pDb = NULL; SMCreateStbReq createReq = {0}; tstrncpy(createReq.name, pStream->targetSTbName, TSDB_TABLE_FNAME_LEN); @@ -715,10 +715,10 @@ int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) } static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; int32_t code = -1; - SStreamObj *pStream = NULL; - SDbObj *pDb = NULL; + SStreamObj * pStream = NULL; + SDbObj * pDb = NULL; SCMCreateStreamReq createStreamReq = {0}; SStreamObj streamObj = {0}; @@ -761,7 +761,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { int32_t numOfStream = 0; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pMnode->pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -858,12 +858,12 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { SName name = {0}; tNameFromString(&name, createStreamReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - //reuse this function for stream + // reuse this function for stream - //TODO + // TODO if (createStreamReq.sql != NULL) { - auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, - createStreamReq.sql, strlen(createStreamReq.sql)); + auditRecord(pReq, pMnode->clusterId, "createStream", name.dbname, name.tname, createStreamReq.sql, + strlen(createStreamReq.sql)); } _OVER: if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { @@ -877,15 +877,30 @@ _OVER: return code; } +int64_t mndStreamGenChkpId(SMnode *pMnode) { + SStreamObj *pStream = NULL; + void * pIter = NULL; + SSdb * pSdb = pMnode->pSdb; + int64_t maxChkpId = 0; + while (1) { + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) break; + + maxChkpId = TMAX(maxChkpId, pStream->checkpointId); + sdbRelease(pSdb, pStream); + } + return maxChkpId + 1; +} + static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; if (sdbGetSize(pSdb, SDB_STREAM) <= 0) { return 0; } SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg)); - pMsg->checkpointId = taosGetTimestampMs(); + pMsg->checkpointId = mndStreamGenChkpId(pMnode); int32_t size = sizeof(SMStreamDoCheckpointMsg); SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size}; @@ -919,7 +934,7 @@ static int32_t mndBuildStreamCheckpointSourceReq2(void **pBuf, int32_t *pLen, in return -1; } - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void * abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); tEncodeStreamCheckpointSourceReq(&encoder, &req); @@ -1042,7 +1057,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream int32_t totLevel = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < totLevel; i++) { - SArray *pLevel = taosArrayGetP(pStream->tasks, i); + SArray * pLevel = taosArrayGetP(pStream->tasks, i); SStreamTask *pTask = taosArrayGetP(pLevel, 0); if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { @@ -1059,7 +1074,7 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream return -1; } - void *buf; + void * buf; int32_t tlen; if (mndBuildStreamCheckpointSourceReq2(&buf, &tlen, pTask->info.nodeId, chkptId, pTask->id.streamId, pTask->id.taskId) < 0) { @@ -1070,7 +1085,8 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream STransAction action = {0}; SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); - initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY); + initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, + TSDB_CODE_SYN_PROPOSE_NOT_READY); mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -1110,9 +1126,9 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream } static const char *mndGetStreamDB(SMnode *pMnode) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) { @@ -1126,9 +1142,9 @@ static const char *mndGetStreamDB(SMnode *pMnode) { } static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; + void * pIter = NULL; SStreamObj *pStream = NULL; int32_t code = 0; @@ -1149,19 +1165,18 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { return 0; } - for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) { - SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i); + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) { + SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i); if (pNodeEntry->stageUpdated) { mDebug("stream task not ready due to node update detected, checkpoint not issued"); return 0; } } - bool allReady = true; + bool allReady = true; SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allReady); if (!allReady) { - mWarn("not all vnodes are ready, ignore the checkpoint") - taosArrayDestroy(pNodeSnapshot); + mWarn("not all vnodes are ready, ignore the checkpoint") taosArrayDestroy(pNodeSnapshot); return 0; } @@ -1182,15 +1197,15 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { taosThreadMutexLock(&execInfo.lock); for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { - STaskId *p = taosArrayGet(execInfo.pTaskList, i); - STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); + STaskId * p = taosArrayGet(execInfo.pTaskList, i); + STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); if (pEntry == NULL) { continue; } if (pEntry->status != TASK_STATUS__READY) { mDebug("s-task:0x%" PRIx64 "-0x%x (nodeId:%d) status:%s not ready, checkpoint msg not issued", - pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status)); + pEntry->id.streamId, (int32_t)pEntry->id.taskId, 0, streamTaskGetStatusStr(pEntry->status)); ready = false; break; } @@ -1250,7 +1265,7 @@ static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq) { } static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamObj *pStream = NULL; SMDropStreamReq dropReq = {0}; @@ -1327,7 +1342,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SName name = {0}; tNameFromString(&name, dropReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); - //reuse this function for stream + // reuse this function for stream auditRecord(pReq, pMnode->clusterId, "dropStream", name.dbname, name.tname, dropReq.sql, dropReq.sqlLen); @@ -1379,7 +1394,7 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { } int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -1387,7 +1402,7 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) } int32_t numOfStreams = 0; - void *pIter = NULL; + void * pIter = NULL; while (1) { SStreamObj *pStream = NULL; pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -1406,8 +1421,8 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams) } static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; @@ -1483,8 +1498,8 @@ static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { } static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SMnode * pMnode = pReq->info.node; + SSdb * pSdb = pMnode->pSdb; int32_t numOfRows = 0; SStreamObj *pStream = NULL; @@ -1573,13 +1588,13 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // status char status[20 + VARSTR_HEADER_SIZE] = {0}; - STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - STaskStatusEntry* pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); + STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; + STaskStatusEntry *pe = taosHashGet(execInfo.pTaskMap, &id, sizeof(id)); if (pe == NULL) { continue; } - const char* pStatus = streamTaskGetStatusStr(pe->status); + const char *pStatus = streamTaskGetStatusStr(pe->status); STR_TO_VARSTR(status, pStatus); // status @@ -1591,24 +1606,24 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock colDataSetVal(pColInfo, numOfRows, (const char *)&pe->stage, false); // input queue - char vbuf[30] = {0}; - char buf[25] = {0}; - const char* queueInfoStr = "%4.2fMiB (%5.2f%)"; + char vbuf[30] = {0}; + char buf[25] = {0}; + const char *queueInfoStr = "%4.2fMiB (%5.2f%)"; sprintf(buf, queueInfoStr, pe->inputQUsed, pe->inputRate); STR_TO_VARSTR(vbuf, buf); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false); + colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); // output queue -// sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate); -// STR_TO_VARSTR(vbuf, buf); + // sprintf(buf, queueInfoStr, pe->outputQUsed, pe->outputRate); + // STR_TO_VARSTR(vbuf, buf); -// pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); -// colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false); + // pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + // colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false); if (pTask->info.taskLevel == TASK_LEVEL__SINK) { - const char* sinkStr = "%.2fMiB"; + const char *sinkStr = "%.2fMiB"; sprintf(buf, sinkStr, pe->sinkDataSize); } else if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) { // offset info @@ -1619,7 +1634,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock STR_TO_VARSTR(vbuf, buf); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char*)vbuf, false); + colDataSetVal(pColInfo, numOfRows, (const char *)vbuf, false); numOfRows++; } @@ -1663,7 +1678,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { } int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { - SArray* tasks = pStream->tasks; + SArray *tasks = pStream->tasks; int32_t size = taosArrayGetSize(tasks); for (int32_t i = 0; i < size; i++) { @@ -1700,7 +1715,7 @@ static int32_t mndPersistStreamLog(STrans *pTrans, const SStreamObj *pStream, in } static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamObj *pStream = NULL; SMPauseStreamReq pauseReq = {0}; @@ -1816,7 +1831,7 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUn } static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamObj *pStream = NULL; SMResumeStreamReq pauseReq = {0}; @@ -1901,7 +1916,7 @@ static void initNodeUpdateMsg(SStreamTaskNodeUpdateMsg *pMsg, const SVgroupChang } static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupChangeInfo *pInfo, int32_t nodeId, - SStreamTaskId* pId, int32_t transId) { + SStreamTaskId *pId, int32_t transId) { SStreamTaskNodeUpdateMsg req = {0}; initNodeUpdateMsg(&req, pInfo, pId, transId); @@ -1924,7 +1939,7 @@ static int32_t doBuildStreamTaskUpdateMsg(void **pBuf, int32_t *pLen, SVgroupCha return -1; } - void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + void * abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); tEncodeStreamTaskUpdateMsg(&encoder, &req); @@ -1991,7 +2006,7 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p for (int32_t k = 0; k < numOfTasks; ++k) { SStreamTask *pTask = taosArrayGetP(pLevel, k); - void *pBuf = NULL; + void * pBuf = NULL; int32_t len = 0; streamTaskUpdateEpsetInfo(pTask, pInfo->pUpdateNodeList); doBuildStreamTaskUpdateMsg(&pBuf, &len, pInfo, pTask->info.nodeId, &pTask->id, pTrans->id); @@ -2012,7 +2027,7 @@ static int32_t createStreamUpdateTrans(SStreamObj *pStream, SVgroupChangeInfo *p static bool isNodeEpsetChanged(const SEpSet *pPrevEpset, const SEpSet *pCurrent) { const SEp *pEp = GET_ACTIVE_EP(pPrevEpset); - const SEp* p = GET_ACTIVE_EP(pCurrent); + const SEp *p = GET_ACTIVE_EP(pCurrent); if (pEp->port == p->port && strncmp(pEp->fqdn, p->fqdn, TSDB_FQDN_LEN) == 0) { return false; @@ -2066,9 +2081,9 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP return info; } -static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool* allReady) { - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; +static SArray *mndTakeVgroupSnapshot(SMnode *pMnode, bool *allReady) { + SSdb * pSdb = pMnode->pSdb; + void * pIter = NULL; SVgObj *pVgroup = NULL; *allReady = true; @@ -2115,8 +2130,8 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange // check all streams that involved this vnode should update the epset info SStreamObj *pStream = NULL; - void *pIter = NULL; - STrans *pTrans = NULL; + void * pIter = NULL; + STrans * pTrans = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -2177,9 +2192,9 @@ static int32_t mndProcessVgroupChange(SMnode *pMnode, SVgroupChangeInfo *pChange } static SArray *extractNodeListFromStream(SMnode *pMnode) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; SHashObj *pHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); while (1) { @@ -2226,9 +2241,9 @@ static SArray *extractNodeListFromStream(SMnode *pMnode) { } static void doExtractTasksFromStream(SMnode *pMnode) { - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); @@ -2263,11 +2278,11 @@ static int32_t doRemoveTasks(SStreamExecInfo *pExecNode, STaskId *pRemovedId) { return TSDB_CODE_SUCCESS; } -static bool taskNodeExists(SArray* pList, int32_t nodeId) { +static bool taskNodeExists(SArray *pList, int32_t nodeId) { size_t num = taosArrayGetSize(pList); - for(int32_t i = 0; i < num; ++i) { - SNodeEntry* pEntry = taosArrayGet(pList, i); + for (int32_t i = 0; i < num; ++i) { + SNodeEntry *pEntry = taosArrayGet(pList, i); if (pEntry->nodeId == nodeId) { return true; } @@ -2277,12 +2292,12 @@ static bool taskNodeExists(SArray* pList, int32_t nodeId) { } int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { - SArray* pRemovedTasks = taosArrayInit(4, sizeof(STaskId)); + SArray *pRemovedTasks = taosArrayInit(4, sizeof(STaskId)); int32_t numOfTask = taosArrayGetSize(execInfo.pTaskList); - for(int32_t i = 0; i < numOfTask; ++i) { - STaskId* pId = taosArrayGet(execInfo.pTaskList, i); - STaskStatusEntry* pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); + for (int32_t i = 0; i < numOfTask; ++i) { + STaskId * pId = taosArrayGet(execInfo.pTaskList, i); + STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, pId, sizeof(*pId)); bool existed = taskNodeExists(pNodeSnapshot, pEntry->nodeId); if (!existed) { @@ -2290,21 +2305,21 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { } } - for(int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) { - STaskId* pId = taosArrayGet(pRemovedTasks, i); + for (int32_t i = 0; i < taosArrayGetSize(pRemovedTasks); ++i) { + STaskId *pId = taosArrayGet(pRemovedTasks, i); doRemoveTasks(&execInfo, pId); } mDebug("remove invalid stream tasks:%d, remain:%d", (int32_t)taosArrayGetSize(pRemovedTasks), - (int32_t) taosArrayGetSize(execInfo.pTaskList)); + (int32_t)taosArrayGetSize(execInfo.pTaskList)); int32_t size = taosArrayGetSize(pNodeSnapshot); - SArray* pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); - for(int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) { - SNodeEntry* p = taosArrayGet(execInfo.pNodeEntryList, i); + SArray *pValidNodeEntryList = taosArrayInit(4, sizeof(SNodeEntry)); + for (int32_t i = 0; i < taosArrayGetSize(execInfo.pNodeEntryList); ++i) { + SNodeEntry *p = taosArrayGet(execInfo.pNodeEntryList, i); - for(int32_t j = 0; j < size; ++j) { - SNodeEntry* pEntry = taosArrayGet(pNodeSnapshot, j); + for (int32_t j = 0; j < size; ++j) { + SNodeEntry *pEntry = taosArrayGet(pNodeSnapshot, j); if (pEntry->nodeId == p->nodeId) { taosArrayPush(pValidNodeEntryList, p); break; @@ -2315,7 +2330,7 @@ int32_t removeExpirednodeEntryAndTask(SArray *pNodeSnapshot) { execInfo.pNodeEntryList = taosArrayDestroy(execInfo.pNodeEntryList); execInfo.pNodeEntryList = pValidNodeEntryList; - mDebug("remain %d valid node entries", (int32_t) taosArrayGetSize(pValidNodeEntryList)); + mDebug("remain %d valid node entries", (int32_t)taosArrayGetSize(pValidNodeEntryList)); taosArrayDestroy(pRemovedTasks); return 0; } @@ -2347,7 +2362,7 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { return 0; } - bool allVnodeReady = true; + bool allVnodeReady = true; SArray *pNodeSnapshot = mndTakeVgroupSnapshot(pMnode, &allVnodeReady); if (!allVnodeReady) { taosArrayDestroy(pNodeSnapshot); @@ -2361,7 +2376,6 @@ static int32_t mndProcessNodeCheckReq(SRpcMsg *pMsg) { SVgroupChangeInfo changeInfo = mndFindChangedNodeInfo(pMnode, execInfo.pNodeEntryList, pNodeSnapshot); if (taosArrayGetSize(changeInfo.pUpdateNodeList) > 0) { - // kill current active checkpoint transaction, since the transaction is vnode wide. doKillActiveCheckpointTrans(pMnode); code = mndProcessVgroupChange(pMnode, &changeInfo); @@ -2396,7 +2410,7 @@ typedef struct SMStreamNodeCheckMsg { static int32_t mndProcessNodeCheck(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; if (sdbGetSize(pSdb, SDB_STREAM) <= 0) { return 0; } @@ -2420,7 +2434,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { SStreamTask *pTask = taosArrayGetP(pLevel, j); STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); + void * p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); if (p == NULL) { STaskStatusEntry entry = {0}; streamTaskStatusInit(&entry, pTask); @@ -2434,7 +2448,7 @@ void keepStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { } } -void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) { +void removeStreamTasksInBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode) { int32_t level = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < level; i++) { SArray *pLevel = taosArrayGetP(pStream->tasks, i); @@ -2444,12 +2458,12 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) { SStreamTask *pTask = taosArrayGetP(pLevel, j); STaskId id = {.streamId = pTask->id.streamId, .taskId = pTask->id.taskId}; - void *p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); + void * p = taosHashGet(pExecNode->pTaskMap, &id, sizeof(id)); if (p != NULL) { taosHashRemove(pExecNode->pTaskMap, &id, sizeof(id)); - for(int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { - STaskId* pId = taosArrayGet(pExecNode->pTaskList, k); + for (int32_t k = 0; k < taosArrayGetSize(pExecNode->pTaskList); ++k) { + STaskId *pId = taosArrayGet(pExecNode->pTaskList, k); if (pId->taskId == id.taskId && pId->streamId == id.streamId) { taosArrayRemove(pExecNode->pTaskList, k); mInfo("s-task:0x%x removed from buffer, remain:%d", (int32_t)id.taskId, @@ -2457,7 +2471,6 @@ void removeStreamTasksInBuf(SStreamObj* pStream, SStreamExecInfo * pExecNode) { break; } } - } } } @@ -2487,7 +2500,7 @@ STrans *doCreateTrans(SMnode *pMnode, SStreamObj *pStream, const char *name) { return pTrans; } -int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) { +int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { STrans *pTrans = doCreateTrans(pMnode, pStream, "stream-task-reset"); if (pTrans == NULL) { return terrno; @@ -2504,7 +2517,7 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) { SStreamTask *pTask = taosArrayGetP(pLevel, k); // todo extract method, with pause stream task - SVResetStreamTaskReq* pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq)); + SVResetStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResetStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to malloc in reset stream, size:%" PRIzu ", code:%s", sizeof(SVResetStreamTaskReq), @@ -2550,9 +2563,9 @@ int32_t createStreamResetStatusTrans(SMnode* pMnode, SStreamObj* pStream) { int32_t doKillActiveCheckpointTrans(SMnode *pMnode) { int32_t transId = 0; - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; STrans *pTrans = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_TRANS, pIter, (void **)&pTrans); @@ -2583,13 +2596,13 @@ int32_t doKillActiveCheckpointTrans(SMnode *pMnode) { return TSDB_CODE_SUCCESS; } -int32_t mndResetFromCheckpoint(SMnode* pMnode) { +int32_t mndResetFromCheckpoint(SMnode *pMnode) { doKillActiveCheckpointTrans(pMnode); // set all tasks status to be normal, refactor later to be stream level, instead of vnode level. - SSdb *pSdb = pMnode->pSdb; + SSdb * pSdb = pMnode->pSdb; SStreamObj *pStream = NULL; - void *pIter = NULL; + void * pIter = NULL; while (1) { pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) { @@ -2608,15 +2621,15 @@ int32_t mndResetFromCheckpoint(SMnode* pMnode) { return 0; } -int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) { +int32_t setNodeEpsetExpiredFlag(const SArray *pNodeList) { int32_t num = taosArrayGetSize(pNodeList); for (int k = 0; k < num; ++k) { - int32_t* pVgId = taosArrayGet(pNodeList, k); + int32_t *pVgId = taosArrayGet(pNodeList, k); int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList); for (int i = 0; i < numOfNodes; ++i) { - SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i); + SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, i); if (pNodeEntry->nodeId == *pVgId) { mInfo("vgId:%d expired in stream task, needs update nodeEp", *pVgId); @@ -2629,12 +2642,11 @@ int32_t setNodeEpsetExpiredFlag(const SArray* pNodeList) { return TSDB_CODE_SUCCESS; } -static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) { +static void updateStageInfo(STaskStatusEntry *pTaskEntry, int32_t stage) { int32_t numOfNodes = taosArrayGetSize(execInfo.pNodeEntryList); - for(int32_t j = 0; j < numOfNodes; ++j) { - SNodeEntry* pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j); + for (int32_t j = 0; j < numOfNodes; ++j) { + SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeEntryList, j); if (pNodeEntry->nodeId == pTaskEntry->nodeId) { - mInfo("vgId:%d stage updated from %d to %d, nodeUpdate trigger by s-task:0x%" PRIx64, pTaskEntry->nodeId, pTaskEntry->stage, stage, pTaskEntry->id.taskId); @@ -2646,7 +2658,7 @@ static void updateStageInfo(STaskStatusEntry* pTaskEntry, int32_t stage) { } int32_t mndProcessStreamHb(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; + SMnode * pMnode = pReq->info.node; SStreamHbMsg req = {0}; bool checkpointFailed = false; @@ -2699,15 +2711,15 @@ int32_t mndProcessStreamHb(SRpcMsg *pReq) { pTaskEntry->status = p->status; if (p->status != TASK_STATUS__READY) { - mDebug("received s-task:0x%"PRIx64" not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status)); + mDebug("received s-task:0x%" PRIx64 " not in ready status:%s", p->id.taskId, streamTaskGetStatusStr(p->status)); } } // current checkpoint is failed, rollback from the checkpoint trans // kill the checkpoint trans and then set all tasks status to be normal if (checkpointFailed && activeCheckpointId != 0) { - bool allReady = true; - SArray* p = mndTakeVgroupSnapshot(pMnode, &allReady); + bool allReady = true; + SArray *p = mndTakeVgroupSnapshot(pMnode, &allReady); taosArrayDestroy(p); if (allReady) { diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index f2ef00c534..6f5b370826 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "rsync.h" #include "executor.h" #include "sndInt.h" #include "tstream.h" @@ -122,6 +123,9 @@ SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) { goto FAIL; } + stopRsync(); + startRsync(); + // todo fix it: send msg to mnode to rollback to an existed checkpoint streamMetaInitForSnode(pSnode->pMeta); return pSnode; diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index dcc9f9a115..dc43da7fe7 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -8,7 +8,6 @@ set( "src/vnd/vnodeCommit.c" "src/vnd/vnodeQuery.c" "src/vnd/vnodeModule.c" - "src/vnd/vnodeCos.c" "src/vnd/vnodeSvr.c" "src/vnd/vnodeSync.c" "src/vnd/vnodeSnapshot.c" @@ -161,75 +160,6 @@ target_link_libraries( PUBLIC index ) -if(${BUILD_S3}) - -if(${BUILD_WITH_S3}) - target_include_directories( - vnode - - PUBLIC "$ENV{HOME}/.cos-local.2/include" - ) - - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - set(CMAKE_PREFIX_PATH $ENV{HOME}/.cos-local.2) - find_library(S3_LIBRARY s3) - find_library(CURL_LIBRARY curl $ENV{HOME}/.cos-local.2/lib NO_DEFAULT_PATH) - find_library(XML2_LIBRARY xml2) - find_library(SSL_LIBRARY ssl $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) - find_library(CRYPTO_LIBRARY crypto $ENV{HOME}/.cos-local.2/lib64 NO_DEFAULT_PATH) - target_link_libraries( - vnode - - # s3 - PUBLIC ${S3_LIBRARY} - PUBLIC ${CURL_LIBRARY} - PUBLIC ${SSL_LIBRARY} - PUBLIC ${CRYPTO_LIBRARY} - PUBLIC ${XML2_LIBRARY} - ) - - add_definitions(-DUSE_S3) -endif() - -if(${BUILD_WITH_COS}) - -set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") -find_library(APR_LIBRARY apr-1 PATHS /usr/local/apr/lib/) -find_library(APR_UTIL_LIBRARY aprutil-1 PATHS /usr/local/apr/lib/) -find_library(MINIXML_LIBRARY mxml) -find_library(CURL_LIBRARY curl) -target_link_libraries( - vnode - - # s3 - PUBLIC cos_c_sdk_static - PUBLIC ${APR_UTIL_LIBRARY} - PUBLIC ${APR_LIBRARY} - PUBLIC ${MINIXML_LIBRARY} - PUBLIC ${CURL_LIBRARY} -) - -# s3 -FIND_PROGRAM(APR_CONFIG_BIN NAMES apr-config apr-1-config PATHS /usr/bin /usr/local/bin /usr/local/apr/bin/) -IF (APR_CONFIG_BIN) - EXECUTE_PROCESS( - COMMAND ${APR_CONFIG_BIN} --includedir - OUTPUT_VARIABLE APR_INCLUDE_DIR - OUTPUT_STRIP_TRAILING_WHITESPACE - ) -ENDIF() -include_directories (${APR_INCLUDE_DIR}) -target_include_directories( - vnode - PUBLIC "${TD_SOURCE_DIR}/contrib/cos-c-sdk-v5/cos_c_sdk" - PUBLIC "$ENV{HOME}/.cos-local.1/include" - ) - - add_definitions(-DUSE_COS) -endif(${BUILD_WITH_COS}) - -endif() - IF (TD_GRANT) TARGET_LINK_LIBRARIES(vnode PUBLIC grant) ENDIF () diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index aaf0973b41..29eaa0509a 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -108,6 +108,7 @@ struct SRSmaStat { int64_t refId; // shared by fetch tasks volatile int64_t nBufItems; // number of items in queue buffer SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) + volatile int32_t execStat; // 0 succeed, other failed volatile int32_t nFetchAll; // active number of fetch all volatile int8_t triggerStat; // shared by fetch tasks volatile int8_t commitStat; // 0 not in committing, 1 in committing @@ -115,6 +116,7 @@ struct SRSmaStat { SRSmaFS fs; // for recovery/snapshot r/w SHashObj *infoHash; // key: suid, value: SRSmaInfo tsem_t notEmpty; // has items in queue buffer + SArray *blocks; // SArray }; struct SSmaStat { @@ -136,13 +138,18 @@ struct SSmaStat { #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { - int8_t level : 4; - int8_t fetchLevel : 4; - int8_t triggerStat; - uint16_t nScanned; - int32_t maxDelay; // ms - tmr_h tmrId; - void *pStreamState; + int8_t level; + int8_t fetchLevel; + int8_t triggerStat; + int32_t nScanned; + int32_t streamFlushed : 1; + int32_t maxDelay : 31; // ms + int64_t submitReqVer; + int64_t fetchResultVer; + tmr_h tmrId; + void *pStreamState; + void *pStreamTask; // SStreamTask + SArray *pResList; }; struct SRSmaInfo { @@ -160,12 +167,10 @@ struct SRSmaInfo { STaosQall *qall; // buffer qall of SubmitReq }; -#define RSMA_INFO_HEAD_LEN offsetof(SRSmaInfo, items) -#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) -#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) -#define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i]) -#define RSMA_INFO_IQTASK(r, i) ((r)->iTaskInfo[i]) -#define RSMA_INFO_ITEM(r, i) (&(r)->items[i]) +#define RSMA_INFO_IS_DEL(r) ((r)->delFlag == 1) +#define RSMA_INFO_SET_DEL(r) ((r)->delFlag = 1) +#define RSMA_INFO_QTASK(r, i) ((r)->taskInfo[i]) +#define RSMA_INFO_ITEM(r, i) (&(r)->items[i]) enum { TASK_TRIGGER_STAT_INIT = 0, @@ -214,11 +219,11 @@ static FORCE_INLINE void tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { int32_t smaPreClose(SSma *pSma); // rsma -void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); +void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer, int8_t rollback); int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); -// int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); +int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer, int8_t rollback); void tdRSmaQTaskInfoGetFullPath(SVnode *pVnode, tb_uid_t suid, int8_t level, STfs *pTfs, char *outputName); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 76d89be802..675bfa334a 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -43,9 +43,9 @@ extern "C" { typedef struct STqOffsetStore STqOffsetStore; -// tqPush #define STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID (-1) -#define STREAM_EXEC_TASK_STATUS_CHECK_ID (-2) +#define STREAM_EXEC_START_ALL_TASKS_ID (-2) +#define STREAM_EXEC_RESTART_ALL_TASKS_ID (-3) #define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) // tqExec @@ -156,9 +156,6 @@ char* tqOffsetBuildFName(const char* path, int32_t fVer); int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tqStream -int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); -int32_t tqScanWal(STQ* pTq); -int32_t tqStartStreamTask(STQ* pTq); int32_t tqResetStreamTaskStatus(STQ* pTq); int32_t tqStopStreamTasks(STQ* pTq); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 89d70bfabb..264c64e0fc 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -209,6 +209,7 @@ int32_t tsdbBegin(STsdb* pTsdb); // int32_t tsdbCommit(STsdb* pTsdb, SCommitInfo* pInfo); int32_t tsdbCacheCommit(STsdb* pTsdb); int32_t tsdbCompact(STsdb* pTsdb, SCompactInfo* pInfo); +int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync); // int32_t tsdbFinishCommit(STsdb* pTsdb); // int32_t tsdbRollbackCommit(STsdb* pTsdb); int tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq2* pMsg); @@ -231,7 +232,12 @@ int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) int32_t tqProcessTaskCheckpointReadyMsg(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskResetReq(STQ* pTq, SRpcMsg* pMsg); -int32_t tqLaunchStreamTaskAsync(STQ* pTq); + +int32_t tqStartStreamTaskAsync(STQ* pTq, bool restart); +int32_t tqRestartStreamTasks(STQ* pTq); +int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); +int32_t tqScanWal(STQ* pTq); +int32_t tqStartStreamTasks(STQ* pTq); int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); @@ -274,7 +280,7 @@ int32_t smaPrepareAsyncCommit(SSma* pSma); int32_t smaCommit(SSma* pSma, SCommitInfo* pInfo); int32_t smaFinishCommit(SSma* pSma); int32_t smaPostCommit(SSma* pSma); -int32_t smaDoRetention(SSma* pSma, int64_t now); +int32_t smaRetention(SSma* pSma, int64_t now); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index c26157f4b7..92b8c09fbc 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -156,10 +156,10 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { nLoops = 0; while (1) { if (atomic_load_32(&pRSmaStat->nFetchAll) <= 0) { - smaDebug("vgId:%d, rsma commit:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit); + smaDebug("vgId:%d, rsma commit, type:%d, fetch tasks are all finished", SMA_VID(pSma), isCommit); break; } else { - smaDebug("vgId:%d, rsma commit%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit); + smaDebug("vgId:%d, rsma commit, type:%d, fetch tasks are not all finished yet", SMA_VID(pSma), isCommit); } TD_SMA_LOOPS_CHECK(nLoops, 1000); } @@ -169,22 +169,24 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma, bool isCommit) { * 1) This is high cost task and should not put in asyncPreCommit originally. * 2) But, if put in asyncCommit, would trigger taskInfo cloning frequently. */ - smaInfo("vgId:%d, rsma commit:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit, + smaInfo("vgId:%d, rsma commit, type:%d, wait for all items to be consumed, TID:%p", SMA_VID(pSma), isCommit, (void *)taosGetSelfPthreadId()); nLoops = 0; while (atomic_load_64(&pRSmaStat->nBufItems) > 0) { TD_SMA_LOOPS_CHECK(nLoops, 1000); } + smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); if (!isCommit) goto _exit; - // code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); + code = atomic_load_32(&pRSmaStat->execStat); + TSDB_CHECK_CODE(code, lino, _exit); + + code = tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)); TSDB_CHECK_CODE(code, lino, _exit); smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - smaInfo("vgId:%d, rsma commit, all items are consumed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); - // all rsma results are written completely STsdb *pTsdb = NULL; if ((pTsdb = VND_RSMA1(pSma->pVnode))) { diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 04a254fc7a..dd12f2bca2 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -179,7 +179,7 @@ static void tRSmaInfoHashFreeNode(void *data) { if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) { taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES); } - tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true); + tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo); } } @@ -209,6 +209,12 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS pRSmaStat->pSma = (SSma *)pSma; atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INIT); tsem_init(&pRSmaStat->notEmpty, 0, 0); + if (!(pRSmaStat->blocks = taosArrayInit(1, sizeof(SSDataBlock)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + SSDataBlock datablock = {.info.type = STREAM_CHECKPOINT}; + taosArrayPush(pRSmaStat->blocks, &datablock); // init smaMgmt smaInit(); @@ -290,6 +296,7 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { // step 5: free pStat tsem_destroy(&(pStat->notEmpty)); + taosArrayDestroy(pStat->blocks); taosMemoryFreeClear(pStat); } } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 14c5baa402..252a3ade36 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -15,12 +15,14 @@ #include "sma.h" #include "tq.h" +#include "tstream.h" #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt #define RSMA_FETCH_DELAY_MAX (120000) // ms #define RSMA_FETCH_ACTIVE_MAX (1000) // ms #define RSMA_FETCH_INTERVAL (5000) // ms +#define RSMA_TASK_FLAG "rsma" #define RSMA_NEED_FETCH(r) (RSMA_INFO_ITEM((r), 0)->fetchLevel || RSMA_INFO_ITEM((r), 1)->fetchLevel) @@ -42,8 +44,8 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); static void tdFreeRSmaSubmitItems(SArray *pItems); static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo); -static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, - int64_t suid); +static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo, + int32_t execType, int8_t *streamFlushed); static void tdRSmaFetchTrigger(void *param, void *tmrId); static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables); @@ -72,41 +74,39 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l * * @param pSma * @param pInfo - * @param isDeepFree Only stop tmrId and free pTSchema for deep free * @return void* */ -void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { +void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { if (pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; - if (isDeepFree && pItem->tmrId) { + if (pItem->tmrId) { smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId, pInfo->suid, i + 1); taosTmrStopA(&pItem->tmrId); } - if (isDeepFree && pItem->pStreamState) { + if (pItem->pStreamState) { streamStateClose(pItem->pStreamState, false); } - if (isDeepFree && pInfo->taskInfo[i]) { - tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); + if (pItem->pStreamTask) { + tFreeStreamTask(pItem->pStreamTask); } - } - if (isDeepFree) { - taosMemoryFreeClear(pInfo->pTSchema); + taosArrayDestroy(pItem->pResList); + tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); } - if (isDeepFree) { - if (pInfo->queue) { - taosCloseQueue(pInfo->queue); - pInfo->queue = NULL; - } - if (pInfo->qall) { - taosFreeQall(pInfo->qall); - pInfo->qall = NULL; - } + taosMemoryFreeClear(pInfo->pTSchema); + + if (pInfo->queue) { + taosCloseQueue(pInfo->queue); + pInfo->queue = NULL; + } + if (pInfo->qall) { + taosFreeQall(pInfo->qall); + pInfo->qall = NULL; } taosMemoryFree(pInfo); @@ -135,7 +135,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, return TSDB_CODE_FAILED; } - if (!taosArrayGetSize(tbUids)) { + int32_t nTables = taosArrayGetSize(tbUids); + + if (0 == nTables) { smaDebug("vgId:%d, no need to update tbUidList for suid:%" PRIi64 " since Empty tbUids", SMA_VID(pSma), *suid); return TSDB_CODE_SUCCESS; } @@ -156,8 +158,9 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, terrstr()); return TSDB_CODE_FAILED; } - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 " uid:%" PRIi64 " level %d", - SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)taosArrayGet(tbUids, 0), i); + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p. suid:%" PRIi64 " uid:%" PRIi64 + "nTables:%d level %d", + SMA_VID(pSma), pRSmaInfo->taskInfo[i], *suid, *(int64_t *)TARRAY_GET_ELEM(tbUids, 0), nTables, i); } } @@ -174,8 +177,8 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) { return TSDB_CODE_FAILED; } - void *pIter = taosHashIterate(pStore->uidHash, NULL); - while (pIter) { + void *pIter = NULL; + while ((pIter = taosHashIterate(pStore->uidHash, pIter))) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); SArray *pTbUids = *(SArray **)pIter; @@ -183,8 +186,6 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) { taosHashCancelIterate(pStore->uidHash, pIter); return TSDB_CODE_FAILED; } - - pIter = taosHashIterate(pStore->uidHash, pIter); } return TSDB_CODE_SUCCESS; } @@ -232,14 +233,37 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } +static void tdRSmaTaskInit(SStreamMeta *pMeta, SRSmaInfoItem *pItem, SStreamTaskId *pId) { + STaskId id = {.streamId = pId->streamId, .taskId = pId->taskId}; + taosRLockLatch(&pMeta->lock); + SStreamTask **ppTask = (SStreamTask **)taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); + if (ppTask && *ppTask) { + pItem->submitReqVer = (*ppTask)->chkInfo.checkpointVer; + pItem->fetchResultVer = (*ppTask)->info.triggerParam; + } + taosRUnLockLatch(&pMeta->lock); +} + +static void tdRSmaTaskRemove(SStreamMeta *pMeta, int64_t streamId, int32_t taskId) { + streamMetaUnregisterTask(pMeta, streamId, taskId); + taosWLockLatch(&pMeta->lock); + int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); + if (streamMetaCommit(pMeta) < 0) { + // persist to disk + } + taosWUnLockLatch(&pMeta->lock); + smaDebug("vgId:%d, rsma task:%" PRIi64 ",%d dropped, remain tasks:%d", pMeta->vgId, streamId, taskId, numOfTasks); +} + static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx) { if ((param->qmsgLen > 0) && param->qmsg[idx]) { - SRetention *pRetention = SMA_RETENTION(pSma); - STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); - SVnode *pVnode = pSma->pVnode; - char taskInfDir[TSDB_FILENAME_LEN] = {0}; - void *pStreamState = NULL; + SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); + SRetention *pRetention = SMA_RETENTION(pSma); + STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + SVnode *pVnode = pSma->pVnode; + char taskInfDir[TSDB_FILENAME_LEN] = {0}; + void *pStreamState = NULL; // set the backend of stream state tdRSmaQTaskInfoGetFullPath(pVnode, pRSmaInfo->suid, idx + 1, pVnode->pTfs, taskInfDir); @@ -254,25 +278,46 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat taosMemoryFree(s); } - SStreamTask task = {.id.taskId = 0, .id.streamId = 0}; // TODO: assign value - task.pMeta = pVnode->pTq->pStreamMeta; - pStreamState = streamStateOpen(taskInfDir, &task, true, -1, -1); + SStreamTask *pStreamTask = taosMemoryCalloc(1, sizeof(*pStreamTask)); + if (!pStreamTask) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pItem->pStreamTask = pStreamTask; + pStreamTask->id.taskId = 0; + pStreamTask->id.streamId = pRSmaInfo->suid + idx; + pStreamTask->chkInfo.startTs = taosGetTimestampMs(); + pStreamTask->pMeta = pVnode->pTq->pStreamMeta; + pStreamTask->exec.qmsg = taosMemoryMalloc(strlen(RSMA_TASK_FLAG) + 1); + sprintf(pStreamTask->exec.qmsg, "%s", RSMA_TASK_FLAG); + pStreamTask->chkInfo.checkpointId = streamMetaGetLatestCheckpointId(pStreamTask->pMeta); + tdRSmaTaskInit(pStreamTask->pMeta, pItem, &pStreamTask->id); + pStreamState = streamStateOpen(taskInfDir, pStreamTask, true, -1, -1); if (!pStreamState) { terrno = TSDB_CODE_RSMA_STREAM_STATE_OPEN; return TSDB_CODE_FAILED; } + pItem->pStreamState = pStreamState; - SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .pStateBackend = pStreamState}; + tdRSmaTaskRemove(pStreamTask->pMeta, pStreamTask->id.streamId, pStreamTask->id.taskId); + + SReadHandle handle = {.vnode = pVnode, .initTqReader = 1, .skipRollup = 1, .pStateBackend = pStreamState}; initStorageAPI(&handle.api); - pRSmaInfo->taskInfo[idx] = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle, TD_VID(pVnode), 0); if (!pRSmaInfo->taskInfo[idx]) { terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; return TSDB_CODE_FAILED; } - SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); - pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE; // fetch the data when reboot - pItem->pStreamState = pStreamState; + + if (!(pItem->pResList = taosArrayInit(1, POINTER_BYTES))) { + return TSDB_CODE_FAILED; + } + + if (pItem->fetchResultVer < pItem->submitReqVer) { + // fetch the data when reboot + pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE; + } + if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); @@ -291,10 +336,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 + smaInfo("vgId:%d, open rsma task:%p table:%" PRIi64 " level:%" PRIi8 ", checkpointId:%" PRIi64 + ", submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64 ", maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%" PRIi32, - TD_VID(pVnode), pItem, pRSmaInfo->suid, (int8_t)(idx + 1), param->maxdelay[idx], param->watermark[idx], - pItem->maxDelay); + TD_VID(pVnode), pItem->pStreamTask, pRSmaInfo->suid, (int8_t)(idx + 1), pStreamTask->chkInfo.checkpointId, + pItem->submitReqVer, pItem->fetchResultVer, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); } return TSDB_CODE_SUCCESS; } @@ -362,7 +408,7 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_SUCCESS; _err: - tdFreeRSmaInfo(pSma, pRSmaInfo, true); + tdFreeRSmaInfo(pSma, pRSmaInfo); return TSDB_CODE_FAILED; } @@ -498,11 +544,10 @@ static void tdUidStoreDestory(STbUidStore *pStore) { if (pStore->uidHash) { if (pStore->tbUids) { // When pStore->tbUids not NULL, the pStore->uidHash has k/v; otherwise pStore->uidHash only has keys. - void *pIter = taosHashIterate(pStore->uidHash, NULL); - while (pIter) { + void *pIter = NULL; + while ((pIter = taosHashIterate(pStore->uidHash, pIter))) { SArray *arr = *(SArray **)pIter; taosArrayDestroy(arr); - pIter = taosHashIterate(pStore->uidHash, pIter); } } taosHashCleanup(pStore->uidHash); @@ -562,7 +607,7 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq2 *pMsg, STbUidStore *pStore) { * @param now * @return int32_t */ -int32_t smaDoRetention(SSma *pSma, int64_t now) { +int32_t smaRetention(SSma *pSma, int64_t now) { int32_t code = TSDB_CODE_SUCCESS; if (!VND_IS_RSMA(pSma->pVnode)) { return code; @@ -570,8 +615,8 @@ int32_t smaDoRetention(SSma *pSma, int64_t now) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { - // code = tsdbDoRetention(pSma->pRSmaTsdb[i], now); - // if (code) goto _end; + code = tsdbRetention(pSma->pRSmaTsdb[i], now, pSma->pVnode->config.sttTrigger == 1); + if (code) goto _end; } } @@ -579,17 +624,14 @@ _end: return code; } -static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, - int64_t suid) { +static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, SRSmaInfo *pInfo, + int32_t execType, int8_t *streamFlushed) { int32_t code = 0; int32_t lino = 0; SSDataBlock *output = NULL; - - SArray *pResList = taosArrayInit(1, POINTER_BYTES); - if (pResList == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - TSDB_CHECK_CODE(code, lino, _exit); - } + SArray *pResList = pItem->pResList; + STSchema *pTSchema = pInfo->pTSchema; + int64_t suid = pInfo->suid; while (1) { uint64_t ts; @@ -604,32 +646,46 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma if (taosArrayGetSize(pResList) == 0) { break; } -#if 0 - char flag[10] = {0}; - snprintf(flag, 10, "level %" PRIi8, pItem->level); - blockDebugShowDataBlocks(pResList, flag); -#endif + for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) { output = taosArrayGetP(pResList, i); + if (output->info.type == STREAM_CHECKPOINT) { + if (streamFlushed) *streamFlushed = 1; + continue; + } smaDebug("vgId:%d, result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%" PRIi64, SMA_VID(pSma), output->info.id.uid, output->info.id.groupId, output->info.rows); STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]); SSubmitReq2 *pReq = NULL; - // TODO: the schema update should be handled later(TD-17965) if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, output->info.id.groupId, SMA_VID(pSma), suid) < 0) { code = terrno ? terrno : TSDB_CODE_RSMA_RESULT; TSDB_CHECK_CODE(code, lino, _exit); } + // reset the output version to handle reboot + if (STREAM_GET_ALL == execType && output->info.version == 0) { + // the submitReqVer keeps unchanged since tdExecuteRSmaImpl and tdRSmaFetchAllResult are executed synchronously + output->info.version = pItem->submitReqVer; + } + if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { - code = terrno ? terrno : TSDB_CODE_RSMA_RESULT; + if (terrno == TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE) { + // TODO: reconfigure SSubmitReq2 + } else { + if (terrno == 0) terrno = TSDB_CODE_RSMA_RESULT; + code = terrno; + } tDestroySubmitReq(pReq, TSDB_MSG_FLG_ENCODE); taosMemoryFree(pReq); TSDB_CHECK_CODE(code, lino, _exit); } + if (STREAM_GET_ALL == execType) { + atomic_store_64(&pItem->fetchResultVer, output->info.version); + } + smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64, SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version); @@ -648,7 +704,6 @@ _exit: } else { smaDebug("vgId:%d, %s succeed, suid:%" PRIi64 ", level:%" PRIi8, SMA_VID(pSma), __func__, suid, pItem->level); } - taosArrayDestroy(pResList); qCleanExecTaskBlockBuf(taskInfo); return code; } @@ -743,8 +798,9 @@ static int32_t tdRsmaPrintSubmitReq(SSma *pSma, SSubmitReq *pReq) { */ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, ERsmaExecType type, int8_t level) { - int32_t idx = level - 1; - void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); + int32_t idx = level - 1; + void *qTaskInfo = RSMA_INFO_QTASK(pInfo, idx); + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); if (!qTaskInfo) { smaDebug("vgId:%d, no qTaskInfo to execute rsma %" PRIi8 " task for suid:%" PRIu64, SMA_VID(pSma), level, @@ -772,10 +828,14 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_FAILED; } - SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, idx); - tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo->pTSchema, pInfo->suid); + if (STREAM_INPUT__MERGED_SUBMIT == inputType) { + SPackedData *packData = POINTER_SHIFT(pMsg, sizeof(SPackedData) * (msgSize - 1)); + atomic_store_64(&pItem->submitReqVer, packData->ver); + } - return TSDB_CODE_SUCCESS; + terrno = tdRSmaExecAndSubmitResult(pSma, qTaskInfo, pItem, pInfo, STREAM_NORMAL, NULL); + + return terrno ? TSDB_CODE_FAILED : TDB_CODE_SUCCESS; } /** @@ -881,7 +941,12 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); if (!pEnv) { // only applicable when rsma env exists - return TSDB_CODE_SUCCESS; + return TDB_CODE_SUCCESS; + } + + if ((terrno = atomic_load_32(&SMA_RSMA_STAT(pSma)->execStat))) { + smaError("vgId:%d, failed to process rsma submit since invalid exec code: %s", SMA_VID(pSma), terrstr()); + goto _err; } STbUidStore uidStore = {0}; @@ -913,7 +978,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, int64_t version, void *pReq, void *pMsg, return TSDB_CODE_SUCCESS; _err: tdUidStoreDestory(&uidStore); - return TSDB_CODE_FAILED; + return terrno; } /** @@ -988,7 +1053,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { code = terrno; TSDB_CHECK_CODE(code, lino, _exit); } - +#if 0 // reload all ctbUids for suid uidStore.suid = suid; if (vnodeGetCtbIdList(pVnode, suid, uidStore.tbUids) < 0) { @@ -1002,7 +1067,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { } taosArrayClear(uidStore.tbUids); - +#endif smaDebug("vgId:%d, rsma restore env success for %" PRIi64, TD_VID(pVnode), suid); } } @@ -1050,39 +1115,141 @@ _err: return code; } -#if 0 + int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { int32_t code = 0; int32_t lino = 0; + int32_t nTaskInfo = 0; SSma *pSma = pRSmaStat->pSma; SVnode *pVnode = pSma->pVnode; - SRSmaFS fs = {0}; if (taosHashGetSize(pInfoHash) <= 0) { return TSDB_CODE_SUCCESS; } - void *infoHash = NULL; - while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; - - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - continue; - } - - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i); - if (pItem && pItem->pStreamState) { - if (streamStateCommit(pItem->pStreamState) < 0) { - code = TSDB_CODE_RSMA_STREAM_STATE_COMMIT; - TSDB_CHECK_CODE(code, lino, _exit); + // stream state: trigger checkpoint + do { + void *infoHash = NULL; + while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + continue; + } + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pRSmaInfo->taskInfo[i]) { + code = qSetSMAInput(pRSmaInfo->taskInfo[i], pRSmaStat->blocks, 1, STREAM_INPUT__CHECKPOINT); + if (code) { + taosHashCancelIterate(pInfoHash, infoHash); + TSDB_CHECK_CODE(code, lino, _exit); + } + pRSmaInfo->items[i].streamFlushed = 0; + ++nTaskInfo; } - smaDebug("vgId:%d, rsma persist, stream state commit success, table %" PRIi64 ", level %d", TD_VID(pVnode), - pRSmaInfo->suid, i + 1); } } - } + } while (0); + // stream state: wait checkpoint ready in async mode + do { + int32_t nStreamFlushed = 0; + int32_t nSleep = 0; + void *infoHash = NULL; + while (true) { + while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + continue; + } + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (pRSmaInfo->taskInfo[i] && (0 == pRSmaInfo->items[i].streamFlushed)) { + int8_t streamFlushed = 0; + code = tdRSmaExecAndSubmitResult(pSma, pRSmaInfo->taskInfo[i], &pRSmaInfo->items[i], pRSmaInfo, + STREAM_CHECKPOINT, &streamFlushed); + if (code) { + taosHashCancelIterate(pInfoHash, infoHash); + TSDB_CHECK_CODE(code, lino, _exit); + } + + if (streamFlushed) { + pRSmaInfo->items[i].streamFlushed = 1; + if (++nStreamFlushed >= nTaskInfo) { + smaInfo("vgId:%d, rsma commit, checkpoint ready, %d us consumed, received/total: %d/%d", TD_VID(pVnode), + nSleep * 10, nStreamFlushed, nTaskInfo); + taosHashCancelIterate(pInfoHash, infoHash); + goto _checkpoint; + } + } + } + } + } + taosUsleep(10); + ++nSleep; + smaDebug("vgId:%d, rsma commit, wait for checkpoint ready, %d us elapsed, received/total: %d/%d", TD_VID(pVnode), + nSleep * 10, nStreamFlushed, nTaskInfo); + } + } while (0); + +_checkpoint: + // stream state: build checkpoint in backend + do { + SStreamMeta *pMeta = NULL; + int64_t checkpointId = taosGetTimestampNs(); + bool checkpointBuilt = false; + void *infoHash = NULL; + while ((infoHash = taosHashIterate(pInfoHash, infoHash))) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + continue; + } + + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pRSmaInfo, i); + if (pItem && pItem->pStreamTask) { + SStreamTask *pTask = pItem->pStreamTask; + atomic_store_32(&pTask->pMeta->chkptNotReadyTasks, 1); + pTask->checkpointingId = checkpointId; + pTask->chkInfo.checkpointId = pTask->checkpointingId; + pTask->chkInfo.checkpointVer = pItem->submitReqVer; + pTask->info.triggerParam = pItem->fetchResultVer; + + if (!checkpointBuilt) { + // the stream states share one checkpoint + code = streamTaskBuildCheckpoint(pTask); + if (code) { + taosHashCancelIterate(pInfoHash, infoHash); + TSDB_CHECK_CODE(code, lino, _exit); + } + pMeta = pTask->pMeta; + checkpointBuilt = true; + } + + taosWLockLatch(&pMeta->lock); + if (streamMetaSaveTask(pMeta, pTask)) { + taosWUnLockLatch(&pMeta->lock); + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; + taosHashCancelIterate(pInfoHash, infoHash); + TSDB_CHECK_CODE(code, lino, _exit); + } + taosWUnLockLatch(&pMeta->lock); + smaDebug("vgId:%d, rsma commit, succeed to commit task:%p, submitReqVer:%" PRIi64 ", fetchResultVer:%" PRIi64 + ", table:%" PRIi64 ", level:%d", + TD_VID(pVnode), pTask, pItem->submitReqVer, pItem->fetchResultVer, pRSmaInfo->suid, i + 1); + } + } + } + if (pMeta) { + taosWLockLatch(&pMeta->lock); + if (streamMetaCommit(pMeta)) { + taosWUnLockLatch(&pMeta->lock); + code = terrno ? terrno : TSDB_CODE_OUT_OF_MEMORY; + TSDB_CHECK_CODE(code, lino, _exit); + } + taosWUnLockLatch(&pMeta->lock); + } + if (checkpointBuilt) { + smaInfo("vgId:%d, rsma commit, succeed to commit checkpoint:%" PRIi64, TD_VID(pVnode), checkpointId); + } + } while (0); _exit: if (code) { smaError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code)); @@ -1091,7 +1258,7 @@ _exit: terrno = code; return code; } -#endif + /** * @brief trigger to get rsma result in async mode * @@ -1165,14 +1332,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { smaDebug("vgId:%d, rsma fetch task planned for level:%" PRIi8 " suid:%" PRIi64 " since stat is active", SMA_VID(pSma), pItem->level, pRSmaInfo->suid); // async process - pItem->fetchLevel = pItem->level; -#if 0 - // debugging codes - SRSmaInfo *qInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaInfo->suid); - SRSmaInfoItem *qItem = RSMA_INFO_ITEM(qInfo, pItem->level - 1); - make sure(qItem->level == pItem->level); - make sure(qItem->fetchLevel == pItem->fetchLevel); -#endif + atomic_store_8(&pItem->fetchLevel, 1); + if (atomic_load_8(&pRSmaInfo->assigned) == 0) { tsem_post(&(pStat->notEmpty)); } @@ -1217,15 +1378,15 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) { SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1); - if (pItem->fetchLevel) { - pItem->fetchLevel = 0; + + if (1 == atomic_val_compare_exchange_8(&pItem->fetchLevel, 1, 0)) { qTaskInfo_t taskInfo = RSMA_INFO_QTASK(pInfo, i - 1); if (!taskInfo) { continue; } if ((++pItem->nScanned * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { - smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16 " maxDelay:%d, fetch executed", + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch executed", SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay); } else { int64_t curMs = taosGetTimestampMs(); @@ -1245,14 +1406,15 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) { if ((terrno = qSetSMAInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { goto _err; } - if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo->pTSchema, pInfo->suid) < 0) { + if (tdRSmaExecAndSubmitResult(pSma, taskInfo, pItem, pInfo, STREAM_GET_ALL, NULL) < 0) { + atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno); goto _err; } - smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16 " maxDelay:%d, fetch finished", + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch finished", SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay); } else { - smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi16 + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi32 " maxDelay:%d, fetch not executed as fetch level is %" PRIi8, SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay, pItem->fetchLevel); } @@ -1275,6 +1437,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA .msgStr = POINTER_SHIFT(msg, sizeof(int32_t) + sizeof(int64_t))}; if (!taosArrayPush(pSubmitArr, &packData)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; tdFreeRSmaSubmitItems(pSubmitArr); goto _err; } @@ -1294,6 +1457,7 @@ static int32_t tdRSmaBatchExec(SSma *pSma, SRSmaInfo *pInfo, STaosQall *qall, SA } return TSDB_CODE_SUCCESS; _err: + atomic_store_32(&SMA_RSMA_STAT(pSma)->execStat, terrno); smaError("vgId:%d, batch exec for suid:%" PRIi64 " execType:%d size:%d failed since %s", SMA_VID(pSma), pInfo->suid, type, (int32_t)taosArrayGetSize(pSubmitArr), terrstr()); tdFreeRSmaSubmitItems(pSubmitArr); @@ -1369,15 +1533,11 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { if (ASSERTS(oldVal >= 0, "oldVal of nFetchAll: %d < 0", oldVal)) { code = TSDB_CODE_APP_ERROR; + taosHashCancelIterate(infoHash, pIter); TSDB_CHECK_CODE(code, lino, _exit); } - int8_t curStat = atomic_load_8(RSMA_COMMIT_STAT(pRSmaStat)); - if (curStat == 1) { - smaDebug("vgId:%d, fetch all not exec as commit stat is %" PRIi8, SMA_VID(pSma), curStat); - } else { - tdRSmaFetchAllResult(pSma, pInfo); - } + tdRSmaFetchAllResult(pSma, pInfo); if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) { atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index fff58feeb1..1c1a4a192c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1064,7 +1064,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms return code; } -static void doStartStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { +static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) { const char* id = pTask->id.idStr; int64_t nextProcessedVer = pStreamTask->hTaskInfo.haltVer; @@ -1105,7 +1105,11 @@ static void doStartStep2(SStreamTask* pTask, SStreamTask* pStreamTask, STQ* pTq) } } -// this function should be executed by only one thread +static void ddxx() { + +} + +// this function should be executed by only one thread, so we set an sentinel to protect this function int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; SStreamMeta* pMeta = pTq->pStreamMeta; @@ -1134,6 +1138,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { } } + // let's decide which step should be executed now if (pTask->execInfo.step1Start == 0) { ASSERT(pTask->status.pauseAllowed == false); int64_t ts = taosGetTimestampMs(); @@ -1147,7 +1152,8 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { } } else { if (pTask->execInfo.step2Start == 0) { - tqDebug("s-task:%s resume from paused, original step1 startTs:%" PRId64, id, pTask->execInfo.step1Start); + tqDebug("s-task:%s continue exec scan-history(step1), original step1 startTs:%" PRId64 ", already elapsed:%.2fs", + id, pTask->execInfo.step1Start, pTask->execInfo.step1El); } else { tqDebug("s-task:%s already in step2, no need to scan-history data, step2 starTs:%"PRId64, id, pTask->execInfo.step2Start); atomic_store_32(&pTask->status.inScanHistorySentinel, 0); @@ -1167,20 +1173,37 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { return 0; } - streamScanHistoryData(pTask); + int64_t st = taosGetTimestampMs(); + SScanhistoryDataInfo retInfo = streamScanHistoryData(pTask, st); - double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0; - if (streamTaskGetStatus(pTask, NULL) == TASK_STATUS__PAUSE) { + double el = (taosGetTimestampMs() - st) / 1000.0; + pTask->execInfo.step1El += el; + + if (retInfo.ret == TASK_SCANHISTORY_QUIT || retInfo.ret == TASK_SCANHISTORY_REXEC) { int8_t status = streamTaskSetSchedStatusInactive(pTask); - tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d", pTask->id.idStr, el, status); - atomic_store_32(&pTask->status.inScanHistorySentinel, 0); + + if (retInfo.ret == TASK_SCANHISTORY_REXEC) { + streamReExecScanHistoryFuture(pTask, retInfo.idleTime); + } else { + char* p = NULL; + ETaskStatus s = streamTaskGetStatus(pTask, &p); + + if (s == TASK_STATUS__PAUSE) { + tqDebug("s-task:%s is paused in the step1, elapsed time:%.2fs total:%.2fs, sched-status:%d", pTask->id.idStr, + el, pTask->execInfo.step1El, status); + } else if (s == TASK_STATUS__STOP || s == TASK_STATUS__DROPPING) { + tqDebug("s-task:%s status:%p not continue scan-history data, total elapsed time:%.2fs quit", pTask->id.idStr, p, + pTask->execInfo.step1El); + } + } + streamMetaReleaseTask(pMeta, pTask); return 0; } // the following procedure should be executed, no matter status is stop/pause or not - tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, el); + tqDebug("s-task:%s scan-history(step 1) ended, elapsed time:%.2fs", id, pTask->execInfo.step1El); if (pTask->info.fillHistory) { SStreamTask* pStreamTask = NULL; @@ -1203,23 +1226,20 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { code = streamTaskHandleEvent(pStreamTask->status.pSM, TASK_EVENT_HALT); if (code == TSDB_CODE_SUCCESS) { - doStartStep2(pTask, pStreamTask, pTq); + doStartFillhistoryStep2(pTask, pStreamTask, pTq); } else { tqError("s-task:%s failed to halt s-task:%s, not launch step2", id, pStreamTask->id.idStr); } streamMetaReleaseTask(pMeta, pStreamTask); - } else { STimeWindow* pWindow = &pTask->dataRange.window; ASSERT(HAS_RELATED_FILLHISTORY_TASK(pTask)); - // Not update the fill-history time window until the state transfer is completed if the related fill-history task - // exists. - tqDebug( - "s-task:%s scan-history in stream time window completed, now start to handle data from WAL, startVer:%" PRId64 - ", window:%" PRId64 " - %" PRId64, - id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); + // Not update the fill-history time window until the state transfer is completed. + tqDebug("s-task:%s scan-history in stream time window completed, start to handle data from WAL, startVer:%" PRId64 + ", window:%" PRId64 " - %" PRId64, + id, pTask->chkInfo.nextProcessVer, pWindow->skey, pWindow->ekey); code = streamTaskScanHistoryDataComplete(pTask); } @@ -1297,14 +1317,15 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = pReq->taskId; int32_t vgId = TD_VID(pTq->pVnode); - if (taskId == STREAM_EXEC_TASK_STATUS_CHECK_ID) { - tqStartStreamTask(pTq); - return 0; - } - if (taskId == STREAM_EXEC_EXTRACT_DATA_IN_WAL_ID) { // all tasks are extracted submit data from the wal tqScanWal(pTq); return 0; + } else if (taskId == STREAM_EXEC_START_ALL_TASKS_ID) { + tqStartStreamTasks(pTq); + return 0; + } else if (taskId == STREAM_EXEC_RESTART_ALL_TASKS_ID) { + tqRestartStreamTasks(pTq); + return 0; } SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, pReq->streamId, taskId); @@ -1492,6 +1513,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion, streamSchedExec(pTask); } } else if (status == TASK_STATUS__UNINIT) { + // todo: fill-history task init ? if (pTask->info.fillHistory == 0) { EStreamTaskEvent event = HAS_RELATED_FILLHISTORY_TASK(pTask) ? TASK_EVENT_INIT_STREAM_SCANHIST : TASK_EVENT_INIT; streamTaskHandleEvent(pTask->status.pSM, event); @@ -1890,7 +1912,7 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { int32_t numOfTasks = streamMetaGetNumOfTasks(pMeta); int32_t updateTasks = taosHashGetSize(pMeta->updateInfo.pTasks); - pMeta->startInfo.startAllTasksFlag = 1; + pMeta->startInfo.tasksWillRestart = 1; if (updateTasks < numOfTasks) { tqDebug("vgId:%d closed tasks:%d, unclosed:%d, all tasks will be started when nodeEp update completed", vgId, @@ -1899,45 +1921,11 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { } else { if (!pTq->pVnode->restored) { tqDebug("vgId:%d vnode restore not completed, not restart the tasks, clear the start after nodeUpdate flag", vgId); - pMeta->startInfo.startAllTasksFlag = 0; + pMeta->startInfo.tasksWillRestart = 0; streamMetaWUnLock(pMeta); } else { - tqInfo("vgId:%d tasks are all updated and stopped, restart them", vgId); - terrno = 0; - - streamMetaWUnLock(pMeta); - - while (streamMetaTaskInTimer(pMeta)) { - tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); - taosMsleep(100); - } - - streamMetaWLock(pMeta); - - int32_t code = streamMetaReopen(pMeta); - if (code != 0) { - tqError("vgId:%d failed to reopen stream meta", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { - tqError("vgId:%d failed to load stream tasks", vgId); - streamMetaWUnLock(pMeta); - taosArrayDestroy(req.pNodeList); - return -1; - } - - if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { - tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); - tqResetStreamTaskStatus(pTq); - tqLaunchStreamTaskAsync(pTq); - } else { - tqInfo("vgId:%d, follower node not start stream tasks", vgId); - } - streamMetaWUnLock(pMeta); + tqStartStreamTaskAsync(pTq, true); } } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 51d51ebbef..4b64737936 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -25,7 +25,7 @@ typedef struct STableSinkInfo { tstr name; } STableSinkInfo; -static bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks); +static bool hasOnlySubmitData(const SArray* pBlocks, int32_t numOfBlocks); static int32_t tsAscendingSortFn(const void* p1, const void* p2); static int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDataBlock, char* stbFullName, SSubmitTbData* pTableData); diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 2d94f23009..26849f8578 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -22,6 +22,8 @@ static int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle); static int32_t setWalReaderStartOffset(SStreamTask* pTask, int32_t vgId); static bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver); +static bool taskReadyForDataFromWal(SStreamTask* pTask); +static bool doPutDataIntoInputQFromWal(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems); // extract data blocks(submit/delete) from WAL, and add them into the input queue for all the sources tasks. int32_t tqScanWal(STQ* pTq) { @@ -58,7 +60,7 @@ int32_t tqScanWal(STQ* pTq) { return 0; } -int32_t tqStartStreamTask(STQ* pTq) { +int32_t tqStartStreamTasks(STQ* pTq) { int32_t code = TSDB_CODE_SUCCESS; int32_t vgId = TD_VID(pTq->pVnode); SStreamMeta* pMeta = pTq->pStreamMeta; @@ -73,6 +75,7 @@ int32_t tqStartStreamTask(STQ* pTq) { streamMetaWLock(pMeta); pTaskList = taosArrayDup(pMeta->pTaskList, NULL); taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); pMeta->startInfo.startTs = taosGetTimestampMs(); streamMetaWUnLock(pMeta); @@ -97,7 +100,7 @@ int32_t tqStartStreamTask(STQ* pTq) { streamLaunchFillHistoryTask(pTask); } - streamMetaUpdateTaskReadyInfo(pTask); + streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); streamMetaReleaseTask(pMeta, pTask); continue; } @@ -115,7 +118,67 @@ int32_t tqStartStreamTask(STQ* pTq) { return code; } -int32_t tqLaunchStreamTaskAsync(STQ* pTq) { +int32_t tqRestartStreamTasks(STQ* pTq) { + SStreamMeta* pMeta = pTq->pStreamMeta; + int32_t vgId = pMeta->vgId; + int32_t code = 0; + int64_t st = taosGetTimestampMs(); + + while(1) { + int32_t startVal = atomic_val_compare_exchange_32(&pMeta->startInfo.taskStarting, 0, 1); + if (startVal == 0) { + break; + } + + tqDebug("vgId:%d in start stream tasks procedure, wait for 500ms and recheck", vgId); + taosMsleep(500); + } + + terrno = 0; + tqInfo("vgId:%d tasks are all updated and stopped, restart all tasks, triggered by transId:%d", vgId, + pMeta->updateInfo.transId); + + while (streamMetaTaskInTimer(pMeta)) { + tqDebug("vgId:%d some tasks in timer, wait for 100ms and recheck", pMeta->vgId); + taosMsleep(100); + } + + streamMetaWLock(pMeta); + + code = streamMetaReopen(pMeta); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to reopen stream meta", vgId); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + int64_t el = taosGetTimestampMs() - st; + + tqInfo("vgId:%d close&reload state elapsed time:%.3fms", vgId, el/1000.); + + code = streamMetaLoadAllTasks(pTq->pStreamMeta); + if (code != TSDB_CODE_SUCCESS) { + tqError("vgId:%d failed to load stream tasks, code:%s", vgId, tstrerror(terrno)); + streamMetaWUnLock(pMeta); + code = terrno; + return code; + } + + if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { + tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); + tqResetStreamTaskStatus(pTq); + tqStartStreamTasks(pTq); + } else { + tqInfo("vgId:%d, follower node not start stream tasks", vgId); + } + + streamMetaWUnLock(pMeta); + code = terrno; + return code; +} + +int32_t tqStartStreamTaskAsync(STQ* pTq, bool restart) { SStreamMeta* pMeta = pTq->pStreamMeta; int32_t vgId = pMeta->vgId; @@ -132,10 +195,10 @@ int32_t tqLaunchStreamTaskAsync(STQ* pTq) { return -1; } - tqDebug("vgId:%d check %d stream task(s) status async", vgId, numOfTasks); + tqDebug("vgId:%d start all %d stream task(s) async", vgId, numOfTasks); pRunReq->head.vgId = vgId; pRunReq->streamId = 0; - pRunReq->taskId = STREAM_EXEC_TASK_STATUS_CHECK_ID; + pRunReq->taskId = restart? STREAM_EXEC_RESTART_ALL_TASKS_ID:STREAM_EXEC_START_ALL_TASKS_ID; SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); @@ -320,14 +383,13 @@ bool handleFillhistoryScanComplete(SStreamTask* pTask, int64_t ver) { return false; } -static bool taskReadyForDataFromWal(SStreamTask* pTask) { +bool taskReadyForDataFromWal(SStreamTask* pTask) { // non-source or fill-history tasks don't need to response the WAL scan action. if ((pTask->info.taskLevel != TASK_LEVEL__SOURCE) || (pTask->status.downstreamReady == 0)) { return false; } // not in ready state, do not handle the data from wal -// int32_t status = pTask->status.taskStatus; char* p = NULL; int32_t status = streamTaskGetStatus(pTask, &p); if (streamTaskGetStatus(pTask, &p) != TASK_STATUS__READY) { @@ -359,7 +421,7 @@ static bool taskReadyForDataFromWal(SStreamTask* pTask) { return true; } -static bool doPutDataIntoInputQFromWal(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems) { +bool doPutDataIntoInputQFromWal(SStreamTask* pTask, int64_t maxVer, int32_t* numOfItems) { const char* id = pTask->id.idStr; int32_t numOfNewItems = 0; @@ -449,7 +511,6 @@ int32_t doScanWalForAllTasks(SStreamMeta* pStreamMeta, bool* pScanIdle) { int64_t maxVer = (pTask->info.fillHistory == 1) ? pTask->dataRange.range.maxVer : INT64_MAX; taosThreadMutexLock(&pTask->lock); - tqDebug("s-task:%s lock", pTask->id.idStr); char* p = NULL; ETaskStatus status = streamTaskGetStatus(pTask, &p); diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index bf6f0cf4d6..918d0bd7d0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -12,11 +12,11 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "cos.h" #include "tsdb.h" #include "tsdbDataFileRW.h" #include "tsdbReadUtil.h" #include "vnd.h" -#include "vndCos.h" #define ROCKS_BATCH_SIZE (4096) @@ -3099,7 +3099,7 @@ static int32_t tsdbCacheLoadBlockS3(STsdbFD *pFD, uint8_t **ppBlock) { } */ int64_t block_offset = (pFD->blkno - 1) * tsS3BlockSize * pFD->szPage; - code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, ppBlock); + code = s3GetObjectBlock(pFD->objName, block_offset, tsS3BlockSize * pFD->szPage, 0, ppBlock); if (code != TSDB_CODE_SUCCESS) { // taosMemoryFree(pBlock); // code = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index e9e848f1b0..b6aa791cf0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -28,13 +28,18 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p // bool allNullRow = true; if (HASTYPE(pReader->type, CACHESCAN_RETRIEVE_LAST)) { + uint64_t ts = 0; + SFirstLastRes* p; + col_id_t colId; for (int32_t i = 0; i < pReader->numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotIds[i]); - SFirstLastRes* p = (SFirstLastRes*)varDataVal(pRes[i]); int32_t slotId = slotIds[i]; SLastCol* pColVal = (SLastCol*)taosArrayGet(pRow, i); + colId = pColVal->colVal.cid; + p = (SFirstLastRes*)varDataVal(pRes[i]); p->ts = pColVal->ts; + ts = p->ts; p->isNull = !COL_VAL_IS_VALUE(&pColVal->colVal); // allNullRow = p->isNull & allNullRow; @@ -55,6 +60,19 @@ static int32_t saveOneRow(SArray* pRow, SSDataBlock* pBlock, SCacheRowsReader* p varDataSetLen(pRes[i], pColInfoData->info.bytes - VARSTR_HEADER_SIZE); colDataSetVal(pColInfoData, numOfRows, (const char*)pRes[i], false); } + for (int32_t idx = 0; idx < taosArrayGetSize(pBlock->pDataBlock); ++idx) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, idx); + if (pCol->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID && pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP) { + colDataSetVal(pCol, numOfRows, (const char*)&ts, false); + continue; + } else if (pReader->numOfCols == 1 && idx != dstSlotIds[0] && pCol->info.colId == colId) { + if (!p->isNull) { + colDataSetVal(pCol, numOfRows, p->buf, false); + } else { + colDataSetNULL(pCol, numOfRows); + } + } + } // pBlock->info.rows += allNullRow ? 0 : 1; ++pBlock->info.rows; @@ -144,7 +162,7 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, p->type = type; p->pVnode = pVnode; p->pTsdb = p->pVnode->pTsdb; - p->info.verRange = (SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}; + p->info.verRange = (SVersionRange){.minVer = 0, .maxVer = INT64_MAX}; p->info.suid = suid; p->numOfCols = numOfCols; p->pCidList = pCidList; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 46e3aff0d4..22fb3b84ec 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -192,7 +192,7 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) { committer->ctx->tbid->uid = record->uid; if (metaGetInfo(committer->tsdb->pVnode->pMeta, record->uid, &info, NULL) != 0) { - code = tsdbIterMergerSkipTableData(committer->dataIterMerger, committer->ctx->tbid); + code = tsdbIterMergerSkipTableData(committer->tombIterMerger, committer->ctx->tbid); TSDB_CHECK_CODE(code, lino, _exit); continue; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 38d221d978..02ef75ae86 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -14,15 +14,15 @@ */ #include "tsdbFS2.h" +#include "cos.h" #include "tsdbUpgrade.h" #include "vnd.h" -#include "vndCos.h" #define BLOCK_COMMIT_FACTOR 3 extern int vnodeScheduleTask(int (*execute)(void *), void *arg); extern int vnodeScheduleTaskEx(int tpid, int (*execute)(void *), void *arg); -extern void remove_file(const char *fname); +extern void remove_file(const char *fname, bool last_level); #define TSDB_FS_EDIT_MIN TSDB_FEDIT_COMMIT #define TSDB_FS_EDIT_MAX (TSDB_FEDIT_MERGE + 1) @@ -532,7 +532,8 @@ static int32_t tsdbFSDoSanAndFix(STFileSystem *fs) { if (taosIsDir(file->aname)) continue; if (tsdbFSGetFileObjHashEntry(&fobjHash, file->aname) == NULL) { - remove_file(file->aname); + int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs); + remove_file(file->aname, nlevel > 1 && file->did.level == nlevel - 1); } } @@ -1282,4 +1283,4 @@ int32_t tsdbFSEnableBgTask(STFileSystem *fs) { fs->stop = false; taosThreadMutexUnlock(&fs->tsdb->mutex); return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 9edb03d35b..cc05b8ee18 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -14,6 +14,7 @@ */ #include "tsdbFile2.h" +#include "cos.h" // to_json static int32_t head_to_json(const STFile *file, cJSON *json); @@ -41,10 +42,20 @@ static const struct { [TSDB_FTYPE_STT] = {"stt", stt_to_json, stt_from_json}, }; -void remove_file(const char *fname) { +void remove_file(const char *fname, bool last_level) { int32_t code = taosRemoveFile(fname); if (code) { - tsdbError("file:%s remove failed", fname); + if (tsS3Enabled && last_level) { + const char *object_name = taosDirEntryBaseName((char *)fname); + long s3_size = tsS3Enabled ? s3Size(object_name) : 0; + if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) { + s3DeleteObjects(&object_name, 1); + } else { + tsdbError("file:%s remove failed", fname); + } + } else { + tsdbError("file:%s remove failed", fname); + } } else { tsdbInfo("file:%s is removed", fname); } @@ -224,6 +235,7 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) { fobj[0]->state = TSDB_FSTATE_LIVE; fobj[0]->ref = 1; tsdbTFileName(pTsdb, f, fobj[0]->fname); + fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs); return 0; } @@ -245,7 +257,7 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) { tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); if (nRef == 0) { if (fobj->state == TSDB_FSTATE_DEAD) { - remove_file(fobj->fname); + remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1); } taosMemoryFree(fobj); } @@ -261,7 +273,7 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) { taosThreadMutexUnlock(&fobj->mutex); tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef); if (nRef == 0) { - remove_file(fobj->fname); + remove_file(fobj->fname, fobj->nlevel > 1 && fobj->f->did.level == fobj->nlevel - 1); taosMemoryFree(fobj); } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.h b/source/dnode/vnode/src/tsdb/tsdbFile2.h index 9da198c1f0..b94f7a9fd0 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.h @@ -76,6 +76,7 @@ struct STFileObj { STFile f[1]; int32_t state; int32_t ref; + int32_t nlevel; char fname[TSDB_FILENAME_LEN]; }; diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c index 0e15107606..36675a771c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -846,14 +846,14 @@ static void tLDataIterPinSttBlock(SLDataIter* pIter, const char* id) { if (pInfo->blockData[0].sttBlockIndex == pIter->iSttBlk) { pInfo->blockData[0].pin = true; ASSERT(!pInfo->blockData[1].pin); - tsdbDebug("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id); + tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%" PRId64 " %s", pIter->iSttBlk, pIter->cid, id); return; } if (pInfo->blockData[1].sttBlockIndex == pIter->iSttBlk) { pInfo->blockData[1].pin = true; ASSERT(!pInfo->blockData[0].pin); - tsdbDebug("pin stt-block, blockIndex:%d, stt-fileVer:%"PRId64" %s", pIter->iSttBlk, pIter->cid, id); + tsdbTrace("pin stt-block, blockIndex:%d, stt-fileVer:%"PRId64" %s", pIter->iSttBlk, pIter->cid, id); return; } diff --git a/source/dnode/vnode/src/tsdb/tsdbOpen.c b/source/dnode/vnode/src/tsdb/tsdbOpen.c index c32b2eedd7..a1f864814f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbOpen.c +++ b/source/dnode/vnode/src/tsdb/tsdbOpen.c @@ -89,6 +89,8 @@ int tsdbOpen(SVnode *pVnode, STsdb **ppTsdb, const char *dir, STsdbKeepCfg *pKee return 0; _err: + tsdbCloseFS(&pTsdb->pFS); + taosThreadMutexDestroy(&pTsdb->mutex); taosMemoryFree(pTsdb); return -1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index d1919d95ba..4682c47bd1 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -48,9 +48,9 @@ static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScan static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); -static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost); -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, - int8_t* pLevel); +static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, int32_t order, SCostSummary* pCost); +static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idstr, + int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); static int32_t doBuildDataBlock(STsdbReader* pReader); @@ -384,7 +384,7 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, void initReaderStatus(&pReader->status); - pReader->pTsdb = getTsdbByRetentions(pVnode, pCond->twindows.skey, pVnode->config.tsdbCfg.retentions, idstr, &level); + pReader->pTsdb = getTsdbByRetentions(pVnode, pCond, pVnode->config.tsdbCfg.retentions, idstr, &level); pReader->info.suid = pCond->suid; pReader->info.order = pCond->order; @@ -565,9 +565,9 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, uid, pReader->idStr); if (ASCENDING_TRAVERSE(pReader->info.order)) { - w.skey = pScanInfo->lastKey + step; + w.skey = pScanInfo->lastProcKey + step; } else { - w.ekey = pScanInfo->lastKey + step; + w.ekey = pScanInfo->lastProcKey + step; } if (isEmptyQueryTimeWindow(&w)) { @@ -607,14 +607,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN clearBrinBlockIter(&iter); - pBlockNum->numOfLastFiles = pReader->status.pCurrentFileset->lvlArr->size; - int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; + pBlockNum->numOfSttFiles = pReader->status.pCurrentFileset->lvlArr->size; + int32_t total = pBlockNum->numOfSttFiles + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug( "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfLastFiles, + numOfTables, pBlockNum->numOfBlocks, (int32_t)taosArrayGetSize(pTableScanInfoList), pBlockNum->numOfSttFiles, sizeInDisk / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; @@ -1200,13 +1200,12 @@ static bool overlapWithNeighborBlock2(SFileDataBlockInfo* pBlock, SBrinRecord* p } } -static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, SLastBlockReader* pLastBlockReader, int32_t order) { +static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo, int32_t order) { bool ascScan = ASCENDING_TRAVERSE(order); - bool bHasDataInLastBlock = hasDataInLastBlock(pLastBlockReader); int64_t key = 0; - if (bHasDataInLastBlock) { - int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader); + if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { + int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; key = ascScan ? TMIN(pBlock->record.firstKey, keyInStt) : TMAX(pBlock->record.lastKey, keyInStt); } else { key = ascScan ? pBlock->record.firstKey : pBlock->record.lastKey; @@ -1215,10 +1214,10 @@ static int64_t getBoarderKeyInFiles(SFileDataBlockInfo* pBlock, SLastBlockReader return key; } -static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock, - SLastBlockReader* pLastBlockReader, int32_t order) { +static bool bufferDataInFileBlockGap(TSDBKEY keyInBuf, SFileDataBlockInfo* pBlock, STableBlockScanInfo* pScanInfo, + int32_t order) { bool ascScan = ASCENDING_TRAVERSE(order); - int64_t key = getBoarderKeyInFiles(pBlock, pLastBlockReader, order); + int64_t key = getBoarderKeyInFiles(pBlock, pScanInfo, order); return (ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts < key)) || (!ascScan && (keyInBuf.ts != TSKEY_INITIAL_VAL && keyInBuf.ts > key)); @@ -1302,10 +1301,9 @@ typedef struct { } SDataBlockToLoadInfo; static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, - STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, - STsdbReader* pReader) { - int32_t neighborIndex = 0; + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, STsdbReader* pReader) { SBrinRecord rec = {0}; + int32_t neighborIndex = 0; bool hasNeighbor = getNeighborBlockOfSameTable(&pReader->status.blockIter, pBlockInfo, pScanInfo, &neighborIndex, pReader->info.order, &rec); @@ -1319,9 +1317,11 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->hasDupTs = (pBlockInfo->record.numRow > pBlockInfo->record.count) || (pBlockInfo->record.count <= 0); pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, &pBlockInfo->record, pReader->info.order); - if (hasDataInLastBlock(pLastBlockReader)) { - int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); - pInfo->overlapWithLastBlock = !(pBlockInfo->record.lastKey < tsLast || pBlockInfo->record.firstKey > tsLast); + ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); + if (pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA) { + int64_t nextProcKeyInStt = pScanInfo->sttKeyInfo.nextProcKey; + pInfo->overlapWithLastBlock = + !(pBlockInfo->record.lastKey < nextProcKeyInStt || pBlockInfo->record.firstKey > nextProcKeyInStt); } pInfo->moreThanCapcity = pBlockInfo->record.numRow > pReader->resBlockInfo.capacity; @@ -1336,9 +1336,9 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* // 5. delete info should not overlap with current block data // 6. current block should not contain the duplicated ts static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, - TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + TSDBKEY keyInBuf) { SDataBlockToLoadInfo info = {0}; - getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader); + getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader); bool loadDataBlock = (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf || @@ -1358,9 +1358,9 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlock } static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, - TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + TSDBKEY keyInBuf) { SDataBlockToLoadInfo info = {0}; - getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader, pReader); + getBlockToLoadInfo(&info, pBlockInfo, pScanInfo, keyInBuf, pReader); bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || info.overlapWithDelInfo || info.overlapWithLastBlock); return isCleanFileBlock; @@ -1417,14 +1417,15 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return code; } -static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, - SVersionRange* pVerRange) { +static bool nextRowFromSttBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, + SVersionRange* pVerRange) { int32_t step = ASCENDING_TRAVERSE(pLastBlockReader->order) ? 1 : -1; while (1) { bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); if (!hasVal) { // the next value will be the accessed key in stt - pScanInfo->lastKeyInStt += step; + pScanInfo->sttKeyInfo.status = STT_FILE_NO_DATA; + pScanInfo->sttKeyInfo.nextProcKey += step; return false; } @@ -1433,10 +1434,11 @@ static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBloc int64_t ver = pRow->pBlockData->aVersion[pRow->iRow]; pLastBlockReader->currentKey = key; - pScanInfo->lastKeyInStt = key; + pScanInfo->sttKeyInfo.nextProcKey = key; if (!hasBeenDropped(pScanInfo->delSkyline, &pScanInfo->sttBlockDelIndex, key, ver, pLastBlockReader->order, pVerRange)) { + pScanInfo->sttKeyInfo.status = STT_FILE_HAS_DATA; return true; } } @@ -1457,7 +1459,7 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas // avoid the fetch next row replace the referenced stt block in buffer doPinSttBlock(pLastBlockReader); - bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo, &pReader->info.verRange); + bool hasVal = nextRowFromSttBlocks(pLastBlockReader, pScanInfo, &pReader->info.verRange); doUnpinSttBlock(pLastBlockReader); if (hasVal) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); @@ -1694,7 +1696,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } if (copied) { - pBlockScanInfo->lastKey = tsLastBlock; + pBlockScanInfo->lastProcKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { code = tsdbRowMergerAdd(pMerger, &fRow, NULL); @@ -2062,9 +2064,9 @@ static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbRea STbData* d = NULL; TSDBKEY startKey = {0}; if (ASCENDING_TRAVERSE(pReader->info.order)) { - startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey + 1, .version = pReader->info.verRange.minVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastProcKey + 1, .version = pReader->info.verRange.minVer}; } else { - startKey = (TSDBKEY){.ts = pBlockScanInfo->lastKey - 1, .version = pReader->info.verRange.maxVer}; + startKey = (TSDBKEY){.ts = pBlockScanInfo->lastProcKey - 1, .version = pReader->info.verRange.maxVer}; } int32_t code = @@ -2129,9 +2131,9 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan STimeWindow w = pLBlockReader->window; if (ASCENDING_TRAVERSE(pLBlockReader->order)) { - w.skey = pScanInfo->lastKeyInStt; + w.skey = pScanInfo->sttKeyInfo.nextProcKey; } else { - w.ekey = pScanInfo->lastKeyInStt; + w.ekey = pScanInfo->sttKeyInfo.nextProcKey; } int64_t st = taosGetTimestampUs(); @@ -2164,7 +2166,7 @@ static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScan initMemDataIterator(pScanInfo, pReader); initDelSkylineIterator(pScanInfo, pReader->info.order, &pReader->cost); - code = nextRowFromLastBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange); + code = nextRowFromSttBlocks(pLBlockReader, pScanInfo, &pReader->info.verRange); int64_t el = taosGetTimestampUs() - st; pReader->cost.initLastBlockReader += (el / 1000.0); @@ -2209,7 +2211,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc } if (copied) { - pBlockScanInfo->lastKey = key; + pBlockScanInfo->lastProcKey = key; return TSDB_CODE_SUCCESS; } else { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); @@ -2354,16 +2356,16 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); // it is a clean block, load it directly - if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf, pLastBlockReader) && - (pRecord->numRow <= pReader->resBlockInfo.capacity)) { - if (asc || (!hasDataInLastBlock(pLastBlockReader))) { + int64_t cap = pReader->resBlockInfo.capacity; + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlockScanInfo, keyInBuf) && (pRecord->numRow <= cap)) { + if (asc || (pBlockScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA)) { code = copyBlockDataToSDataBlock(pReader); if (code) { goto _end; } // record the last key value - pBlockScanInfo->lastKey = asc ? pRecord->lastKey : pRecord->firstKey; + pBlockScanInfo->lastProcKey = asc ? pRecord->lastKey : pRecord->firstKey; goto _end; } } @@ -2378,6 +2380,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } SBlockData* pBlockData = &pReader->status.fileBlockData; + initLastBlockReader(pLastBlockReader, pBlockScanInfo, pReader); while (1) { bool hasBlockData = false; @@ -2527,7 +2530,7 @@ TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SArray* pTableList) { SReaderStatus* pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; - pBlockNum->numOfLastFiles = 0; + pBlockNum->numOfSttFiles = 0; size_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBrinBlk)); @@ -2564,7 +2567,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum, SAr return code; } - if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { + if (pBlockNum->numOfBlocks + pBlockNum->numOfSttFiles > 0) { break; } } @@ -2684,11 +2687,13 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } } -static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, SLastBlockReader* pLastBlockReader, bool asc) { - if(!hasDataInLastBlock(pLastBlockReader)) { +static bool notOverlapWithSttFiles(SFileDataBlockInfo* pBlockInfo, STableBlockScanInfo* pScanInfo, bool asc) { + ASSERT(pScanInfo->sttKeyInfo.status != STT_FILE_READER_UNINIT); + + if(pScanInfo->sttKeyInfo.status == STT_FILE_NO_DATA) { return true; } else { - int64_t keyInStt = getCurrentKeyInLastBlock(pLastBlockReader); + int64_t keyInStt = pScanInfo->sttKeyInfo.nextProcKey; return (asc && pBlockInfo->record.lastKey < keyInStt) || (!asc && pBlockInfo->record.firstKey > keyInStt); } } @@ -2717,10 +2722,12 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { return terrno; } - initLastBlockReader(pLastBlockReader, pScanInfo, pReader); - TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); + if (pScanInfo->sttKeyInfo.status == STT_FILE_READER_UNINIT) { + initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + } - if (fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf, pLastBlockReader)) { + TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); + if (fileBlockShouldLoad(pReader, pBlockInfo, pScanInfo, keyInBuf)) { code = doLoadFileBlockData(pReader, pBlockIter, &pStatus->fileBlockData, pScanInfo->uid); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2728,13 +2735,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); - } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pLastBlockReader, pReader->info.order)) { + } else if (bufferDataInFileBlockGap(keyInBuf, pBlockInfo, pScanInfo, pReader->info.order)) { // data in memory that are earlier than current file block and stt blocks // rows in buffer should be less than the file block in asc, greater than file block in desc - int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pLastBlockReader, pReader->info.order); + int64_t endKey = getBoarderKeyInFiles(pBlockInfo, pScanInfo, pReader->info.order); code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { - if (notOverlapWithSttFiles(pBlockInfo, pLastBlockReader, asc)) { + if (notOverlapWithSttFiles(pBlockInfo, pScanInfo, asc)) { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->resBlockInfo.pResBlock->info; pInfo->rows = pBlockInfo->record.numRow; @@ -2745,7 +2752,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlockInfo->record.lastKey, pReader->info.order); // update the last key for the corresponding table - pScanInfo->lastKey = asc ? pInfo->window.ekey : pInfo->window.skey; + pScanInfo->lastProcKey = asc ? pInfo->window.ekey : pInfo->window.skey; tsdbDebug("%p uid:%" PRIu64 " clean file block retrieved from file, global index:%d, " "table index:%d, rows:%d, brange:%" PRId64 "-%" PRId64 ", %s", @@ -2760,8 +2767,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { tsdbDebug("load data in last block firstly %s", pReader->idStr); int64_t st = taosGetTimestampUs(); + // let's load data from stt files + initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + // no data in last block, no need to proceed. while (hasDataInLastBlock(pLastBlockReader)) { + ASSERT(pScanInfo->sttKeyInfo.status == STT_FILE_HAS_DATA); + code = buildComposedDataBlockImpl(pReader, pScanInfo, &pReader->status.fileBlockData, pLastBlockReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2988,7 +3000,7 @@ static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) if (pBlockInfo) { STableBlockScanInfo* pScanInfo = tSimpleHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); if (pScanInfo) { - lastKey = pScanInfo->lastKey; + lastKey = pScanInfo->lastProcKey; } pDumpInfo->totalRows = pBlockInfo->record.numRow; @@ -3013,7 +3025,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl } // all data files are consumed, try data in buffer - if (num.numOfBlocks + num.numOfLastFiles == 0) { + if (num.numOfBlocks + num.numOfSttFiles == 0) { pReader->status.loadFromFile = false; taosArrayDestroy(pTableList); return code; @@ -3140,9 +3152,9 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { } } -static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idStr, +static STsdb* getTsdbByRetentions(SVnode* pVnode, SQueryTableDataCond* pCond, SRetention* retentions, const char* idStr, int8_t* pLevel) { - if (VND_IS_RSMA(pVnode)) { + if (VND_IS_RSMA(pVnode) && !pCond->skipRollup) { int8_t level = 0; int8_t precision = pVnode->config.tsdbCfg.precision; int64_t now = taosGetTimestamp(precision); @@ -3158,7 +3170,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret } break; } - if ((now - pRetention->keep) <= (winSKey + offset)) { + if ((now - pRetention->keep) <= (pCond->twindows.skey + offset)) { break; } ++level; @@ -3458,15 +3470,15 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger, SVersionRange* pVerRange, const char* idStr) { - while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo, pVerRange)) { + while (nextRowFromSttBlocks(pLastBlockReader, pScanInfo, pVerRange)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW* pRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tsdbRowMergerAdd(pMerger, pRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, - pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, - idStr); + pScanInfo->sttBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), + pScanInfo->sttKeyInfo.nextProcKey, idStr); break; } } @@ -3722,7 +3734,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT pBlock->info.dataLoad = 1; pBlock->info.rows += 1; - pScanInfo->lastKey = pTSRow->ts; + pScanInfo->lastProcKey = pTSRow->ts; return TSDB_CODE_SUCCESS; } @@ -3856,14 +3868,15 @@ int32_t tsdbSetTableList2(STsdbReader* pReader, const void* pTableList, int32_t // todo extract method if (ASCENDING_TRAVERSE(pReader->info.order)) { int64_t skey = pReader->info.window.skey; - pInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; - pInfo->lastKeyInStt = skey; + pInfo->lastProcKey = (skey > INT64_MIN) ? (skey - 1) : skey; + pInfo->sttKeyInfo.nextProcKey = skey; } else { int64_t ekey = pReader->info.window.ekey; - pInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; - pInfo->lastKeyInStt = ekey; + pInfo->lastProcKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; + pInfo->sttKeyInfo.nextProcKey = ekey; } + pInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; tSimpleHashPut(pReader->status.pTableMap, &pInfo->uid, sizeof(uint64_t), &pInfo, POINTER_BYTES); } @@ -4224,7 +4237,7 @@ int32_t tsdbReaderSuspend2(STsdbReader* pReader) { if (pBlockScanInfo) { // save lastKey to restore memory iterator STimeWindow w = pReader->resBlockInfo.pResBlock->info.window; - pBlockScanInfo->lastKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey; + pBlockScanInfo->lastProcKey = ASCENDING_TRAVERSE(pReader->info.order) ? w.ekey : w.skey; // reset current current table's data block scan info, pBlockScanInfo->iterInit = false; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index a4d2eef093..af7cae33fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -157,17 +157,18 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf if (ASCENDING_TRAVERSE(pTsdbReader->info.order)) { int64_t skey = pTsdbReader->info.window.skey; - pScanInfo->lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; - pScanInfo->lastKeyInStt = skey; + pScanInfo->lastProcKey = (skey > INT64_MIN) ? (skey - 1) : skey; + pScanInfo->sttKeyInfo.nextProcKey = skey; } else { int64_t ekey = pTsdbReader->info.window.ekey; - pScanInfo->lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; - pScanInfo->lastKeyInStt = ekey; + pScanInfo->lastProcKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; + pScanInfo->sttKeyInfo.nextProcKey = ekey; } + pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; tSimpleHashPut(pTableMap, &pScanInfo->uid, sizeof(uint64_t), &pScanInfo, POINTER_BYTES); tsdbTrace("%p check table uid:%" PRId64 " from lastKey:%" PRId64 " %s", pTsdbReader, pScanInfo->uid, - pScanInfo->lastKey, pTsdbReader->idStr); + pScanInfo->lastProcKey, pTsdbReader->idStr); } taosSort(pUidList->tableUidList, numOfTables, sizeof(uint64_t), uidComparFunc); @@ -200,8 +201,8 @@ void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step) { } pInfo->delSkyline = taosArrayDestroy(pInfo->delSkyline); - pInfo->lastKey = ts; - pInfo->lastKeyInStt = ts + step; + pInfo->lastProcKey = ts; + pInfo->sttKeyInfo.nextProcKey = ts + step; } } @@ -241,6 +242,7 @@ static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) { taosArrayClear(pScanInfo->pBlockList); taosArrayClear(pScanInfo->pBlockIdxList); taosArrayClear(pScanInfo->pFileDelData); // del data from each file set + pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; } void cleanupInfoFoxNextFileset(SSHashObj* pTableMap) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index ea8efe37ee..47585ea6e3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -63,20 +63,31 @@ typedef struct STableDataBlockIdx { int32_t globalIndex; } STableDataBlockIdx; +typedef enum ESttKeyStatus { + STT_FILE_READER_UNINIT = 0x0, + STT_FILE_NO_DATA = 0x1, + STT_FILE_HAS_DATA = 0x2, +} ESttKeyStatus; + +typedef struct SSttKeyInfo { + ESttKeyStatus status; // this value should be updated when switch to the next fileset + int64_t nextProcKey; +} SSttKeyInfo; + typedef struct STableBlockScanInfo { - uint64_t uid; - TSKEY lastKey; - TSKEY lastKeyInStt; // last accessed key in stt - SArray* pBlockList; // block data index list, SArray - SArray* pBlockIdxList; // SArray - SArray* pMemDelData; // SArray - SArray* pFileDelData; // SArray from each file set - SIterInfo iter; // mem buffer skip list iterator - SIterInfo iiter; // imem buffer skip list iterator - SArray* delSkyline; // delete info for this table - int32_t fileDelIndex; // file block delete index - int32_t sttBlockDelIndex; // delete index for last block - bool iterInit; // whether to initialize the in-memory skip list iterator or not + uint64_t uid; + TSKEY lastProcKey; + SSttKeyInfo sttKeyInfo; + SArray* pBlockList; // block data index list, SArray + SArray* pBlockIdxList; // SArray + SArray* pMemDelData; // SArray + SArray* pFileDelData; // SArray from each file set + SIterInfo iter; // mem buffer skip list iterator + SIterInfo iiter; // imem buffer skip list iterator + SArray* delSkyline; // delete info for this table + int32_t fileDelIndex; // file block delete index + int32_t sttBlockDelIndex; // delete index for last block + bool iterInit; // whether to initialize the in-memory skip list iterator or not } STableBlockScanInfo; typedef struct SResultBlockInfo { @@ -108,7 +119,7 @@ typedef struct STableUidList { typedef struct { int32_t numOfBlocks; - int32_t numOfLastFiles; + int32_t numOfSttFiles; } SBlockNumber; typedef struct SBlockIndex { diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index def9a73d10..8b9cae42fc 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#include "cos.h" #include "tsdb.h" -#include "vndCos.h" static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { int32_t code = 0; @@ -338,9 +338,9 @@ static int32_t tsdbReadFileS3(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64 // 2, retrieve pgs from s3 uint8_t *pBlock = NULL; int64_t retrieve_offset = PAGE_OFFSET(pgno, pFD->szPage); - int64_t pgnoEnd = pgno - 1 + (size - n + szPgCont - 1) / szPgCont; + int64_t pgnoEnd = pgno - 1 + (bOffset + size - n + szPgCont - 1) / szPgCont; int64_t retrieve_size = (pgnoEnd - pgno + 1) * pFD->szPage; - code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, &pBlock); + code = s3GetObjectBlock(pFD->objName, retrieve_offset, retrieve_size, 1, &pBlock); if (code != TSDB_CODE_SUCCESS) { goto _exit; } diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 0fc1e1b64b..194c2784f4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -13,9 +13,10 @@ * along with this program. If not, see . */ +#include "cos.h" #include "tsdb.h" #include "tsdbFS2.h" -#include "vndCos.h" +#include "vnd.h" typedef struct { STsdb *tsdb; @@ -291,15 +292,15 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { if (expLevel < 0) { // remove the fileset for (int32_t ftype = 0; (ftype < TSDB_FTYPE_MAX) && (fobj = fset->farr[ftype], 1); ++ftype) { if (fobj == NULL) continue; - + /* int32_t nlevel = tfsGetLevel(rtner->tsdb->pVnode->pTfs); if (tsS3Enabled && nlevel > 1 && TSDB_FTYPE_DATA == ftype && fobj->f->did.level == nlevel - 1) { code = tsdbRemoveFileObjectS3(rtner, fobj); TSDB_CHECK_CODE(code, lino, _exit); - } else { - code = tsdbDoRemoveFileObject(rtner, fobj); - TSDB_CHECK_CODE(code, lino, _exit); - } + } else {*/ + code = tsdbDoRemoveFileObject(rtner, fobj); + TSDB_CHECK_CODE(code, lino, _exit); + //} } SSttLvl *lvl; @@ -387,6 +388,8 @@ _exit: return code; } +static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); } + static int32_t tsdbDoRetentionSync(void *arg) { int32_t code = 0; int32_t lino = 0; @@ -409,6 +412,7 @@ _exit: TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } tsem_post(&((SRtnArg *)arg)->tsdb->pVnode->canCommit); + tsdbFreeRtnArg(arg); return code; } @@ -438,7 +442,7 @@ _exit: return code; } -static void tsdbFreeRtnArg(void *arg) { taosMemoryFree(arg); } + int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { int32_t code = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 1e6526da48..e8be48499b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#include "cos.h" #include "tsdb.h" -#include "vndCos.h" /** * @brief max key by precision @@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq2 *pMsg, SSubmitRsp2 arrSize = taosArrayGetSize(pMsg->aSubmitTbData); // scan and convert - if (tsdbScanAndConvertSubmitMsg(pTsdb, pMsg) < 0) { + if ((terrno = tsdbScanAndConvertSubmitMsg(pTsdb, pMsg)) < 0) { if (terrno != TSDB_CODE_TDB_TABLE_RECONFIGURE) { tsdbError("vgId:%d, failed to insert data since %s", TD_VID(pTsdb->pVnode), tstrerror(terrno)); } diff --git a/source/dnode/vnode/src/vnd/vnodeModule.c b/source/dnode/vnode/src/vnd/vnodeModule.c index 6ccce5c9d7..df08fb8a2b 100644 --- a/source/dnode/vnode/src/vnd/vnodeModule.c +++ b/source/dnode/vnode/src/vnd/vnodeModule.c @@ -13,8 +13,8 @@ * along with this program. If not, see . */ +#include "cos.h" #include "vnd.h" -#include "vndCos.h" typedef struct SVnodeTask SVnodeTask; struct SVnodeTask { diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index 3bdecee79b..ff79e83d72 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ +#include "cos.h" #include "sync.h" #include "tsdb.h" #include "vnd.h" -#include "vndCos.h" int32_t vnodeGetPrimaryDir(const char *relPath, int32_t diskPrimary, STfs *pTfs, char *buf, size_t bufLen) { if (pTfs) { diff --git a/source/dnode/vnode/src/vnd/vnodeRetention.c b/source/dnode/vnode/src/vnd/vnodeRetention.c index f3344d1d7d..c510c0fe92 100644 --- a/source/dnode/vnode/src/vnd/vnodeRetention.c +++ b/source/dnode/vnode/src/vnd/vnodeRetention.c @@ -15,8 +15,12 @@ #include "vnd.h" -extern int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync); - int32_t vnodeDoRetention(SVnode *pVnode, int64_t now) { - return tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); + int32_t code = TSDB_CODE_SUCCESS; + + code = tsdbRetention(pVnode->pTsdb, now, pVnode->config.sttTrigger == 1); + + if (TSDB_CODE_SUCCESS == code) code = smaRetention(pVnode->pSma, now); + + return code; } \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index b31463ac00..eadfd39d0b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -18,7 +18,7 @@ #include "tmsg.h" #include "tstrbuild.h" #include "vnd.h" -#include "vndCos.h" +#include "cos.h" #include "vnode.h" #include "vnodeInt.h" @@ -928,6 +928,17 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, goto _exit; } + if(tsEnableAuditCreateTable){ + char* str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); + if (str == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + rcode = -1; + goto _exit; + } + strcpy(str, pCreateReq->name); + taosArrayPush(tbNames, &str); + } + // validate hash sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name); if (vnodeValidateTableHash(pVnode, tbName) < 0) { @@ -951,12 +962,6 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, } taosArrayPush(rsp.pArray, &cRsp); - - if (tsEnableAuditCreateTable) { - char *str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN); - strcpy(str, pCreateReq->name); - taosArrayPush(tbNames, &str); - } } vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids)); @@ -985,10 +990,10 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, tNameFromString(&name, pVnode->config.dbname, T_NAME_ACCT | T_NAME_DB); SStringBuilder sb = {0}; - for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { - char **key = (char **)taosArrayGet(tbNames, iReq); + for(int32_t i = 0; i < tbNames->size; i++){ + char** key = (char**)taosArrayGet(tbNames, i); taosStringBuilderAppendStringLen(&sb, *key, strlen(*key)); - if (iReq < req.nReqs - 1) { + if(i < tbNames->size - 1){ taosStringBuilderAppendChar(&sb, ','); } taosMemoryFreeClear(*key); @@ -1669,7 +1674,7 @@ _exit: atomic_add_fetch_64(&pVnode->statis.nBatchInsert, 1); if (code == 0) { atomic_add_fetch_64(&pVnode->statis.nBatchInsertSuccess, 1); - tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); + code = tdProcessRSmaSubmit(pVnode->pSma, ver, pSubmitReq, pReq, len, STREAM_INPUT__DATA_SUBMIT); } // clear diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index 4a0c987e57..3944f8ed91 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -557,7 +557,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) SStreamMeta* pMeta = pVnode->pTq->pStreamMeta; streamMetaWLock(pMeta); - if (pMeta->startInfo.startAllTasksFlag) { + if (pMeta->startInfo.tasksWillRestart) { vInfo("vgId:%d, sync restore finished, stream tasks will be launched by other thread", vgId); streamMetaWUnLock(pMeta); return; @@ -570,7 +570,7 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) } else { vInfo("vgId:%d sync restore finished, start to launch stream tasks", pVnode->config.vgId); tqResetStreamTaskStatus(pVnode->pTq); - tqLaunchStreamTaskAsync(pVnode->pTq); + tqStartStreamTaskAsync(pVnode->pTq, false); } } else { vInfo("vgId:%d, sync restore finished, not launch stream tasks since not leader", vgId); diff --git a/source/libs/command/inc/commandInt.h b/source/libs/command/inc/commandInt.h index c704eb3951..535167e65c 100644 --- a/source/libs/command/inc/commandInt.h +++ b/source/libs/command/inc/commandInt.h @@ -112,7 +112,6 @@ extern "C" { #define COMMAND_CATALOG_DEBUG "catalogDebug" #define COMMAND_ENABLE_MEM_DEBUG "enableMemDebug" #define COMMAND_DISABLE_MEM_DEBUG "disableMemDebug" -#define COMMAND_ASYNCLOG "asynclog" typedef struct SExplainGroup { int32_t nodeNum; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 4cc76c6572..12063d4883 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -808,16 +808,6 @@ static int32_t execAlterCmd(char* cmd, char* value, bool* processed) { return code; } qInfo("memory dbg disabled"); - } else if (0 == strcasecmp(cmd, COMMAND_ASYNCLOG)) { - int newAsyncLogValue = (strlen(value) == 0) ? 1 : atoi(value); - if (newAsyncLogValue != 0 && newAsyncLogValue != 1) { - code = TSDB_CODE_INVALID_CFG_VALUE; - qError("failed to alter asynclog, error:%s", tstrerror(code)); - goto _return; - } - - code = TSDB_CODE_SUCCESS; - tsAsyncLog = newAsyncLogValue; } else { goto _return; } @@ -844,10 +834,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { goto _return; } - bool forbidden = false; - taosLocalCfgForbiddenToChange(pStmt->config, &forbidden); - if (forbidden) { - terrno = TSDB_CODE_OPS_NOT_SUPPORT; + if (cfgCheckRangeForDynUpdate(tsCfg, pStmt->config, pStmt->value, false)) { return terrno; } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 740ff7b0dc..946d3311a4 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -40,8 +40,10 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) typedef struct SGroupResInfo { - int32_t index; - SArray* pRows; // SArray + int32_t index; // rows consumed in func:doCopyToSDataBlockXX + int32_t iter; // relate to index-1, last consumed data's slot id in hash table + void* dataPos; // relate to index-1, last consumed data's position, in the nodelist of cur slot + SArray* pRows; // SArray char* pBuf; bool freeItem; } SGroupResInfo; @@ -178,7 +180,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode); -int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); +int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode, const SReadHandle* readHandle); void cleanupQueryTableDataCond(SQueryTableDataCond* pCond); int32_t convertFillType(int32_t mode); diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 288919d709..331ce44366 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -580,6 +580,7 @@ typedef struct SStreamSessionAggOperatorInfo { bool reCkBlock; SSDataBlock* pCheckpointRes; bool clearState; + bool recvGetAll; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -603,6 +604,7 @@ typedef struct SStreamStateAggOperatorInfo { SArray* historyWins; bool reCkBlock; SSDataBlock* pCheckpointRes; + bool recvGetAll; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -677,6 +679,12 @@ void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); void doBuildResultDatablock(struct SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); +/** + * @brief copydata from hash table, instead of copying from SGroupResInfo's pRow + */ +int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, bool ignoreGroup); + bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); bool hasSlimitOffsetInfo(SLimitInfo* pLimitInfo); void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 29d098494b..6d59698855 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -54,6 +54,19 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM #define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) +static void setColIdForCacheReadBlock(SSDataBlock* pBlock, SNodeList* pTargets) { + SNode* pNode; + int32_t idx = 0; + FOREACH(pNode, pTargets) { + if (nodeType(pNode) == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)pNode; + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, idx); + pColInfo->info.colId = pCol->colId; + } + idx++; + } +} + SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -114,10 +127,12 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe capacity = TMIN(totalTables, 4096); pInfo->pBufferredRes = createOneDataBlock(pInfo->pRes, false); + setColIdForCacheReadBlock(pInfo->pBufferredRes, pScanNode->pTargets); blockDataEnsureCapacity(pInfo->pBufferredRes, capacity); } else { // by tags pInfo->retrieveType = CACHESCAN_RETRIEVE_TYPE_SINGLE | SCAN_ROW_TYPE(pScanNode->ignoreNull); capacity = 1; // only one row output + setColIdForCacheReadBlock(pInfo->pRes, pScanNode->pTargets); } initResultSizeInfo(&pOperator->resultInfo, capacity); @@ -191,9 +206,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { SSDataBlock* pRes = pInfo->pRes; if (pInfo->indexOfBufferedRes < pInfo->pBufferredRes->info.rows) { - for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) { - SColMatchItem* pMatchInfo = taosArrayGet(pInfo->matchInfo.pList, i); - int32_t slotId = pMatchInfo->dstSlotId; + for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBufferredRes->pDataBlock); ++i) { + SColumnInfoData* pCol = taosArrayGet(pRes->pDataBlock, i); + int32_t slotId = pCol->info.slotId; SColumnInfoData* pSrc = taosArrayGet(pInfo->pBufferredRes->pDataBlock, slotId); SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, slotId); @@ -201,8 +216,10 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { if (colDataIsNull_s(pSrc, pInfo->indexOfBufferedRes)) { colDataSetNULL(pDst, 0); } else { - char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); - colDataSetVal(pDst, 0, p, false); + if (pSrc->pData) { + char* p = colDataGetData(pSrc, pInfo->indexOfBufferedRes); + colDataSetVal(pDst, 0, p, false); + } } } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 753d3e680c..39b47504c6 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -1713,7 +1713,7 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode) { return c; } -int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode) { +int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode, const SReadHandle* readHandle) { pCond->order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; pCond->numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); @@ -1732,6 +1732,7 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; + pCond->skipRollup = readHandle->skipRollup; int32_t j = 0; for (int32_t i = 0; i < pCond->numOfCols; ++i) { diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 60dc6f0185..a0e35fff87 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -69,12 +69,14 @@ static int32_t doSetSMABlock(SOperatorInfo* pOperator, void* input, size_t numOf } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; - SPackedData tmp = { - .pDataBlock = pDataBlock, - }; + SPackedData tmp = {.pDataBlock = pDataBlock}; taosArrayPush(pInfo->pBlockLists, &tmp); } pInfo->blockType = STREAM_INPUT__DATA_BLOCK; + } else if (type == STREAM_INPUT__CHECKPOINT) { + SPackedData tmp = {.pDataBlock = input}; + taosArrayPush(pInfo->pBlockLists, &tmp); + pInfo->blockType = STREAM_INPUT__CHECKPOINT; } return TSDB_CODE_SUCCESS; @@ -633,7 +635,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo blockIndex += 1; current += p->info.rows; - ASSERT(p->info.rows > 0); + ASSERT(p->info.rows > 0 || p->info.type == STREAM_CHECKPOINT); taosArrayPush(pResList, &p); if (current >= rowsThreshold) { @@ -871,32 +873,6 @@ int32_t qGetExplainExecInfo(qTaskInfo_t tinfo, SArray* pExecInfoList) { return getOperatorExplainExecInfo(pTaskInfo->pRoot, pExecInfoList); } -int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) { - SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; - if (pTaskInfo->pRoot == NULL) { - return TSDB_CODE_INVALID_PARA; - } - - int32_t nOptrWithVal = 0; - // int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal); - // if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) { - // taosMemoryFreeClear(*pOutput); - // *len = 0; - // } - return 0; -} - -int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) { - SExecTaskInfo* pTaskInfo = (struct SExecTaskInfo*)tinfo; - - if (pTaskInfo == NULL || pInput == NULL || len == 0) { - return TSDB_CODE_INVALID_PARA; - } - - return 0; - // return decodeOperator(pTaskInfo->pRoot, pInput, len); -} - int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; @@ -1072,7 +1048,7 @@ int32_t qRestoreStreamOperatorOption(qTaskInfo_t tinfo) { } } -bool qStreamRecoverScanFinished(qTaskInfo_t tinfo) { +bool qStreamScanhistoryFinished(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; return pTaskInfo->streamInfo.recoverScanFinished; } diff --git a/source/libs/executor/src/executorInt.c b/source/libs/executor/src/executorInt.c index 8ad174f366..138da16324 100644 --- a/source/libs/executor/src/executorInt.c +++ b/source/libs/executor/src/executorInt.c @@ -655,6 +655,85 @@ int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPos return 0; } +int32_t doCopyToSDataBlockByHash(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, + SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, int32_t threshold, + bool ignoreGroup) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + + size_t keyLen = 0; + int32_t numOfRows = tSimpleHashGetSize(pHashmap); + + // begin from last iter + void* pData = pGroupResInfo->dataPos; + int32_t iter = pGroupResInfo->iter; + while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { + void* key = tSimpleHashGetKey(pData, &keyLen); + SResultRowPosition* pos = pData; + uint64_t groupId = *(uint64_t*)key; + + SFilePage* page = getBufPage(pBuf, pos->pageId); + if (page == NULL) { + qError("failed to get buffer, code:%s, %s", tstrerror(terrno), GET_TASKID(pTaskInfo)); + T_LONG_JMP(pTaskInfo->env, terrno); + } + + SResultRow* pRow = (SResultRow*)((char*)page + pos->offset); + + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); + + // no results, continue to check the next one + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + pGroupResInfo->iter = iter; + pGroupResInfo->dataPos = pData; + + releaseBufPage(pBuf, page); + continue; + } + + if (!ignoreGroup) { + if (pBlock->info.id.groupId == 0) { + pBlock->info.id.groupId = groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.id.groupId != groupId) { + releaseBufPage(pBuf, page); + break; + } + } + } + + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + uint32_t newSize = pBlock->info.rows + pRow->numOfRows + ((numOfRows - iter) > 1 ? 1 : 0); + blockDataEnsureCapacity(pBlock, newSize); + qDebug("datablock capacity not sufficient, expand to required:%d, current capacity:%d, %s", newSize, + pBlock->info.capacity, GET_TASKID(pTaskInfo)); + // todo set the pOperator->resultInfo size + } + + pGroupResInfo->index += 1; + pGroupResInfo->iter = iter; + pGroupResInfo->dataPos = pData; + + copyResultrowToDataBlock(pExprInfo, numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo); + + releaseBufPage(pBuf, page); + pBlock->info.rows += pRow->numOfRows; + if (pBlock->info.rows >= threshold) { + break; + } + } + + qDebug("%s result generated, rows:%" PRId64 ", groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows, + pBlock->info.id.groupId); + pBlock->info.dataLoad = 1; + blockDataUpdateTsWindow(pBlock, 0); + return 0; +} + int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo, int32_t threshold, bool ignoreGroup) { SExprInfo* pExprInfo = pSup->pExprInfo; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 467a49b37a..eb18278870 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -370,6 +370,72 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { return (pRes->info.rows == 0) ? NULL : pRes; } +bool hasRemainResultByHash(SOperatorInfo* pOperator) { + SGroupbyOperatorInfo* pInfo = pOperator->info; + SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable; + return pInfo->groupResInfo.index < tSimpleHashGetSize(pHashmap); +} + +void doBuildResultDatablockByHash(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf) { + SGroupbyOperatorInfo* pInfo = pOperator->info; + SSHashObj* pHashmap = pInfo->aggSup.pResultRowHashTable; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SSDataBlock* pBlock = pInfo->binfo.pRes; + + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResultByHash(pOperator)) { + return; + } + + pBlock->info.id.groupId = 0; + if (!pInfo->binfo.mergeResultBlock) { + doCopyToSDataBlockByHash(pTaskInfo, pBlock, &pOperator->exprSupp, pInfo->aggSup.pResultBuf, &pInfo->groupResInfo, + pHashmap, pOperator->resultInfo.threshold, false); + } else { + while (hasRemainResultByHash(pOperator)) { + doCopyToSDataBlockByHash(pTaskInfo, pBlock, &pOperator->exprSupp, pInfo->aggSup.pResultBuf, &pInfo->groupResInfo, + pHashmap, pOperator->resultInfo.threshold, true); + if (pBlock->info.rows >= pOperator->resultInfo.threshold) { + break; + } + pBlock->info.id.groupId = 0; + } + + // clear the group id info in SSDataBlock, since the client does not need it + pBlock->info.id.groupId = 0; + } +} + +static SSDataBlock* buildGroupResultDataBlockByHash(SOperatorInfo* pOperator) { + SGroupbyOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pRes = pInfo->binfo.pRes; + + // after filter, if result block turn to null, get next from whole set + while (1) { + doBuildResultDatablockByHash(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + + doFilter(pRes, pOperator->exprSupp.pFilterInfo, NULL); + if (!hasRemainResultByHash(pOperator)) { + setOperatorCompleted(pOperator); + // clean hash after completed + tSimpleHashCleanup(pInfo->aggSup.pResultRowHashTable); + pInfo->aggSup.pResultRowHashTable = NULL; + break; + } + if (pRes->info.rows > 0) { + break; + } + } + + pOperator->resultInfo.totalRows += pRes->info.rows; + return (pRes->info.rows == 0) ? NULL : pRes; +} + static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -379,9 +445,10 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; if (pOperator->status == OP_RES_TO_RETURN) { - return buildGroupResultDataBlock(pOperator); + return buildGroupResultDataBlockByHash(pOperator); } - + SGroupResInfo* pGroupResInfo = &pInfo->groupResInfo; + int32_t order = pInfo->binfo.inputTsOrder; int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; @@ -425,10 +492,20 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { } } #endif - initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0); + // initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, 0); + if (pGroupResInfo->pRows != NULL) { + taosArrayDestroy(pGroupResInfo->pRows); + } + if (pGroupResInfo->pBuf) { + taosMemoryFree(pGroupResInfo->pBuf); + pGroupResInfo->pBuf = NULL; + } + pGroupResInfo->index = 0; + pGroupResInfo->iter = 0; + pGroupResInfo->dataPos = NULL; pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; - return buildGroupResultDataBlock(pOperator); + return buildGroupResultDataBlockByHash(pOperator); } SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index efbc978323..c47e14ad0d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1035,7 +1035,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, } initLimitInfo(pScanNode->node.pLimit, pScanNode->node.pSlimit, &pInfo->base.limitInfo); - code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode); + code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3533,7 +3533,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN goto _error; } - code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode); + code = initQueryTableDataCond(&pInfo->base.cond, pTableScanNode, readHandle); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pInfo->base.matchInfo.pList); goto _error; diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 8bfa8e1a5d..0d82fcc9c3 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -373,8 +373,11 @@ void clearGroupResInfo(SGroupResInfo* pGroupResInfo) { if (pGroupResInfo->freeItem) { int32_t size = taosArrayGetSize(pGroupResInfo->pRows); for (int32_t i = pGroupResInfo->index; i < size; i++) { - void* pVal = taosArrayGetP(pGroupResInfo->pRows, i); - taosMemoryFree(pVal); + SRowBuffPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); + if (!pPos->needFree && !pPos->pRowBuff) { + taosMemoryFreeClear(pPos->pKey); + taosMemoryFree(pPos); + } } pGroupResInfo->freeItem = false; } @@ -2532,6 +2535,15 @@ void doStreamSessionSaveCheckpoint(SOperatorInfo* pOperator) { taosMemoryFree(buf); } +static void resetUnCloseSessionWinInfo(SSHashObj* winMap) { + void* pIte = NULL; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(winMap, pIte, &iter)) != NULL) { + SResultWindowInfo* pResInfo = pIte; + pResInfo->pStatePos->beUsed = true; + } +} + static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; @@ -2546,6 +2558,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (opRes) { return opRes; } + + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + setOperatorCompleted(pOperator); return NULL; } @@ -2583,6 +2601,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { + pInfo->recvGetAll = true; getAllSessionWindow(pAggSup->pResultRows, pInfo->pStUpdated); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { @@ -2838,6 +2857,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); pInfo->clearState = false; + pInfo->recvGetAll = false; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; // for stream void* buff = NULL; @@ -3454,6 +3475,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return resBlock; } + if (pInfo->recvGetAll) { + pInfo->recvGetAll = false; + resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); + } + setOperatorCompleted(pOperator); return NULL; } @@ -3482,6 +3508,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { + pInfo->recvGetAll = true; getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); continue; } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { @@ -3712,6 +3739,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys } pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); + pInfo->recvGetAll = false; // for stream void* buff = NULL; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fdbc0b4038..fbabef8fd3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1989,7 +1989,7 @@ static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { } if (para2Bytes <= 0 || para2Bytes > 4096) { // cast dst var type length limits to 4096 bytes return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, - "CAST function converted length should be in range [0, 4096] bytes"); + "CAST function converted length should be in range (0, 4096] bytes"); } // add database precision as param @@ -2772,7 +2772,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_cache_last", .type = FUNCTION_TYPE_CACHE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -3312,26 +3312,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = castFunction, .finalizeFunc = NULL }, - { - .name = "to_timestamp", - .type = FUNCTION_TYPE_TO_TIMESTAMP, - .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, - .translateFunc = translateToTimestamp, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = toTimestampFunction, - .finalizeFunc = NULL - }, - { - .name = "to_char", - .type = FUNCTION_TYPE_TO_CHAR, - .classification = FUNC_MGT_SCALAR_FUNC, - .translateFunc = translateToChar, - .getEnvFunc = NULL, - .initFunc = NULL, - .sprocessFunc = toCharFunction, - .finalizeFunc = NULL - }, { .name = "to_iso8601", .type = FUNCTION_TYPE_TO_ISO8601, @@ -3709,6 +3689,26 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .sprocessFunc = qVgIdFunction, .finalizeFunc = NULL }, + { + .name = "to_timestamp", + .type = FUNCTION_TYPE_TO_TIMESTAMP, + .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, + .translateFunc = translateToTimestamp, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toTimestampFunction, + .finalizeFunc = NULL + }, + { + .name = "to_char", + .type = FUNCTION_TYPE_TO_CHAR, + .classification = FUNC_MGT_SCALAR_FUNC, + .translateFunc = translateToChar, + .getEnvFunc = NULL, + .initFunc = NULL, + .sprocessFunc = toCharFunction, + .finalizeFunc = NULL + }, }; // clang-format on diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index c97c920a3b..c9b49ee30f 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1773,6 +1773,7 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) { static const char* jkLastRowScanPhysiPlanGroupTags = "GroupTags"; static const char* jkLastRowScanPhysiPlanGroupSort = "GroupSort"; +static const char* jkLastRowScanPhysiPlanTargets = "Targets"; static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) { const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj; @@ -1784,6 +1785,9 @@ static int32_t physiLastRowScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddBoolToObject(pJson, jkLastRowScanPhysiPlanGroupSort, pNode->groupSort); } + if (TSDB_CODE_SUCCESS == code) { + code = nodeListToJson(pJson, jkLastRowScanPhysiPlanTargets, pNode->pTargets); + } return code; } @@ -1798,6 +1802,9 @@ static int32_t jsonToPhysiLastRowScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBoolValue(pJson, jkLastRowScanPhysiPlanGroupSort, &pNode->groupSort); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeList(pJson, jkLastRowScanPhysiPlanTargets, &pNode->pTargets); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index 99100b2a1d..ea59d93d7f 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2052,7 +2052,8 @@ enum { PHY_LAST_ROW_SCAN_CODE_SCAN = 1, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, PHY_LAST_ROW_SCAN_CODE_GROUP_SORT, - PHY_LAST_ROW_SCAN_CODE_IGNULL + PHY_LAST_ROW_SCAN_CODE_IGNULL, + PHY_LAST_ROW_SCAN_CODE_TARGETS }; static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2068,6 +2069,9 @@ static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeBool(pEncoder, PHY_LAST_ROW_SCAN_CODE_IGNULL, pNode->ignoreNull); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_LAST_ROW_SCAN_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } return code; } @@ -2091,6 +2095,9 @@ static int32_t msgToPhysiLastRowScanNode(STlvDecoder* pDecoder, void* pObj) { case PHY_LAST_ROW_SCAN_CODE_IGNULL: code = tlvDecodeBool(pTlv, &pNode->ignoreNull); break; + case PHY_LAST_ROW_SCAN_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 4f6d3d95e1..ee22caf574 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1285,6 +1285,7 @@ void nodesDestroyNode(SNode* pNode) { SLastRowScanPhysiNode* pPhyNode = (SLastRowScanPhysiNode*)pNode; destroyScanPhysiNode((SScanPhysiNode*)pNode); nodesDestroyList(pPhyNode->pGroupTags); + nodesDestroyList(pPhyNode->pTargets); break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 23ce3aa77d..441f4da3b1 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -176,12 +176,15 @@ static int32_t calcConstStmtCondition(SCalcConstContext* pCxt, SNode** pCond, bo static EDealRes doFindAndReplaceNode(SNode** pNode, void* pContext) { SCalcConstContext* pCxt = pContext; if (pCxt->replaceCxt.pTarget == *pNode) { + char aliasName[TSDB_COL_NAME_LEN] = {0}; + strcpy(aliasName, ((SExprNode*)*pNode)->aliasName); nodesDestroyNode(*pNode); *pNode = nodesCloneNode(pCxt->replaceCxt.pNew); if (NULL == *pNode) { pCxt->code = TSDB_CODE_OUT_OF_MEMORY; return DEAL_RES_ERROR; } + strcpy(((SExprNode*)*pNode)->aliasName, aliasName); pCxt->replaceCxt.replaced = true; return DEAL_RES_END; @@ -211,7 +214,6 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d } char aliasName[TSDB_COL_NAME_LEN] = {0}; - strcpy(aliasName, ((SExprNode*)pProject)->aliasName); int32_t code = TSDB_CODE_SUCCESS; if (dual) { code = scalarCalculateConstantsFromDual(pProject, pNew); @@ -219,15 +221,20 @@ static int32_t calcConstProject(SCalcConstContext* pCxt, SNode* pProject, bool d code = scalarCalculateConstants(pProject, pNew); } if (TSDB_CODE_SUCCESS == code) { - strcpy(((SExprNode*)*pNew)->aliasName, aliasName); if (QUERY_NODE_VALUE == nodeType(*pNew) && NULL != pAssociation) { int32_t size = taosArrayGetSize(pAssociation); for (int32_t i = 0; i < size; ++i) { SAssociationNode* pAssNode = taosArrayGet(pAssociation, i); SNode** pCol = pAssNode->pPlace; if (*pCol == pAssNode->pAssociationNode) { + strcpy(aliasName, ((SExprNode*)*pCol)->aliasName); + SArray* pOrigAss = NULL; + TSWAP(((SExprNode*)*pCol)->pAssociation, pOrigAss); nodesDestroyNode(*pCol); *pCol = nodesCloneNode(*pNew); + TSWAP(pOrigAss, ((SExprNode*)*pCol)->pAssociation); + taosArrayDestroy(pOrigAss); + strcpy(((SExprNode*)*pCol)->aliasName, aliasName); if (NULL == *pCol) { code = TSDB_CODE_OUT_OF_MEMORY; break; @@ -285,7 +292,7 @@ static SNode* createConstantValue() { static int32_t calcConstProjections(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { SNode* pProj = NULL; WHERE_EACH(pProj, pSelect->pProjectionList) { - if (subquery && !pSelect->isDistinct && isUselessCol((SExprNode*)pProj)) { + if (subquery && !pSelect->isDistinct && !pSelect->tagScan && isUselessCol((SExprNode*)pProj)) { ERASE_NODE(pSelect->pProjectionList); continue; } diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 0d520b97f6..2b8516d37b 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -1584,8 +1584,7 @@ typedef union SRowsDataContext{ SStbRowsDataContext* pStbRowsCxt; } SRowsDataContext; -static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, - char* ctbName, bool* pFoundCtbName) { +static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* pStbRowsCxt, SToken* pToken, bool* pFoundCtbName) { *pFoundCtbName = false; int32_t code = checkAndTrimValue(pToken, pCxt->tmpTokenBuf, &pCxt->msg); if (code == TSDB_CODE_SUCCESS){ @@ -1595,7 +1594,13 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext* if (pToken->n > 0) { if (pToken->n <= TSDB_TABLE_NAME_LEN - 1) { - memcpy(pStbRowsCxt->ctbName.tname, pToken->z, pToken->n); + for (int i = 0; i < pToken->n; ++i) { + if (pToken->z[i] == '.') { + return buildInvalidOperationMsg(&pCxt->msg, "tbname can not contain '.'"); + } else { + pStbRowsCxt->ctbName.tname[i] = pToken->z[i]; + } + } pStbRowsCxt->ctbName.tname[pToken->n] = '\0'; *pFoundCtbName = true; } else { @@ -1677,8 +1682,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt* } } else if (pCols->pColIndex[i] == tbnameIdx) { - char ctbName[TSDB_TABLE_NAME_LEN]; - code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, ctbName, bFoundTbName); + code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, bFoundTbName); } if (code == TSDB_CODE_SUCCESS && i < pCols->numOfBound - 1) { @@ -1757,7 +1761,7 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod .requestId = pCxt->pComCxt->requestId, .requestObjRefId = pCxt->pComCxt->requestRid, .mgmtEps = pCxt->pComCxt->mgmtEpSet}; - code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStmt->targetTableName, &vg); + code = catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, &conn, &pStbRowsCxt->ctbName, &vg); if (code == TSDB_CODE_SUCCESS) { taosHashPut(pStmt->pVgroupsHashObj, (const char*)(&vg.vgId), sizeof(vg.vgId), &vg, sizeof(vg)); pStbRowsCxt->pCtbMeta->uid = taosHashGetSize(pStmt->pSubTableHashObj) + 1; diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 222aec9813..e0d154a130 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -266,7 +266,7 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols if (NULL == pScanCols) { if (NULL == pScanPseudoCols) { - return SCAN_TYPE_TABLE; + return (!tagScan) ? SCAN_TYPE_TABLE : SCAN_TYPE_TAG; } return FUNCTION_TYPE_BLOCK_DIST_INFO == ((SFunctionNode*)nodesListGetNode(pScanPseudoCols, 0))->funcType ? SCAN_TYPE_BLOCK_INFO diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8d4c042960..706fada4e8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2478,6 +2478,27 @@ static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast return false; } +/// @brief check if we can apply last row scan optimization +/// @param lastColNum how many distinct last col specified +/// @param lastColId only used when lastColNum equals 1, the col id of the only one last col +/// @param selectNonPKColNum num of normal cols +/// @param selectNonPKColId only used when selectNonPKColNum equals 1, the col id of the only one select col +static bool lastRowScanOptCheckColNum(int32_t lastColNum, col_id_t lastColId, + int32_t selectNonPKColNum, col_id_t selectNonPKColId) { + // multi select non pk col + last func: select c1, c2, last(c1) + if (selectNonPKColNum > 1 && lastColNum > 0) return false; + + if (selectNonPKColNum == 1) { + // select last(c1), last(c2), c1 ... + // which is not possible currently + if (lastColNum > 1) return false; + + // select last(c1), c2 ... + if (lastColNum == 1 && lastColId != selectNonPKColId) return false; + } + return true; +} + static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { @@ -2493,9 +2514,10 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { return false; } - bool hasLastFunc = false; - bool hasSelectFunc = false; - SNode* pFunc = NULL; + bool hasNonPKSelectFunc = false; + SNode* pFunc = NULL; + int32_t lastColNum = 0, selectNonPKColNum = 0; + col_id_t lastColId = -1, selectNonPKColId = -1; FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { SFunctionNode* pAggFunc = (SFunctionNode*)pFunc; if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { @@ -2505,16 +2527,33 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { if (pCol->colType != COLUMN_TYPE_COLUMN) { return false; } + if (lastColId != pCol->colId) { + lastColId = pCol->colId; + lastColNum++; + } } - if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { + if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { return false; } - hasLastFunc = true; + if (!lastRowScanOptCheckColNum(lastColNum, lastColId, selectNonPKColNum, selectNonPKColId)) + return false; } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) { - if (hasLastFunc) { + SNode* pParam = nodesListGetNode(pAggFunc->pParameterList, 0); + if (QUERY_NODE_COLUMN == nodeType(pParam)) { + SColumnNode* pCol = (SColumnNode*)pParam; + if (PRIMARYKEY_TIMESTAMP_COL_ID != pCol->colId) { + if (selectNonPKColId != pCol->colId) { + selectNonPKColId = pCol->colId; + selectNonPKColNum++; + } + } else { + continue; + } + } else if (lastColNum > 0) { return false; } - hasSelectFunc = true; + if (!lastRowScanOptCheckColNum(lastColNum, lastColId, selectNonPKColNum, selectNonPKColId)) + return false; } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) { return false; @@ -2581,6 +2620,9 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic SLastRowScanOptSetColDataTypeCxt cxt = {.doAgg = true, .pLastCols = NULL}; SNode* pNode = NULL; + SColumnNode* pPKTsCol = NULL; + SColumnNode* pNonPKCol = NULL; + FOREACH(pNode, pAgg->pAggFuncs) { SFunctionNode* pFunc = (SFunctionNode*)pNode; int32_t funcType = pFunc->funcType; @@ -2597,6 +2639,16 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic nodesWalkExpr(nodesListGetNode(pFunc->pParameterList, 0), lastRowScanOptSetColDataType, &cxt); nodesListErase(pFunc->pParameterList, nodesListGetCell(pFunc->pParameterList, 1)); } + } else if (FUNCTION_TYPE_SELECT_VALUE == funcType) { + pNode = nodesListGetNode(pFunc->pParameterList, 0); + if (nodeType(pNode) == QUERY_NODE_COLUMN) { + SColumnNode* pCol = (SColumnNode*)pNode; + if (pCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { + pPKTsCol = pCol; + } else { + pNonPKCol = pCol; + } + } } } @@ -2608,6 +2660,16 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true); nodesWalkExprs(pScan->pScanPseudoCols, lastRowScanOptSetColDataType, &cxt); lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false); + if (pPKTsCol && pScan->node.pTargets->length == 1) { + // when select last(ts),ts from ..., we add another ts to targets + sprintf(pPKTsCol->colName, "#sel_val.%p", pPKTsCol); + nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pPKTsCol)); + } + if (pNonPKCol && cxt.pLastCols->length == 1 && nodesEqualNode((SNode*)pNonPKCol, nodesListGetNode(cxt.pLastCols, 0))) { + // when select last(c1), c1 from ..., we add c1 to targets + sprintf(pNonPKCol->colName, "#sel_val.%p", pNonPKCol); + nodesListAppend(pScan->node.pTargets, nodesCloneNode((SNode*)pNonPKCol)); + } nodesClearList(cxt.pLastCols); } pAgg->hasLastRow = false; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index d6799a25a7..5cf3426e6f 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -552,6 +552,7 @@ static int32_t createLastRowScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSu if (NULL == pScan) { return TSDB_CODE_OUT_OF_MEMORY; } + pScan->pTargets = nodesCloneList(pScanLogicNode->node.pTargets); pScan->pGroupTags = nodesCloneList(pScanLogicNode->pGroupTags); if (NULL != pScanLogicNode->pGroupTags && NULL == pScan->pGroupTags) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index dbdd79cc65..e7c6297f44 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1230,7 +1230,6 @@ int32_t toTimestampFunction(SScalarParam* pInput, int32_t inputNum, SScalarParam code = taosChar2Ts(format, &formats, tsStr, &ts, precision, errMsg, 128); if (code) { qError("func to_timestamp failed %s", errMsg); - code = code == -1 ? TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR : TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR; break; } colDataSetVal(pOutput->columnData, i, (char *)&ts, false); diff --git a/source/libs/stream/inc/streamInt.h b/source/libs/stream/inc/streamInt.h index c63b51d745..b76a967d0d 100644 --- a/source/libs/stream/inc/streamInt.h +++ b/source/libs/stream/inc/streamInt.h @@ -106,7 +106,6 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); int32_t streamSaveAllTaskStatus(SStreamMeta* pMeta, int64_t checkpointId); -int32_t streamTaskBuildCheckpoint(SStreamTask* pTask); int32_t streamSendCheckMsg(SStreamTask* pTask, const SStreamTaskCheckReq* pReq, int32_t nodeId, SEpSet* pEpSet); int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t srcTaskId, int32_t index, int64_t checkpointId); @@ -127,13 +126,11 @@ int32_t streamNotifyUpstreamContinue(SStreamTask* pTask); int32_t streamTaskFillHistoryFinished(SStreamTask* pTask); int32_t streamTransferStateToStreamTask(SStreamTask* pTask); -int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate); +int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, const char*); STaskId streamTaskExtractKey(const SStreamTask* pTask); void streamTaskInitForLaunchHTask(SHistoryTaskInfo* pInfo); void streamTaskSetRetryInfoForLaunch(SHistoryTaskInfo* pInfo); -void streamMetaResetStartInfo(STaskStartInfo* pMeta); - SStreamQueue* streamQueueOpen(int64_t cap); void streamQueueClose(SStreamQueue* pQueue, int32_t taskId); void streamQueueProcessSuccess(SStreamQueue* queue); @@ -142,6 +139,21 @@ void* streamQueueNextItem(SStreamQueue* pQueue); void streamFreeQitem(SStreamQueueItem* data); int32_t streamQueueGetItemSize(const SStreamQueue* pQueue); +typedef enum UPLOAD_TYPE{ + UPLOAD_DISABLE = -1, + UPLOAD_S3 = 0, + UPLOAD_RSYNC = 1, +} UPLOAD_TYPE; + +UPLOAD_TYPE getUploadType(); +int uploadCheckpoint(char* id, char* path); +int downloadCheckpoint(char* id, char* path); +int deleteCheckpoint(char* id); +int deleteCheckpointFile(char* id, char* name); + +int32_t onNormalTaskReady(SStreamTask* pTask); +int32_t onScanhistoryTaskReady(SStreamTask* pTask); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/streamCheckpoint.c b/source/libs/stream/src/streamCheckpoint.c index 81840aaeb7..5479a2dab2 100644 --- a/source/libs/stream/src/streamCheckpoint.c +++ b/source/libs/stream/src/streamCheckpoint.c @@ -14,6 +14,8 @@ */ #include "streamInt.h" +#include "rsync.h" +#include "cos.h" int32_t tEncodeStreamCheckpointSourceReq(SEncoder* pEncoder, const SStreamCheckpointSourceReq* pReq) { if (tStartEncode(pEncoder) < 0) return -1; @@ -372,3 +374,91 @@ int32_t streamTaskBuildCheckpoint(SStreamTask* pTask) { return code; } + +static int uploadCheckpointToS3(char* id, char* path){ + TdDirPtr pDir = taosOpenDir(path); + if (pDir == NULL) return -1; + + TdDirEntryPtr de = NULL; + while ((de = taosReadDir(pDir)) != NULL) { + char* name = taosGetDirEntryName(de); + if (strcmp(name, ".") == 0 || strcmp(name, "..") == 0 || + taosDirEntryIsDir(de)) continue; + + char filename[PATH_MAX] = {0}; + if(path[strlen(path) - 1] == TD_DIRSEP_CHAR){ + snprintf(filename, sizeof(filename), "%s%s", path, name); + }else{ + snprintf(filename, sizeof(filename), "%s%s%s", path, TD_DIRSEP, name); + } + + char object[PATH_MAX] = {0}; + snprintf(object, sizeof(object), "%s%s%s", id, TD_DIRSEP, name); + + if(s3PutObjectFromFile2(filename, object) != 0){ + taosCloseDir(&pDir); + return -1; + } + stDebug("[s3] upload checkpoint:%s", filename); + } + taosCloseDir(&pDir); + + return 0; +} + +UPLOAD_TYPE getUploadType(){ + if(strlen(tsSnodeAddress) != 0){ + return UPLOAD_RSYNC; + }else if(tsS3StreamEnabled){ + return UPLOAD_S3; + }else{ + return UPLOAD_DISABLE; + } +} + +int uploadCheckpoint(char* id, char* path){ + if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ + stError("uploadCheckpoint parameters invalid"); + return -1; + } + if(strlen(tsSnodeAddress) != 0){ + return uploadRsync(id, path); + }else if(tsS3StreamEnabled){ + return uploadCheckpointToS3(id, path); + } + return 0; +} + +int downloadCheckpoint(char* id, char* path){ + if(id == NULL || path == NULL || strlen(id) == 0 || strlen(path) == 0 || strlen(path) >= PATH_MAX){ + stError("downloadCheckpoint parameters invalid"); + return -1; + } + if(strlen(tsSnodeAddress) != 0){ + return downloadRsync(id, path); + }else if(tsS3StreamEnabled){ + return s3GetObjectsByPrefix(id, path); + } + return 0; +} + +int deleteCheckpoint(char* id){ + if(id == NULL || strlen(id) == 0){ + stError("deleteCheckpoint parameters invalid"); + return -1; + } + if(strlen(tsSnodeAddress) != 0){ + return deleteRsync(id); + }else if(tsS3StreamEnabled){ + s3DeleteObjectsByPrefix(id); + } + return 0; +} + +int deleteCheckpointFile(char* id, char* name){ + char object[128] = {0}; + snprintf(object, sizeof(object), "%s/%s", id, name); + char *tmp = object; + s3DeleteObjects((const char**)&tmp, 1); + return 0; +} diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index edfc66762d..5665e7a917 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1007,7 +1007,6 @@ int32_t streamAddEndScanHistoryMsg(SStreamTask* pTask, SRpcHandleInfo* pRpcInfo, info.msg.info = *pRpcInfo; taosThreadMutexLock(&pTask->lock); - stDebug("s-task:%s lock", pTask->id.idStr); if (pTask->pRspMsgList == NULL) { pTask->pRspMsgList = taosArrayInit(4, sizeof(SStreamContinueExecInfo)); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 49f691c558..a6101b0932 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -18,7 +18,8 @@ // maximum allowed processed block batches. One block may include several submit blocks #define MAX_STREAM_EXEC_BATCH_NUM 32 #define STREAM_RESULT_DUMP_THRESHOLD 300 -#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) +#define STREAM_RESULT_DUMP_SIZE_THRESHOLD (1048576 * 1) // 1MiB result data +#define STREAM_SCAN_HISTORY_TIMESLICE 1000 // 1000 ms static int32_t streamDoTransferStateToStreamTask(SStreamTask* pTask); @@ -48,10 +49,9 @@ static int32_t doOutputResultBlockImpl(SStreamTask* pTask, SStreamDataBlock* pBl } streamDispatchStreamBlock(pTask); - return code; } - return 0; + return code; } static int32_t doDumpResult(SStreamTask* pTask, SStreamQueueItem* pItem, SArray* pRes, int32_t size, int64_t* totalSize, @@ -187,83 +187,118 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, SStreamQueueItem* pItem, i return code; } -int32_t streamScanHistoryData(SStreamTask* pTask) { +static int32_t handleResultBlocks(SStreamTask* pTask, SArray* pRes, int32_t size) { + int32_t code = TSDB_CODE_SUCCESS; + if (taosArrayGetSize(pRes) > 0) { + SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes); + code = doOutputResultBlockImpl(pTask, pStreamBlocks); + if (code != TSDB_CODE_SUCCESS) { + stDebug("s-task:%s dump fill-history results failed, code:%s", pTask->id.idStr, tstrerror(code)); + } + } else { + taosArrayDestroy(pRes); + } + return code; +} + +static void streamScanHistoryDataImpl(SStreamTask* pTask, SArray* pRes, int32_t* pSize, bool* pFinish) { + int32_t code = TSDB_CODE_SUCCESS; + void* exec = pTask->exec.pExecutor; + int32_t numOfBlocks = 0; + + while (1) { + if (streamTaskShouldStop(pTask)) { + break; + } + + if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { + stDebug("s-task:%s level:%d inputQ is blocked, retry in 5s", pTask->id.idStr, pTask->info.taskLevel); + break; + } + + SSDataBlock* output = NULL; + uint64_t ts = 0; + code = qExecTask(exec, &output, &ts); + if (code != TSDB_CODE_TSC_QUERY_KILLED && code != TSDB_CODE_SUCCESS) { + stError("s-task:%s scan-history data error occurred code:%s, continue scan-history", pTask->id.idStr, + tstrerror(code)); + continue; + } + + // the generated results before fill-history task been paused, should be dispatched to sink node + if (output == NULL) { + (*pFinish) = qStreamScanhistoryFinished(exec); + break; + } + + SSDataBlock block = {0}; + assignOneDataBlock(&block, output); + block.info.childId = pTask->info.selfChildId; + taosArrayPush(pRes, &block); + + (*pSize) += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); + numOfBlocks += 1; + + if (numOfBlocks >= STREAM_RESULT_DUMP_THRESHOLD || (*pSize) >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { + stDebug("s-task:%s scan exec numOfBlocks:%d, size:%.2fKiB output num-limit:%d, size-limit:%.2fKiB reached", + pTask->id.idStr, numOfBlocks, SIZE_IN_KiB(*pSize), STREAM_RESULT_DUMP_THRESHOLD, + SIZE_IN_KiB(STREAM_RESULT_DUMP_SIZE_THRESHOLD)); + break; + } + } +} + +SScanhistoryDataInfo streamScanHistoryData(SStreamTask* pTask, int64_t st) { ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); - int32_t code = TSDB_CODE_SUCCESS; void* exec = pTask->exec.pExecutor; bool finished = false; qSetStreamOpOpen(exec); - while (!finished) { + while (1) { if (streamTaskShouldPause(pTask)) { - double el = (taosGetTimestampMs() - pTask->execInfo.step1Start) / 1000.0; - stDebug("s-task:%s paused from the scan-history task, elapsed time:%.2fsec", pTask->id.idStr, el); - break; + stDebug("s-task:%s paused from the scan-history task", pTask->id.idStr); + // quit from step1, not continue to handle the step2 + return (SScanhistoryDataInfo){TASK_SCANHISTORY_QUIT, 0}; } SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + stError("s-task:%s scan-history prepare result block failed, code:%s, retry later", pTask->id.idStr, + tstrerror(terrno)); + continue; } int32_t size = 0; - int32_t numOfBlocks = 0; - while (1) { - if (streamTaskShouldStop(pTask)) { - taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - return 0; - } + streamScanHistoryDataImpl(pTask, pRes, &size, &finished); - if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { - stDebug("s-task:%s inputQ is blocked, wait for 10sec and retry", pTask->id.idStr); - taosMsleep(10000); - continue; - } - - SSDataBlock* output = NULL; - uint64_t ts = 0; - code = qExecTask(exec, &output, &ts); - if (code != TSDB_CODE_TSC_QUERY_KILLED && code != TSDB_CODE_SUCCESS) { - stError("%s scan-history data error occurred code:%s, continue scan", pTask->id.idStr, tstrerror(code)); - continue; - } - - // the generated results before fill-history task been paused, should be dispatched to sink node - if (output == NULL) { - finished = qStreamRecoverScanFinished(exec); - break; - } - - SSDataBlock block = {0}; - assignOneDataBlock(&block, output); - block.info.childId = pTask->info.selfChildId; - taosArrayPush(pRes, &block); - - size += blockDataGetSize(output) + sizeof(SSDataBlock) + sizeof(SColumnInfoData) * blockDataGetNumOfCols(&block); - - if ((++numOfBlocks) >= STREAM_RESULT_DUMP_THRESHOLD || size >= STREAM_RESULT_DUMP_SIZE_THRESHOLD) { - stDebug("s-task:%s scan exec numOfBlocks:%d, size:%.2fKiB output num-limit:%d, size-limit:%.2fKiB reached", - pTask->id.idStr, numOfBlocks, SIZE_IN_KiB(size), STREAM_RESULT_DUMP_THRESHOLD, - SIZE_IN_KiB(STREAM_RESULT_DUMP_SIZE_THRESHOLD)); - break; - } + if(streamTaskShouldStop(pTask)) { + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + return (SScanhistoryDataInfo){TASK_SCANHISTORY_QUIT, 0}; } - if (taosArrayGetSize(pRes) > 0) { - SStreamDataBlock* pStreamBlocks = createStreamBlockFromResults(NULL, pTask, size, pRes); - code = doOutputResultBlockImpl(pTask, pStreamBlocks); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } else { - taosArrayDestroy(pRes); + // dispatch the generated results + int32_t code = handleResultBlocks(pTask, pRes, size); + + int64_t el = taosGetTimestampMs() - st; + + // downstream task input queue is full, try in 5sec + if (pTask->inputq.status == TASK_INPUT_STATUS__BLOCKED) { + return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 5000}; + } + + if (finished) { + return (SScanhistoryDataInfo){TASK_SCANHISTORY_CONT, 0}; + } + + if (el >= STREAM_SCAN_HISTORY_TIMESLICE) { + stDebug("s-task:%s fill-history:%d time slice exhausted, elapsed time:%.2fs, retry in 100ms", + pTask->id.idStr, pTask->info.fillHistory, el / 1000.0); + return (SScanhistoryDataInfo){TASK_SCANHISTORY_REXEC, 100}; } } - - return 0; } // wait for the stream task to be idle @@ -273,7 +308,7 @@ static void waitForTaskIdle(SStreamTask* pTask, SStreamTask* pStreamTask) { int64_t st = taosGetTimestampMs(); while (!streamTaskIsIdle(pStreamTask)) { stDebug("s-task:%s level:%d wait for stream task:%s to be idle, check again in 100ms", id, pTask->info.taskLevel, - pStreamTask->id.idStr); + pStreamTask->id.idStr); taosMsleep(100); } @@ -647,23 +682,25 @@ int32_t streamExecTask(SStreamTask* pTask) { int32_t streamTaskReleaseState(SStreamTask* pTask) { stDebug("s-task:%s release exec state", pTask->id.idStr); void* pExecutor = pTask->exec.pExecutor; + + int32_t code = TSDB_CODE_SUCCESS; if (pExecutor != NULL) { - int32_t code = qStreamOperatorReleaseState(pExecutor); - return code; - } else { - return TSDB_CODE_SUCCESS; + code = qStreamOperatorReleaseState(pExecutor); } + + return code; } int32_t streamTaskReloadState(SStreamTask* pTask) { stDebug("s-task:%s reload exec state", pTask->id.idStr); void* pExecutor = pTask->exec.pExecutor; + + int32_t code = TSDB_CODE_SUCCESS; if (pExecutor != NULL) { - int32_t code = qStreamOperatorReloadState(pExecutor); - return code; - } else { - return TSDB_CODE_SUCCESS; + code = qStreamOperatorReloadState(pExecutor); } + + return code; } int32_t streamAlignTransferState(SStreamTask* pTask) { diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 76945f17a9..93dcd966b2 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -28,7 +28,6 @@ int32_t streamBackendId = 0; int32_t streamBackendCfWrapperId = 0; int32_t streamMetaId = 0; -static int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta); static void metaHbToMnode(void* param, void* tmrId); static void streamMetaClear(SStreamMeta* pMeta); static int32_t streamMetaBegin(SStreamMeta* pMeta); @@ -150,6 +149,12 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->startInfo.pReadyTaskSet = taosHashInit(64, fp, false, HASH_NO_LOCK); if (pMeta->startInfo.pReadyTaskSet == NULL) { + goto _err; + } + + pMeta->startInfo.pFailedTaskSet = taosHashInit(4, fp, false, HASH_NO_LOCK); + if (pMeta->startInfo.pFailedTaskSet == NULL) { + goto _err; } pMeta->pHbInfo = taosMemoryCalloc(1, sizeof(SMetaHbInfo)); @@ -185,10 +190,10 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); pMeta->chkpSaved = taosArrayInit(4, sizeof(int64_t)); pMeta->chkpInUse = taosArrayInit(4, sizeof(int64_t)); - pMeta->chkpCap = 8; + pMeta->chkpCap = 2; taosInitRWLatch(&pMeta->chkpDirLock); - pMeta->chkpId = streamGetLatestCheckpointId(pMeta); + pMeta->chkpId = streamMetaGetLatestCheckpointId(pMeta); pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId); while (pMeta->streamBackend == NULL) { taosMsleep(100); @@ -221,6 +226,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF if (pMeta->pHbInfo) taosMemoryFreeClear(pMeta->pHbInfo); if (pMeta->updateInfo.pTasks) taosHashCleanup(pMeta->updateInfo.pTasks); if (pMeta->startInfo.pReadyTaskSet) taosHashCleanup(pMeta->startInfo.pReadyTaskSet); + if (pMeta->startInfo.pFailedTaskSet) taosHashCleanup(pMeta->startInfo.pFailedTaskSet); taosMemoryFree(pMeta); stError("failed to open stream meta"); @@ -228,12 +234,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF } int32_t streamMetaReopen(SStreamMeta* pMeta) { - // backup the restart flag - int32_t restartFlag = pMeta->startInfo.startAllTasksFlag; streamMetaClear(pMeta); - pMeta->startInfo.startAllTasksFlag = restartFlag; - // NOTE: role should not be changed during reopen meta pMeta->streamBackendRid = -1; pMeta->streamBackend = NULL; @@ -302,7 +304,10 @@ void streamMetaClear(SStreamMeta* pMeta) { pMeta->numOfPausedTasks = 0; pMeta->chkptNotReadyTasks = 0; - streamMetaResetStartInfo(&pMeta->startInfo); + // the willrestart/starting flag can NOT be cleared + taosHashClear(pMeta->startInfo.pReadyTaskSet); + taosHashClear(pMeta->startInfo.pFailedTaskSet); + pMeta->startInfo.readyTs = 0; } void streamMetaClose(SStreamMeta* pMeta) { @@ -342,6 +347,7 @@ void streamMetaCloseImpl(void* arg) { taosHashCleanup(pMeta->pTaskBackendUnique); taosHashCleanup(pMeta->updateInfo.pTasks); taosHashCleanup(pMeta->startInfo.pReadyTaskSet); + taosHashCleanup(pMeta->startInfo.pFailedTaskSet); taosMemoryFree(pMeta->pHbInfo); taosMemoryFree(pMeta->path); @@ -595,7 +601,7 @@ int32_t streamMetaCommit(SStreamMeta* pMeta) { return 0; } -int64_t streamGetLatestCheckpointId(SStreamMeta* pMeta) { +int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta) { int64_t chkpId = 0; TBC* pCur = NULL; @@ -1093,8 +1099,11 @@ void streamMetaInitForSnode(SStreamMeta* pMeta) { void streamMetaResetStartInfo(STaskStartInfo* pStartInfo) { taosHashClear(pStartInfo->pReadyTaskSet); - pStartInfo->startAllTasksFlag = 0; + taosHashClear(pStartInfo->pFailedTaskSet); + pStartInfo->tasksWillRestart = 0; pStartInfo->readyTs = 0; + // reset the sentinel flag value to be 0 + atomic_store_32(&pStartInfo->taskStarting, 0); } void streamMetaRLock(SStreamMeta* pMeta) { @@ -1104,7 +1113,6 @@ void streamMetaRLock(SStreamMeta* pMeta) { void streamMetaRUnLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-runlock", pMeta->vgId); taosRUnLockLatch(&pMeta->lock); - } void streamMetaWLock(SStreamMeta* pMeta) { stTrace("vgId:%d meta-wlock", pMeta->vgId); diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index eae4605dbc..63ee702ada 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -159,7 +159,8 @@ int32_t streamTaskGetDataFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu // no available token in bucket for sink task, let's wait for a little bit if (taskLevel == TASK_LEVEL__SINK && (!streamTaskExtractAvailableToken(pTask->outputInfo.pTokenBucket, pTask->id.idStr))) { - stDebug("s-task:%s no available token in bucket for sink data, wait for 50ms", id); + stDebug("s-task:%s no available token in bucket for sink data, wait for 10ms", id); + taosMsleep(10); return TSDB_CODE_SUCCESS; } @@ -340,10 +341,11 @@ int32_t streamTaskPutDataIntoInputQ(SStreamTask* pTask, SStreamQueueItem* pItem) return 0; } -// the result should be put into the outputQ in any cases, otherwise, the result may be lost +// the result should be put into the outputQ in any cases, the result may be lost otherwise. int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBlock) { STaosQueue* pQueue = pTask->outputq.queue->pQueue; + // wait for the output queue is available for new data to dispatch while (streamQueueIsFull(pTask->outputq.queue)) { if (streamTaskShouldStop(pTask)) { stInfo("s-task:%s discard result block due to task stop", pTask->id.idStr); @@ -373,7 +375,8 @@ int32_t streamTaskPutDataIntoOutputQ(SStreamTask* pTask, SStreamDataBlock* pBloc return TSDB_CODE_SUCCESS; } -int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate) { +int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t numRate, float quotaRate, + const char* id) { if (numCap < 10 || numRate < 10 || pBucket == NULL) { stError("failed to init sink task bucket, cap:%d, rate:%d", numCap, numRate); return TSDB_CODE_INVALID_PARA; @@ -388,6 +391,7 @@ int32_t streamTaskInitTokenBucket(STokenBucket* pBucket, int32_t numCap, int32_t pBucket->quotaRemain = pBucket->quotaCapacity; pBucket->fillTimestamp = taosGetTimestampMs(); + stDebug("s-task:%s sink quotaRate:%.2fMiB, numRate:%d", id, quotaRate, numRate); return TSDB_CODE_SUCCESS; } @@ -406,12 +410,12 @@ static void fillTokenBucket(STokenBucket* pBucket, const char* id) { double incSize = (delta / 1000.0) * pBucket->quotaRate; if (incSize > 0) { pBucket->quotaRemain = TMIN(pBucket->quotaRemain + incSize, pBucket->quotaCapacity); + pBucket->fillTimestamp = now; } if (incNum > 0 || incSize > 0) { - stDebug("new token and capacity available, current token:%d inc:%d, current quota:%.2fMiB inc:%.2fMiB, ts:%" PRId64 - " idle for %.2f Sec, %s", - pBucket->numOfToken, incNum, pBucket->quotaRemain, incSize, now, delta / 1000.0, id); + stTrace("token/quota available, token:%d inc:%d, quota:%.2fMiB inc:%.3fMiB, ts:%" PRId64 " idle:%" PRId64 "ms, %s", + pBucket->numOfToken, incNum, pBucket->quotaRemain, incSize, now, delta, id); } } diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 3de5de9967..103fd9e087 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -19,7 +19,6 @@ #include "streamBackendRocksdb.h" #include "streamInt.h" #include "tcommon.h" -#include "streamInt.h" enum SBackendFileType { ROCKSDB_OPTIONS_TYPE = 1, @@ -52,6 +51,7 @@ struct SStreamSnapHandle { int8_t filetype; SArray* pFileList; int32_t currFileIdx; + int8_t delFlag; // 0 : not del, 1: del }; struct SStreamSnapBlockHdr { int8_t type; @@ -148,6 +148,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk taosMemoryFree(tdir); return code; } + pHandle->delFlag = 1; chkpId = 0; } @@ -193,8 +194,8 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk taosArrayPush(pFile->pSst, &sst); } } - { - char* buf = taosMemoryCalloc(1, 512); + if (qDebugFlag & DEBUG_TRACE) { + char* buf = taosMemoryCalloc(1, 128 + taosArrayGetSize(pFile->pSst) * 16); sprintf(buf, "[current: %s,", pFile->pCurrent); sprintf(buf + strlen(buf), "MANIFEST: %s,", pFile->pMainfest); sprintf(buf + strlen(buf), "options: %s,", pFile->pOptions); @@ -274,7 +275,7 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) { if (handle->checkpointId == 0) { // del tmp dir if (pFile && taosIsDir(pFile->path)) { - taosRemoveDir(pFile->path); + if (handle->delFlag) taosRemoveDir(pFile->path); } } else { streamBackendDelInUseChkp(handle->handle, handle->checkpointId); @@ -344,10 +345,10 @@ int32_t streamSnapRead(SStreamSnapReader* pReader, uint8_t** ppData, int64_t* si stDebug("%s start to read file %s, current offset:%" PRId64 ", size:%" PRId64 ", file no.%d", STREAM_STATE_TRANSFER, item->name, (int64_t)pHandle->offset, item->size, pHandle->currFileIdx); uint8_t* buf = taosMemoryCalloc(1, sizeof(SStreamSnapBlockHdr) + kBlockSize); - if(buf == NULL){ + if (buf == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); + int64_t nread = taosPReadFile(pHandle->fd, buf + sizeof(SStreamSnapBlockHdr), kBlockSize, pHandle->offset); if (nread == -1) { taosMemoryFree(buf); code = TAOS_SYSTEM_ERROR(terrno); @@ -423,6 +424,7 @@ int32_t streamSnapWriterOpen(void* pMeta, int64_t sver, int64_t ever, char* path pHandle->pFileList = list; pHandle->currFileIdx = 0; pHandle->offset = 0; + pHandle->delFlag = 0; *ppWriter = pWriter; return 0; @@ -480,8 +482,8 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa } int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) { SStreamSnapHandle* handle = &pWriter->handle; - if (qDebugFlag & DEBUG_DEBUG) { - char* buf = (char*)taosMemoryMalloc(1024); + if (qDebugFlag & DEBUG_TRACE) { + char* buf = (char*)taosMemoryMalloc(128 + taosArrayGetSize(handle->pFileList) * 16); int n = sprintf(buf, "["); for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { SBackendFileItem* item = taosArrayGet(handle->pFileList, i); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 5ebc60dc13..e672b256da 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -19,6 +19,10 @@ #include "wal.h" #include "streamsm.h" +#define SCANHISTORY_IDLE_TIME_SLICE 100 // 100ms +#define SCANHISTORY_MAX_IDLE_TIME 10 // 10 sec +#define SCANHISTORY_IDLE_TICK ((SCANHISTORY_MAX_IDLE_TIME * 1000) / SCANHISTORY_IDLE_TIME_SLICE) + typedef struct SLaunchHTaskInfo { SStreamMeta* pMeta; STaskId id; @@ -30,6 +34,12 @@ typedef struct STaskRecheckInfo { void* checkTimer; } STaskRecheckInfo; +typedef struct STaskInitTs { + int64_t start; + int64_t end; + bool success; +} STaskInitTs; + static int32_t streamSetParamForScanHistory(SStreamTask* pTask); static void streamTaskSetRangeStreamCalc(SStreamTask* pTask); static int32_t initScanHistoryReq(SStreamTask* pTask, SStreamScanHistoryReq* pReq, int8_t igUntreated); @@ -57,7 +67,7 @@ int32_t streamTaskSetReady(SStreamTask* pTask) { stDebug("s-task:%s all %d downstream ready, init completed, elapsed time:%" PRId64 "ms, task status:%s", pTask->id.idStr, numOfDowns, el, p); - streamMetaUpdateTaskReadyInfo(pTask); + streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, pTask->execInfo.start, true); return TSDB_CODE_SUCCESS; } @@ -81,6 +91,60 @@ int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated) { return 0; } +static void doReExecScanhistory(void* param, void* tmrId) { + SStreamTask* pTask = param; + pTask->schedHistoryInfo.numOfTicks -= 1; + + char* p = NULL; + ETaskStatus status = streamTaskGetStatus(pTask, &p); + if (status == TASK_STATUS__DROPPING || status == TASK_STATUS__STOP) { + streamMetaReleaseTask(pTask->pMeta, pTask); + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s status:%s not start scan-history again, ref:%d", pTask->id.idStr, p, ref); + return; + } + + if (pTask->schedHistoryInfo.numOfTicks <= 0) { + streamStartScanHistoryAsync(pTask, 0); + + int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s fill-history:%d start scan-history data, out of tmr, ref:%d", pTask->id.idStr, + pTask->info.fillHistory, ref); + + // release the task. + streamMetaReleaseTask(pTask->pMeta, pTask); + } else { + taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer, + &pTask->schedHistoryInfo.pTimer); + } +} + +int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration) { + int32_t numOfTicks = idleDuration / SCANHISTORY_IDLE_TIME_SLICE; + if (numOfTicks <= 0) { + numOfTicks = 1; + } else if (numOfTicks > SCANHISTORY_IDLE_TICK) { + numOfTicks = SCANHISTORY_IDLE_TICK; + } + + // add ref for task + SStreamTask* p = streamMetaAcquireTask(pTask->pMeta, pTask->id.streamId, pTask->id.taskId); + ASSERT(p != NULL); + + pTask->schedHistoryInfo.numOfTicks = numOfTicks; + + int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); + stDebug("s-task:%s scan-history start in %.2fs, ref:%d", pTask->id.idStr, numOfTicks*0.1, ref); + + if (pTask->schedHistoryInfo.pTimer == NULL) { + pTask->schedHistoryInfo.pTimer = taosTmrStart(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer); + } else { + taosTmrReset(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer, &pTask->schedHistoryInfo.pTimer); + } + + return TSDB_CODE_SUCCESS; +} + static int32_t doStartScanHistoryTask(SStreamTask* pTask) { SVersionRange* pRange = &pTask->dataRange.range; if (pTask->info.fillHistory) { @@ -318,6 +382,31 @@ void doProcessDownstreamReadyRsp(SStreamTask* pTask) { streamTaskOnHandleEventSuccess(pTask->status.pSM, event); } +static void addIntoNodeUpdateList(SStreamTask* pTask, int32_t nodeId) { + int32_t vgId = pTask->pMeta->vgId; + + taosThreadMutexLock(&pTask->lock); + int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList); + bool existed = false; + for (int i = 0; i < num; ++i) { + SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, i); + if (p->nodeId == nodeId) { + existed = true; + break; + } + } + + if (!existed) { + SDownstreamTaskEpset t = {.nodeId = nodeId}; + taosArrayPush(pTask->outputInfo.pDownstreamUpdateList, &t); + + stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", pTask->id.idStr, vgId, + t.nodeId, (int32_t)taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList)); + } + + taosThreadMutexUnlock(&pTask->lock); +} + int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp) { ASSERT(pTask->id.taskId == pRsp->upstreamTaskId); const char* id = pTask->id.idStr; @@ -367,40 +456,23 @@ int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRs doProcessDownstreamReadyRsp(pTask); } } else { // not ready, wait for 100ms and retry - if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) { - stError( - "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, " - "not check wait for downstream task nodeUpdate, and all tasks restart", - id, pRsp->upstreamNodeId, pRsp->oldStage, (int32_t)pTask->pMeta->stage); - } else { - if (pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) { + if (pRsp->status == TASK_UPSTREAM_NEW_STAGE || pRsp->status == TASK_DOWNSTREAM_NOT_LEADER) { + if (pRsp->status == TASK_UPSTREAM_NEW_STAGE) { + stError( + "s-task:%s vgId:%d self vnode-transfer/leader-change/restart detected, old stage:%d, current stage:%d, " + "not check wait for downstream task nodeUpdate, and all tasks restart", + id, pRsp->upstreamNodeId, pRsp->oldStage, (int32_t)pTask->pMeta->stage); + } else { stError( "s-task:%s downstream taskId:0x%x (vgId:%d) not leader, self dispatch epset needs to be updated, not check " "downstream again, nodeUpdate needed", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); - - taosThreadMutexLock(&pTask->lock); - int32_t num = taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList); - bool existed = false; - for (int i = 0; i < num; ++i) { - SDownstreamTaskEpset* p = taosArrayGet(pTask->outputInfo.pDownstreamUpdateList, i); - if (p->nodeId == pRsp->downstreamNodeId) { - existed = true; - break; - } - } - - if (!existed) { - SDownstreamTaskEpset t = {.nodeId = pRsp->downstreamNodeId}; - taosArrayPush(pTask->outputInfo.pDownstreamUpdateList, &t); - stInfo("s-task:%s vgId:%d downstream nodeId:%d needs to be updated, total needs updated:%d", id, vgId, - t.nodeId, (int32_t)taosArrayGetSize(pTask->outputInfo.pDownstreamUpdateList)); - } - - taosThreadMutexUnlock(&pTask->lock); - return 0; } + addIntoNodeUpdateList(pTask, pRsp->downstreamNodeId); + streamMetaUpdateTaskDownstreamStatus(pTask, pTask->execInfo.init, taosGetTimestampMs(), false); + + } else { // TASK_DOWNSTREAM_NOT_READY, let's retry in 100ms STaskRecheckInfo* pInfo = createRecheckInfo(pTask, pRsp); int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); @@ -676,9 +748,8 @@ static void tryLaunchHistoryTask(void* param, void* tmrId) { int32_t hTaskId = pHTaskInfo->id.taskId; streamTaskGetStatus(pTask, &p); - stDebug( - "s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", - pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); + stDebug("s-task:%s status:%s failed to launch fill-history task:0x%x, retry launch:%dms, retryCount:%d", + pTask->id.idStr, p, hTaskId, pHTaskInfo->waitInterval, pHTaskInfo->retryTimes); taosTmrReset(tryLaunchHistoryTask, LAUNCH_HTASK_INTERVAL, pInfo, streamEnv.timer, &pHTaskInfo->pTimer); streamMetaReleaseTask(pMeta, pTask); @@ -975,28 +1046,57 @@ void streamTaskEnablePause(SStreamTask* pTask) { pTask->status.pauseAllowed = 1; } -int32_t streamMetaUpdateTaskReadyInfo(SStreamTask* pTask) { +static void displayStatusInfo(SStreamMeta* pMeta, SHashObj* pTaskSet, bool succ) { + int32_t vgId = pMeta->vgId; + void* pIter = NULL; + size_t keyLen = 0; + + stInfo("vgId:%d %d tasks check-downstream completed %s", vgId, taosHashGetSize(pTaskSet), + succ ? "success" : "failed"); + + while ((pIter = taosHashIterate(pTaskSet, pIter)) != NULL) { + STaskInitTs* pInfo = pIter; + void* key = taosHashGetKey(pIter, &keyLen); + + SStreamTask** pTask1 = taosHashGet(pMeta->pTasksMap, key, sizeof(STaskId)); + if (pTask1 == NULL) { + stInfo("s-task:0x%x is dropped already, %s", (int32_t)((STaskId*)key)->taskId, succ ? "success" : "failed"); + } else { + stInfo("s-task:%s level:%d vgId:%d, init:%" PRId64 ", initEnd:%" PRId64 ", %s", (*pTask1)->id.idStr, + (*pTask1)->info.taskLevel, vgId, pInfo->start, pInfo->end, pInfo->success ? "success" : "failed"); + } + } +} + +int32_t streamMetaUpdateTaskDownstreamStatus(SStreamTask* pTask, int64_t startTs, int64_t endTs, bool ready) { SStreamMeta* pMeta = pTask->pMeta; streamMetaWLock(pMeta); STaskId id = streamTaskExtractKey(pTask); - taosHashPut(pMeta->startInfo.pReadyTaskSet, &id, sizeof(id), NULL, 0); + STaskStartInfo* pStartInfo = &pMeta->startInfo; + + SHashObj* pDst = ready? pStartInfo->pReadyTaskSet:pStartInfo->pFailedTaskSet; + + STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready}; + taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs)); int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta); - if (taosHashGetSize(pMeta->startInfo.pReadyTaskSet) == numOfTotal) { - STaskStartInfo* pStartInfo = &pMeta->startInfo; - + if (taosHashGetSize(pStartInfo->pReadyTaskSet) + taosHashGetSize(pStartInfo->pFailedTaskSet) == numOfTotal) { pStartInfo->readyTs = pTask->execInfo.start; pStartInfo->elapsedTime = (pStartInfo->startTs != 0) ? pStartInfo->readyTs - pStartInfo->startTs : 0; - streamMetaResetStartInfo(pStartInfo); - - stDebug("vgId:%d all %d task(s) are started successfully, last ready task:%s level:%d, startTs:%" PRId64 + stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:%s level:%d, startTs:%" PRId64 ", readyTs:%" PRId64 " total elapsed time:%.2fs", pMeta->vgId, numOfTotal, pTask->id.idStr, pTask->info.taskLevel, pStartInfo->startTs, pStartInfo->readyTs, pStartInfo->elapsedTime / 1000.0); + + // print the initialization elapsed time and info + displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true); + displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); + + streamMetaResetStartInfo(pStartInfo); } streamMetaWUnLock(pMeta); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index fb0090ec6d..6ca7bc5e7b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -221,7 +221,6 @@ SStreamState* streamStateOpen(char* path, void* pTask, bool specPath, int32_t sz } pState->pTdbState->pOwner = pTask; - pState->checkPointId = 0; return pState; @@ -274,7 +273,6 @@ int32_t streamStateCommit(SStreamState* pState) { SStreamSnapshot* pShot = getSnapshot(pState->pFileState); flushSnapshot(pState->pFileState, pShot, true); } - pState->checkPointId++; return 0; #else if (tdbCommit(pState->pTdbState->db, pState->pTdbState->txn) < 0) { @@ -288,7 +286,6 @@ int32_t streamStateCommit(SStreamState* pState) { TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } - pState->checkPointId++; return 0; #endif } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index f949d46315..a7fb590d1b 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -400,8 +400,7 @@ void tFreeStreamTask(SStreamTask* pTask) { taosMemoryFree(pTask->outputInfo.pTokenBucket); taosThreadMutexDestroy(&pTask->lock); - taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList); - pTask->outputInfo.pDownstreamUpdateList = NULL; + pTask->outputInfo.pDownstreamUpdateList = taosArrayDestroy(pTask->outputInfo.pDownstreamUpdateList); taosMemoryFree(pTask); stDebug("s-task:0x%x free task completed", taskId); @@ -447,7 +446,7 @@ int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, i // 2MiB per second for sink task // 50 times sink operator per second - streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate); + streamTaskInitTokenBucket(pTask->outputInfo.pTokenBucket, 50, 50, tsSinkDataRate, pTask->id.idStr); TdThreadMutexAttr attr = {0}; int code = taosThreadMutexAttrInit(&attr); diff --git a/source/libs/stream/src/streamTaskSm.c b/source/libs/stream/src/streamTaskSm.c index 09656cbe97..04b449aaaf 100644 --- a/source/libs/stream/src/streamTaskSm.c +++ b/source/libs/stream/src/streamTaskSm.c @@ -315,7 +315,6 @@ int32_t streamTaskOnHandleEventSuccess(SStreamTaskSM* pSM, EStreamTaskEvent even GET_EVT_NAME(event), pSM->current.name, GET_EVT_NAME(pSM->prev.evt)); taosThreadMutexUnlock(&pTask->lock); - stDebug("s-task:%s unlockx", pTask->id.idStr); return TSDB_CODE_STREAM_INVALID_STATETRANS; } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index 584e81fafc..fc47498a3c 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -27,6 +27,9 @@ #define DEFAULT_MAX_STREAM_BUFFER_SIZE (128 * 1024 * 1024) #define MIN_NUM_OF_ROW_BUFF 10240 +#define TASK_KEY "streamFileState" +#define STREAM_STATE_INFO_NAME "StreamStateCheckPoint" + struct SStreamFileState { SList* usedBuffs; SList* freeBuffs; @@ -113,6 +116,15 @@ void* sessionCreateStateKey(SRowBuffPos* pPos, int64_t num) { return pStateKey; } +static void streamFileStateDecode(TSKEY* pKey, void* pBuff, int32_t len) { pBuff = taosDecodeFixedI64(pBuff, pKey); } + +static void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) { + *pLen = sizeof(TSKEY); + (*pVal) = taosMemoryCalloc(1, *pLen); + void* buff = *pVal; + taosEncodeFixedI64(&buff, *pKey); +} + SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_t rowSize, uint32_t selectRowSize, GetTsFun fp, void* pFile, TSKEY delMark, const char* taskId, int64_t checkpointId, int8_t type) { @@ -181,6 +193,15 @@ SStreamFileState* streamFileStateInit(int64_t memSize, uint32_t keySize, uint32_ recoverSesssion(pFileState, checkpointId); } + void* valBuf = NULL; + int32_t len = 0; + int32_t code = streamDefaultGet_rocksdb(pFileState->pFileStore, STREAM_STATE_INFO_NAME, &valBuf, &len); + if (code == TSDB_CODE_SUCCESS) { + ASSERT(len == sizeof(TSKEY)); + streamFileStateDecode(&pFileState->flushMark, valBuf, len); + qDebug("===stream===flushMark read:%" PRId64, pFileState->flushMark); + } + taosMemoryFreeClear(valBuf); return pFileState; _error: @@ -506,15 +527,6 @@ SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) { return pFileState->usedBuffs; } -void streamFileStateDecode(TSKEY* pKey, void* pBuff, int32_t len) { pBuff = taosDecodeFixedI64(pBuff, pKey); } - -void streamFileStateEncode(TSKEY* pKey, void** pVal, int32_t* pLen) { - *pLen = sizeof(TSKEY); - (*pVal) = taosMemoryCalloc(1, *pLen); - void* buff = *pVal; - taosEncodeFixedI64(&buff, *pKey); -} - int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, bool flushState) { int32_t code = TSDB_CODE_SUCCESS; SListIter iter = {0}; @@ -538,6 +550,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, continue; } pPos->beFlushed = true; + pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); qDebug("===stream===flushed start:%" PRId64, pFileState->getTs(pPos->pKey)); if (streamStateGetBatchSize(batch) >= BATCH_LIMIT) { @@ -565,24 +578,12 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, pFileState->id, numOfElems, BATCH_LIMIT, elapsed); if (flushState) { - const char* taskKey = "streamFileState"; - { - char keyBuf[128] = {0}; - void* valBuf = NULL; - int32_t len = 0; - sprintf(keyBuf, "%s:%" PRId64 "", taskKey, ((SStreamState*)pFileState->pFileStore)->checkPointId); - streamFileStateEncode(&pFileState->flushMark, &valBuf, &len); - streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0); - taosMemoryFree(valBuf); - } - { - char keyBuf[128] = {0}; - char valBuf[64] = {0}; - int32_t len = 0; - memcpy(keyBuf, taskKey, strlen(taskKey)); - len = sprintf(valBuf, "%" PRId64 "", ((SStreamState*)pFileState->pFileStore)->checkPointId); - code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0); - } + void* valBuf = NULL; + int32_t len = 0; + streamFileStateEncode(&pFileState->flushMark, &valBuf, &len); + qDebug("===stream===flushMark write:%" PRId64, pFileState->flushMark); + streamStatePutBatch(pFileState->pFileStore, "default", batch, STREAM_STATE_INFO_NAME, valBuf, len, 0); + taosMemoryFree(valBuf); streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); } @@ -591,26 +592,23 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, } int32_t forceRemoveCheckpoint(SStreamFileState* pFileState, int64_t checkpointId) { - const char* taskKey = "streamFileState"; char keyBuf[128] = {0}; - sprintf(keyBuf, "%s:%" PRId64 "", taskKey, checkpointId); + sprintf(keyBuf, "%s:%" PRId64 "", TASK_KEY, checkpointId); return streamDefaultDel_rocksdb(pFileState->pFileStore, keyBuf); } int32_t getSnapshotIdList(SStreamFileState* pFileState, SArray* list) { - const char* taskKey = "streamFileState"; - return streamDefaultIterGet_rocksdb(pFileState->pFileStore, taskKey, NULL, list); + return streamDefaultIterGet_rocksdb(pFileState->pFileStore, TASK_KEY, NULL, list); } int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t code = TSDB_CODE_SUCCESS; - const char* taskKey = "streamFileState"; int64_t maxCheckPointId = 0; { char buf[128] = {0}; void* val = NULL; int32_t len = 0; - memcpy(buf, taskKey, strlen(taskKey)); + memcpy(buf, TASK_KEY, strlen(TASK_KEY)); code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len); if (code != 0 || len == 0 || val == NULL) { return TSDB_CODE_FAILED; @@ -624,7 +622,7 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { char buf[128] = {0}; void* val = 0; int32_t len = 0; - sprintf(buf, "%s:%" PRId64 "", taskKey, i); + sprintf(buf, "%s:%" PRId64 "", TASK_KEY, i); code = streamDefaultGet_rocksdb(pFileState->pFileStore, buf, &val, &len); if (code != 0) { return TSDB_CODE_FAILED; diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index 629b04ae51..d756b71e29 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -18,6 +18,17 @@ TARGET_INCLUDE_DIRECTORIES( PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" ) +ADD_EXECUTABLE(checkpointTest checkpointTest.cpp) +TARGET_LINK_LIBRARIES( + checkpointTest + PUBLIC os common gtest stream executor qcom index transport util +) + +TARGET_INCLUDE_DIRECTORIES( + checkpointTest + PRIVATE "${TD_SOURCE_DIR}/source/libs/stream/inc" +) + add_test( NAME streamUpdateTest COMMAND streamUpdateTest diff --git a/source/libs/stream/test/checkpointTest.cpp b/source/libs/stream/test/checkpointTest.cpp new file mode 100644 index 0000000000..dca2e97c28 --- /dev/null +++ b/source/libs/stream/test/checkpointTest.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" + +#include "rsync.h" +#include "streamInt.h" +#include "cos.h" + +int main(int argc, char **argv) { + testing::InitGoogleTest(&argc, argv); + + if (taosInitCfg("/etc/taos/", NULL, NULL, NULL, NULL, 0) != 0) { + printf("error"); + } + if (s3Init() < 0) { + return -1; + } + strcpy(tsSnodeAddress, "127.0.0.1"); + return RUN_ALL_TESTS(); +} + +TEST(testCase, checkpointUpload_Test) { + stopRsync(); + startRsync(); + + taosSsleep(5); + char* id = "2013892036"; + + uploadCheckpoint(id, "/root/offset/"); +} + +TEST(testCase, checkpointDownload_Test) { + char* id = "2013892036"; + downloadCheckpoint(id, "/root/offset/download/"); +} + +TEST(testCase, checkpointDelete_Test) { + char* id = "2013892036"; + deleteCheckpoint(id); +} + +TEST(testCase, checkpointDeleteFile_Test) { + char* id = "2013892036"; + deleteCheckpointFile(id, "offset-ver0"); +} diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index 8e186af51f..95952c960e 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -865,6 +865,9 @@ static int32_t syncSnapBufferRecv(SSyncSnapshotReceiver *pReceiver, SyncSnapshot ASSERT(pRcvBuf->start <= pRcvBuf->cursor + 1 && pRcvBuf->cursor < pRcvBuf->end); if (pMsg->seq > pRcvBuf->cursor) { + if (pRcvBuf->entries[pMsg->seq % pRcvBuf->size]) { + pRcvBuf->entryDeleteCb(pRcvBuf->entries[pMsg->seq % pRcvBuf->size]); + } pRcvBuf->entries[pMsg->seq % pRcvBuf->size] = pMsg; ppMsg[0] = NULL; pRcvBuf->end = TMAX(pMsg->seq + 1, pRcvBuf->end); @@ -1002,7 +1005,8 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, SRpcMsg *pRpcMsg) { } if (pMsg->term < raftStoreGetTerm(pSyncNode)) { - syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "reject since small term"); + sRError(pReceiver, "reject snap replication with smaller term. msg term:%" PRId64 ", seq:%d", pMsg->term, + pMsg->seq); terrno = TSDB_CODE_SYN_MISMATCHED_SIGNATURE; return -1; } diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 9e6ea94e78..06847c081c 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -268,7 +268,7 @@ void syncPrintSnapshotSenderLog(const char* flags, ELogLevel level, int32_t dfla taosPrintLog(flags, level, dflag, "vgId:%d, %s, sync:%s, snap-sender:%p signature:(%" PRId64 ", %" PRId64 "), {start:%" PRId64 - " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRIu64 " last-cfg:%" PRId64 + " end:%" PRId64 " last-index:%" PRId64 " last-term:%" PRId64 " last-cfg:%" PRId64 ", seq:%d, ack:%d, " " buf:[%" PRId64 " %" PRId64 ", %" PRId64 "), finish:%d, as:%d, to-dnode:%d}" diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 08bb8ecabf..bb282b4ee3 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -310,6 +310,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy GRANT_CFG_SET; SConfigItem *pItem = cfgGetItem(pCfg, name); if (pItem == NULL) { + terrno = TSDB_CODE_CFG_NOT_FOUND; return -1; } @@ -338,6 +339,7 @@ int32_t cfgSetItem(SConfig *pCfg, const char *name, const char *value, ECfgSrcTy break; } +_err_out: terrno = TSDB_CODE_INVALID_CFG; return -1; } @@ -357,6 +359,59 @@ SConfigItem *cfgGetItem(SConfig *pCfg, const char *name) { return NULL; } +int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *pVal, bool isServer) { + ECfgDynType dynType = isServer ? CFG_DYN_SERVER : CFG_DYN_CLIENT; + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (!pItem || (pItem->dynScope & dynType) == 0) { + uError("failed to config:%s, not support", name); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + int32_t ival = (int32_t)atoi(pVal); + if (ival != 0 && ival != 1) { + uError("cfg:%s, type:%s value:%d out of range[0, 1]", pItem->name, + cfgDtypeStr(pItem->dtype), ival); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + } break; + case CFG_DTYPE_INT32: { + int32_t ival = (int32_t)atoi(pVal); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name, + cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + } break; + case CFG_DTYPE_INT64: { + int64_t ival = (int64_t)atoll(pVal); + if (ival < pItem->imin || ival > pItem->imax) { + uError("cfg:%s, type:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, + cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + } break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: { + float fval = (float)atof(pVal); + if (fval < pItem->fmin || fval > pItem->fmax) { + uError("cfg:%s, type:%s value:%f out of range[%f, %f]", pItem->name, cfgDtypeStr(pItem->dtype), fval, + pItem->fmin, pItem->fmax); + terrno = TSDB_CODE_OUT_OF_RANGE; + return -1; + } + } break; + default: + break; + } + return 0; +} + static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { pItem->stype = CFG_STYPE_DEFAULT; pItem->name = taosStrdup(name); @@ -381,43 +436,61 @@ static int32_t cfgAddItem(SConfig *pCfg, SConfigItem *pItem, const char *name) { return 0; } -int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope) { - SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope}; +int32_t cfgAddBool(SConfig *pCfg, const char *name, bool defaultVal, int8_t scope, int8_t dynScope) { + SConfigItem item = {.dtype = CFG_DTYPE_BOOL, .bval = defaultVal, .scope = scope, .dynScope = dynScope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) { +int32_t cfgAddInt32(SConfig *pCfg, const char *name, int32_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, + int8_t dynScope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT32, .i32 = defaultVal, .imin = minval, .imax = maxval, .scope = scope}; + SConfigItem item = {.dtype = CFG_DTYPE_INT32, + .i32 = defaultVal, + .imin = minval, + .imax = maxval, + .scope = scope, + .dynScope = dynScope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope) { +int32_t cfgAddInt64(SConfig *pCfg, const char *name, int64_t defaultVal, int64_t minval, int64_t maxval, int8_t scope, + int8_t dynScope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_INT64, .i64 = defaultVal, .imin = minval, .imax = maxval, .scope = scope}; + SConfigItem item = {.dtype = CFG_DTYPE_INT64, + .i64 = defaultVal, + .imin = minval, + .imax = maxval, + .scope = scope, + .dynScope = dynScope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope) { +int32_t cfgAddFloat(SConfig *pCfg, const char *name, float defaultVal, float minval, float maxval, int8_t scope, + int8_t dynScope) { if (defaultVal < minval || defaultVal > maxval) { terrno = TSDB_CODE_OUT_OF_RANGE; return -1; } - SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, .fval = defaultVal, .fmin = minval, .fmax = maxval, .scope = scope}; + SConfigItem item = {.dtype = CFG_DTYPE_FLOAT, + .fval = defaultVal, + .fmin = minval, + .fmax = maxval, + .scope = scope, + .dynScope = dynScope}; return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { - SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope}; +int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { + SConfigItem item = {.dtype = CFG_DTYPE_STRING, .scope = scope, .dynScope = dynScope}; item.str = taosStrdup(defaultVal); if (item.str == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -426,8 +499,8 @@ int32_t cfgAddString(SConfig *pCfg, const char *name, const char *defaultVal, in return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { - SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope}; +int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { + SConfigItem item = {.dtype = CFG_DTYPE_DIR, .scope = scope, .dynScope = dynScope}; if (cfgCheckAndSetDir(&item, defaultVal) != 0) { return -1; } @@ -435,8 +508,8 @@ int32_t cfgAddDir(SConfig *pCfg, const char *name, const char *defaultVal, int8_ return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { - SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope}; +int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { + SConfigItem item = {.dtype = CFG_DTYPE_LOCALE, .scope = scope, .dynScope = dynScope}; if (cfgCheckAndSetLocale(&item, defaultVal) != 0) { return -1; } @@ -444,8 +517,8 @@ int32_t cfgAddLocale(SConfig *pCfg, const char *name, const char *defaultVal, in return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { - SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope}; +int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { + SConfigItem item = {.dtype = CFG_DTYPE_CHARSET, .scope = scope, .dynScope = dynScope}; if (cfgCheckAndSetCharset(&item, defaultVal) != 0) { return -1; } @@ -453,8 +526,8 @@ int32_t cfgAddCharset(SConfig *pCfg, const char *name, const char *defaultVal, i return cfgAddItem(pCfg, &item, name); } -int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope) { - SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope}; +int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal, int8_t scope, int8_t dynScope) { + SConfigItem item = {.dtype = CFG_DTYPE_TIMEZONE, .scope = scope, .dynScope = dynScope}; if (cfgCheckAndSetTimezone(&item, defaultVal) != 0) { return -1; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6c964c3da0..bcdbb3e3ac 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -613,8 +613,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid function par TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid function para value") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION, "Not buildin function") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps not allowed in function") -TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed, format mismatch") -TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed, wrong timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_FORMAT_ERR, "Func to_timestamp failed for format mismatch") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_TS_ERR, "Func to_timestamp failed for wrong timestamp") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_TO_TIMESTAMP_FAILED_NOT_SUPPORTED, "Func to_timestamp failed for unsupported timestamp format") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 6fc0c15cec..aa6719f604 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -57,7 +57,6 @@ typedef struct { typedef struct { int32_t fileNum; - int32_t maxLines; int32_t lines; int32_t flag; int32_t openInProgress; @@ -122,7 +121,7 @@ static void *taosAsyncOutputLog(void *param); static int32_t taosPushLogBuffer(SLogBuff *pLogBuf, const char *msg, int32_t msgLen); static SLogBuff *taosLogBuffNew(int32_t bufSize); static void taosCloseLogByFd(TdFilePtr pFile); -static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum); +static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum); static FORCE_INLINE void taosUpdateDaylight() { struct tm Tm, *ptm; @@ -186,7 +185,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) { tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; - if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; + if (taosOpenLogFile(fullName, maxFiles) < 0) return -1; if (taosInitSlowLog() < 0) return -1; if (taosStartLog() < 0) return -1; @@ -283,7 +282,7 @@ static void *taosThreadToOpenNewFile(void *param) { TdFilePtr pFile = taosOpenFile(name, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { tsLogObj.openInProgress = 0; - tsLogObj.lines = tsLogObj.maxLines - 1000; + tsLogObj.lines = tsNumOfLogLines - 1000; uError("open new log file fail! reason:%s, reuse lastlog", strerror(errno)); return NULL; } @@ -308,7 +307,7 @@ static void *taosThreadToOpenNewFile(void *param) { static int32_t taosOpenNewLogFile() { taosThreadMutexLock(&tsLogObj.logMutex); - if (tsLogObj.lines > tsLogObj.maxLines && tsLogObj.openInProgress == 0) { + if (tsLogObj.lines > tsNumOfLogLines && tsLogObj.openInProgress == 0) { tsLogObj.openInProgress = 1; uInfo("open new log file ......"); @@ -331,7 +330,7 @@ void taosResetLog() { sprintf(lastName, "%s.%d", tsLogObj.logName, tsLogObj.flag); // force create a new log file - tsLogObj.lines = tsLogObj.maxLines + 10; + tsLogObj.lines = tsNumOfLogLines + 10; taosOpenNewLogFile(); (void)taosRemoveFile(lastName); @@ -384,7 +383,7 @@ static void taosGetLogFileName(char *fn) { } } -static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { +static int32_t taosOpenLogFile(char *fn, int32_t maxFileNum) { #ifdef WINDOWS_STASH /* * always set maxFileNum to 1 @@ -396,7 +395,6 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) { char name[LOG_FILE_NAME_LEN + 50] = "\0"; int32_t logstat0_mtime, logstat1_mtime; - tsLogObj.maxLines = maxLines; tsLogObj.fileNum = maxFileNum; taosGetLogFileName(fn); @@ -497,9 +495,9 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b taosWriteFile(tsLogObj.logHandle->pFile, buffer, len); } - if (tsLogObj.maxLines > 0) { + if (tsNumOfLogLines > 0) { atomic_add_fetch_32(&tsLogObj.lines, 1); - if ((tsLogObj.lines > tsLogObj.maxLines) && (tsLogObj.openInProgress == 0)) { + if ((tsLogObj.lines > tsNumOfLogLines) && (tsLogObj.openInProgress == 0)) { taosOpenNewLogFile(); } } diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index c5d9e830d2..e10ffe7c9b 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -54,12 +54,12 @@ TEST_F(CfgTest, 02_Basic) { SConfig *pConfig = cfgInit(); ASSERT_NE(pConfig, nullptr); - EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0), 0); - EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0), 0); - EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0), 0); - EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0), 0); - EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0), 0); - EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0), 0); + EXPECT_EQ(cfgAddBool(pConfig, "test_bool", 0, 0, 0), 0); + EXPECT_EQ(cfgAddInt32(pConfig, "test_int32", 1, 0, 16, 0, 0), 0); + EXPECT_EQ(cfgAddInt64(pConfig, "test_int64", 2, 0, 16, 0, 0), 0); + EXPECT_EQ(cfgAddFloat(pConfig, "test_float", 3, 0, 16, 0, 0), 0); + EXPECT_EQ(cfgAddString(pConfig, "test_string", "4", 0, 0), 0); + EXPECT_EQ(cfgAddDir(pConfig, "test_dir", TD_TMP_DIR_PATH, 0, 0), 0); EXPECT_EQ(cfgGetSize(pConfig), 6); diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index e00ac4e089..5376a3fb1b 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -70,6 +70,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/func_to_char_timestamp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_cache_scan.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqShow.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/subscribeStb0.py @@ -226,6 +230,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ttlChangeOnWrite.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz1.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/compress_tsz2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/view/non_marterial_view/test_view.py ,,n,system-test,python3 ./test.py -f 0-others/compatibility.py ,,n,system-test,python3 ./test.py -f 0-others/tag_index_basic.py ,,n,system-test,python3 ./test.py -f 0-others/udfpy_main.py @@ -233,6 +238,8 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep1.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py + ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py @@ -283,6 +290,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/rowlength64k_4.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionUS.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/precisionNS.py +,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_ts4219.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py @@ -1176,6 +1184,7 @@ e ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +,,y,script,./test.sh -f tsim/sync/vnodesnapshot-rsma-test.sim ,,n,script,./test.sh -f tsim/valgrind/checkError1.sim ,,n,script,./test.sh -f tsim/valgrind/checkError2.sim ,,n,script,./test.sh -f tsim/valgrind/checkError3.sim @@ -1289,6 +1298,7 @@ e ,,y,script,./test.sh -f tsim/tagindex/add_index.sim ,,n,script,./test.sh -f tsim/tagindex/sma_and_tag_index.sim ,,y,script,./test.sh -f tsim/view/view.sim +,,y,script,./test.sh -f tsim/query/const.sim #develop test diff --git a/tests/parallel_test/run_case.sh b/tests/parallel_test/run_case.sh index 1c57ed9b30..7c80ecdbb7 100755 --- a/tests/parallel_test/run_case.sh +++ b/tests/parallel_test/run_case.sh @@ -81,10 +81,10 @@ pip3 list|grep taospy pip3 uninstall taospy -y pip3 install --default-timeout=120 taospy==2.7.12 -#define taos-ws-py 0.2.8 +#define taos-ws-py 0.3.1 pip3 list|grep taos-ws-py pip3 uninstall taos-ws-py -y -pip3 install --default-timeout=120 taos-ws-py==0.2.8 +pip3 install --default-timeout=600 taos-ws-py==0.3.1 $TIMEOUT_CMD $cmd RET=$? diff --git a/tests/pytest/util/autogen.py b/tests/pytest/util/autogen.py index 6f5c89d15b..e1227a680f 100644 --- a/tests/pytest/util/autogen.py +++ b/tests/pytest/util/autogen.py @@ -133,8 +133,6 @@ class AutoGen: if batch_size == 1 or (i > 0 and i % batch_size == 0) : sql = f"insert into {child_name} values {values}" tdSql.execute(sql) - if batch_size > 40: - tdLog.info(f" insert data i={i}") values = "" # end batch diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 7dcf6bc3f2..c05df0a852 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -88,7 +88,7 @@ class TDSql: expectErrNotOccured = False self.errno = e.errno error_info = repr(e) - self.error_info = error_info[error_info.index('(')+1:-1].split(",")[0].replace("'","") + self.error_info = ','.join(error_info[error_info.index('(')+1:-1].split(",")[:-1]).replace("'","") # self.error_info = (','.join(error_info.split(",")[:-1]).split("(",1)[1:][0]).replace("'","") if expectErrNotOccured: tdLog.exit("%s(%d) failed: sql:%s, expect error not occured" % (caller.filename, caller.lineno, sql)) @@ -106,7 +106,7 @@ class TDSql: tdLog.info("sql:%s, expect error occured" % (sql)) if expectErrInfo != None: - if expectErrInfo == self.error_info: + if expectErrInfo == self.error_info or expectErrInfo in self.error_info: tdLog.info("sql:%s, expected expectErrInfo %s occured" % (sql, expectErrInfo)) else: tdLog.exit("%s(%d) failed: sql:%s, expectErrInfo %s occured, but not expected errno %s" % (caller.filename, caller.lineno, sql, self.error_info, expectErrInfo)) diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 699b390c20..d848dca194 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -19,6 +19,7 @@ exe: gcc $(CFLAGS) ./whiteListTest.c -o $(ROOT)whiteListTest $(LFLAGS) gcc $(CFLAGS) ./insert_stb.c -o $(ROOT)insert_stb $(LFLAGS) gcc $(CFLAGS) ./tmqViewTest.c -o $(ROOT)tmqViewTest $(LFLAGS) + gcc $(CFLAGS) ./stmtQuery.c -o $(ROOT)stmtQuery $(LFLAGS) clean: rm $(ROOT)batchprepare @@ -29,3 +30,4 @@ clean: rm $(ROOT)whiteListTest rm $(ROOT)insert_stb rm $(ROOT)tmqViewTest + rm $(ROOT)stmtQuery diff --git a/tests/script/api/stmtQuery.c b/tests/script/api/stmtQuery.c new file mode 100644 index 0000000000..05ee3b9df7 --- /dev/null +++ b/tests/script/api/stmtQuery.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include +#include "../../../include/client/taos.h" + +#define PRINT_ERROR printf("\033[31m"); +#define PRINT_SUCCESS printf("\033[32m"); + +void execute_simple_sql(void *taos, char *sql) { + TAOS_RES *result = taos_query(taos, sql); + if ( result == NULL || taos_errno(result) != 0) { + PRINT_ERROR + printf("failed to %s, Reason: %s\n", sql, taos_errstr(result)); + taos_free_result(result); + exit(EXIT_FAILURE); + } + taos_free_result(result); + PRINT_SUCCESS + printf("Successfully %s\n", sql); +} + +void check_result(TAOS_RES *result, int expected) { + int rows = 0; + TAOS_ROW row; + while ((row = taos_fetch_row(result))) { + rows++; + } + if (rows == expected) { + PRINT_SUCCESS + printf("%d rows are fetched as expected\n", rows); + } else { + PRINT_ERROR + printf("%d rows are fetched but %d expected\n", rows, expected); + } + taos_free_result(result); +} + +int main(int argc, char *argv[]) { + void *taos = taos_connect("127.0.0.1", "root", "taosdata", NULL, 0); + if (taos == NULL) { + PRINT_ERROR + printf("TDengine error: failed to connect\n"); + exit(EXIT_FAILURE); + } + PRINT_SUCCESS + printf("Successfully connected to TDengine\n"); + + execute_simple_sql(taos, "drop database if exists test"); + execute_simple_sql(taos, "create database test"); + execute_simple_sql(taos, "use test"); + execute_simple_sql(taos, "create table super(ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 binary(8), c6 smallint, c7 tinyint, c8 bool, c9 nchar(8), c10 timestamp) tags (t1 int, t2 bigint, t3 float, t4 double, t5 binary(8), t6 smallint, t7 tinyint, t8 bool, t9 nchar(8))"); + + char *sql = calloc(1, 1024*1024); + int sqlLen = 0; + sqlLen = sprintf(sql, "create table"); + for (int i = 0; i < 10; i++) { + sqlLen += sprintf(sql + sqlLen, " t%d using super tags (%d, 2147483648, 0.1, 0.000000001, 'abcdefgh', 32767, 127, 1, '一二三四五六七八')", i, i); + } + execute_simple_sql(taos, sql); + + strcpy(sql, "insert into t0 (ts, c1) values(now, 1)"); + execute_simple_sql(taos, sql); + + strcpy(sql, "insert into t0 (ts, c1) values(now, 2)"); + execute_simple_sql(taos, sql); + + strcpy(sql, "insert into t0 (ts, c1) values(now, 3)"); + execute_simple_sql(taos, sql); + + + int code = taos_load_table_info(taos, "t0,t1,t2,t3,t4,t5,t6,t7,t8,t9"); + if (code != 0) { + PRINT_ERROR + printf("failed to load table info: 0x%08x\n", code); + exit(EXIT_FAILURE); + } + PRINT_SUCCESS + printf("Successfully load table info\n"); + + TAOS_STMT *stmt = taos_stmt_init(taos); + if (stmt == NULL) { + PRINT_ERROR + printf("TDengine error: failed to init taos_stmt\n"); + exit(EXIT_FAILURE); + } + PRINT_SUCCESS + printf("Successfully init taos_stmt\n"); + + + char* condBuf = "2 or c1 > 0"; + TAOS_MULTI_BIND params[1]; + params[0].buffer_type = TSDB_DATA_TYPE_BINARY; + params[0].buffer_length = strlen(condBuf) + 1; + params[0].buffer = condBuf; + params[0].length = (int*)¶ms[0].buffer_length; + params[0].is_null = NULL; + params[0].num = 1; + + char *stmt_sql = "select * from super where c1 > ?"; + code = taos_stmt_prepare(stmt, stmt_sql, 0); + if (code != 0){ + PRINT_ERROR + printf("failed to execute taos_stmt_prepare. code:0x%x\n", code); + exit(EXIT_FAILURE); + } + PRINT_SUCCESS + printf("Successfully execute taos_stmt_prepare\n"); + + taos_stmt_bind_param(stmt, params); + taos_stmt_add_batch(stmt); + + if (taos_stmt_execute(stmt) != 0) { + PRINT_ERROR + printf("failed to execute query statement.\n"); + exit(EXIT_FAILURE); + } + PRINT_SUCCESS + printf("Successfully execute query statement.\n"); + + TAOS_RES *result = taos_stmt_use_result(stmt); + check_result(result, 1); + + + taos_stmt_close(stmt); + + return 0; +} diff --git a/tests/script/local.supp b/tests/script/local.supp index 562cddba46..c351846f2c 100644 --- a/tests/script/local.supp +++ b/tests/script/local.supp @@ -225,3 +225,1433 @@ fun:vnodeOpen fun:vmOpenVnodeInThread } +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_GLOBAL__sub_I_fs_posix.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_Z41__static_initialization_and_destruction_0ii.constprop.0 + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb13ObjectLibrary7DefaultEv + fun:_GLOBAL__sub_I_fs_posix.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_GLOBAL__sub_I_fs_posix.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb11SystemClock7DefaultEv + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:rocksdb_options_create + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread + fun:clone +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb10FileSystem7DefaultEv + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:rocksdb_options_create + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread + fun:clone +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb14ObjectRegistry7DefaultEv + fun:_ZN7rocksdb14ObjectRegistry11NewInstanceEv + fun:_ZN7rocksdb13ConfigOptionsC1Ev + fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS3_INS_19ColumnFamilyOptionsESaIS8_EERKSsPNS_10FileSystemE + fun:_ZN7rocksdb6DBImpl16WriteOptionsFileEbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread + fun:clone +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10shared_ptrIN7rocksdb13ObjectLibraryEESaIS3_EE17_M_realloc_insertIJRKS3_EEEvN9__gnu_cxx17__normal_iteratorIPS3_S5_EEDpOT_ + fun:_ZN7rocksdb14ObjectRegistryC1ERKSt10shared_ptrINS_13ObjectLibraryEE + fun:_ZN7rocksdb14ObjectRegistry7DefaultEv + fun:_ZN7rocksdb14ObjectRegistry11NewInstanceEv + fun:_ZN7rocksdb13ConfigOptionsC1Ev + fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS3_INS_19ColumnFamilyOptionsESaIS8_EERKSsPNS_10FileSystemE + fun:_ZN7rocksdb6DBImpl16WriteOptionsFileEbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryIKNS_14SliceTransformEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS5_St14default_deleteIS5_EEPSsEERKNS0_12PatternEntryESI_ + fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv +} +{ + + Memcheck:Leak + match-leak-kinds: definite + fun:_ZnwmRKSt9nothrow_t + fun:__cxa_thread_atexit + fun:_ZN7rocksdb17InstrumentedMutex4LockEv + fun:_ZN7rocksdb5TimerD1Ev + fun:__run_exit_handlers + fun:exit + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_GLOBAL__sub_I_fs_posix.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7rocksdb13ObjectLibrary7DefaultEv + fun:_GLOBAL__sub_I_fs_posix.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_GLOBAL__sub_I_fs_posix.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE9_M_rehashEmRKm + fun:_ZN7rocksdb19ThreadStatusUpdater14RegisterThreadENS_12ThreadStatus10ThreadTypeEm + fun:_ZN7rocksdb14ThreadPoolImpl4Impl15BGThreadWrapperEPv + fun:execute_native_thread_routine + fun:start_thread + fun:clone +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE9GetSharedEPNS_5CacheEPNS_11SystemClockEPSt10shared_ptrIS3_E + fun:_ZN7rocksdb13InternalStatsC1EiPNS_11SystemClockEPNS_16ColumnFamilyDataE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKSsjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPSsb + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPmPNS0_15RecoveryContextE + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_10FileSystemEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_GLOBAL__sub_I_fs_posix.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb18BytewiseComparatorEv + fun:_ZN7rocksdb19ColumnFamilyOptionsC1Ev + fun:rocksdb_options_create + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread + fun:clone +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE + fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZZSt9call_onceIZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb22FileChecksumGenFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_22FileChecksumGenFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21DBOptionsConfigurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb19GetDBOptionsFromMapERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE + fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZZSt9call_onceIZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb21SstPartitionerFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_21SstPartitionerFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE + fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZZSt9call_onceIZN7rocksdb10Comparator16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb10Comparator16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_10ComparatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE + fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZZSt9call_onceIZN7rocksdbL22RegisterTableFactoriesERKSsEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSs4_Rep9_S_createEmmRKSaIcE + fun:_ZNSs12_S_constructIPKcEEPcT_S3_RKSaIcESt20forward_iterator_tag + fun:_ZNSsC1EPKcRKSaIcE + fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZZSt9call_onceIZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb23FlushBlockPolicyFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_23FlushBlockPolicyFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb22BlockBasedTableFactory11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12Customizable18ConfigureNewObjectERKNS_13ConfigOptionsEPS0_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb12TableFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_12TableFactoryEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE + fun:_ZN7rocksdb21PersistRocksDBOptionsERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS6_INS_19ColumnFamilyOptionsESaISB_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZZSt9call_onceIZN7rocksdb16CompactionFilter16CreateFromStringERKNS0_13ConfigOptionsERKSsPPKS1_EUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb16CompactionFilter16CreateFromStringERKNS_13ConfigOptionsERKSsPPKS0_ + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo14AsCustomRawPtrIKNS0_16CompactionFilterEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE + fun:_ZN7rocksdb20RocksDBOptionsParser5ParseERKNS_13ConfigOptionsERKSsPNS_10FileSystemE + fun:_ZN7rocksdb20RocksDBOptionsParser28VerifyRocksDBOptionsFromFileERKNS_13ConfigOptionsERKNS_9DBOptionsERKSt6vectorISsSaISsEERKS7_INS_19ColumnFamilyOptionsESaISC_EERKSsPNS_10FileSystemE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb15ColumnFamilySetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb10VersionSetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKSsbbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread + fun:clone +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKSsjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPSsb + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPmPNS0_15RecoveryContextE + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb15ColumnFamilySetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb10VersionSetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKSsbbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:streamBackendInit + fun:streamMetaOpen + fun:tqInitialize + fun:tqOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKSsjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPSsb + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPmPNS0_15RecoveryContextE + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:streamBackendInit + fun:streamMetaOpen + fun:tqInitialize + fun:tqOpen + fun:vnodeOpen +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryIKNS_14SliceTransformEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS5_St14default_deleteIS5_EEPSsEERKNS0_12PatternEntryESI_ + fun:_ZN7rocksdbL29RegisterBuiltinSliceTransformERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb14SliceTransform16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIKS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb14SliceTransform16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIKS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrIKNS0_14SliceTransformEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS3_EESaIS6_EE17_M_realloc_insertIJS6_EEEvN9__gnu_cxx17__normal_iteratorIPS6_S8_EEDpOT_ + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_13MergeOperatorEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL29RegisterBuiltinMergeOperatorsERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb13MergeOperator16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10shared_ptrIS1_EEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb13MergeOperator16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvEZNS0_14OptionTypeInfo17AsCustomSharedPtrINS0_13MergeOperatorEEES9_iNS0_22OptionVerificationTypeENS0_15OptionTypeFlagsEEUlS4_S6_S6_S7_E_E9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE + fun:_ZN7rocksdb29GetColumnFamilyOptionsFromMapERKNS_13ConfigOptionsERKNS_19ColumnFamilyOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPS3_ + fun:_ZN7rocksdb20RocksDBOptionsParser10EndSectionERKNS_13ConfigOptionsENS_13OptionSectionERKSsS6_RKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIS5_SsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE9_M_rehashEmRKm + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKSsPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsEPKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb15ColumnFamilySetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb10VersionSetC1ERKSsPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES2_S2_ + fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKSsbbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKSsRKSt6vectorINS_22ColumnFamilyDescriptorESaIS7_EEPS6_IPNS_18ColumnFamilyHandleESaISD_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKSsPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZNSt10_HashtableISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS6_EESaIS9_EEESaISC_ENSt8__detail10_Select1stESt8equal_toISsESt4hashISsENSE_18_Mod_range_hashingENSE_20_Default_ranged_hashENSE_20_Prime_rehash_policyENSE_17_Hashtable_traitsILb1ELb0ELb1EEEE9_M_rehashEmRKm + fun:_ZNSt8__detail9_Map_baseISsSt4pairIKSsSt6vectorISt10unique_ptrIN7rocksdb13ObjectLibrary5EntryESt14default_deleteIS7_EESaISA_EEESaISD_ENS_10_Select1stESt8equal_toISsESt4hashISsENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb1ELb0ELb1EEELb1EEixEOSs + fun:_ZN7rocksdb13ObjectLibrary10AddFactoryINS_18MemTableRepFactoryEEERKSt8functionIFPT_RKSsPSt10unique_ptrIS4_St14default_deleteIS4_EEPSsEERKNS0_12PatternEntryESH_ + fun:_ZN7rocksdbL33RegisterBuiltinMemTableRepFactoryERNS_13ObjectLibraryERKSs.isra.0 + fun:_ZZSt9call_onceIZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS0_13ConfigOptionsERKSsPSt10unique_ptrIS1_St14default_deleteIS1_EEEUlvE_JEEvRSt9once_flagOT_DpOT0_ENUlvE0_4_FUNEv + fun:__pthread_once_slow + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10unique_ptrIS0_St14default_deleteIS0_EE + fun:_ZN7rocksdb18MemTableRepFactory16CreateFromStringERKNS_13ConfigOptionsERKSsPSt10shared_ptrIS0_E + fun:_ZNSt17_Function_handlerIFN7rocksdb6StatusERKNS0_13ConfigOptionsERKSsS6_PvENS0_UlS4_S6_S6_S7_E3_EE9_M_invokeERKSt9_Any_dataS4_S6_S6_OS7_ + fun:_ZNK7rocksdb14OptionTypeInfo5ParseERKNS_13ConfigOptionsERKSsS5_Pv + fun:_ZN7rocksdb12Configurable11ParseOptionERKNS_13ConfigOptionsERKNS_14OptionTypeInfoERKSsS8_Pv + fun:_ZN7rocksdb18ConfigurableHelper27ConfigureCustomizableOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper15ConfigureOptionERKNS_13ConfigOptionsERNS_12ConfigurableERKNS_14OptionTypeInfoERKSsSA_SA_Pv + fun:_ZN7rocksdb18ConfigurableHelper20ConfigureSomeOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsNS_14OptionTypeInfoESt4hashISsESt8equal_toISsESaISt4pairIKSsS7_EEEPS6_ISsSsS9_SB_SaISC_ISD_SsEEEPv + fun:_ZN7rocksdb18ConfigurableHelper16ConfigureOptionsERKNS_13ConfigOptionsERNS_12ConfigurableERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSF_ + fun:_ZN7rocksdb12Configurable16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb21ConfigurableCFOptions16ConfigureOptionsERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEEPSD_ + fun:_ZN7rocksdb12Configurable16ConfigureFromMapERKNS_13ConfigOptionsERKSt13unordered_mapISsSsSt4hashISsESt8equal_toISsESaISt4pairIKSsSsEEE +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:rocksdb_options_create + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread + fun:clone +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb14ThreadLocalPtr8InstanceEv + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:rocksdb_options_create + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq + fun:vmProcessMgmtQueue + fun:tQWorkerThreadFp + fun:start_thread + fun:clone +} diff --git a/tests/script/tsim/insert/insert_stb.sim b/tests/script/tsim/insert/insert_stb.sim index 890d8937a5..4c0797e2a7 100644 --- a/tests/script/tsim/insert/insert_stb.sim +++ b/tests/script/tsim/insert/insert_stb.sim @@ -51,10 +51,26 @@ print $rows if $rows != 5 then return -1 endi + +sql create database dgxy; +sql use dgxy; +sql create table st(ts timestamp, f int) tags(t int); +sql insert into ct1 using st tags(1) values(now, 1); +sql insert into st(tbname, ts, f) values('ct1', now, 2); +sql select * from ct1; +if $rows != 2 then + return -1 +endi +sql show tables like 'ct1'; +if $rows != 1 then + return -1 +endi + sql_error insert into d2.st values(now, 1, 1) sql_error insert into d2.st(ts, f) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1); sql_error insert into d2.st(ts, f, tbname) values(now, 1, ''); +sql_error insert into d2.st(ts, f, tbname) values(now, 1, 'd2.ct2'); sql_error insert into d2.st(ts, tbname) values(now, 1, 34) sql_error insert into st using st2 tags(2) values(now,1); system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/const.sim b/tests/script/tsim/query/const.sim new file mode 100644 index 0000000000..08f2b909c1 --- /dev/null +++ b/tests/script/tsim/query/const.sim @@ -0,0 +1,11 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql select b.z from (select c.a as z from (select 'a' as a) c) b; +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/tag_scan.sim b/tests/script/tsim/query/tag_scan.sim index 3f0a23fbd6..ee56a214a5 100644 --- a/tests/script/tsim/query/tag_scan.sim +++ b/tests/script/tsim/query/tag_scan.sim @@ -140,4 +140,33 @@ if $data01 != @1aa@ then return -1 endi +sql create table stb34 (ts timestamp, f int) tags(t int); +sql insert into ctb34 using stb34 tags(1) values(now, 1)(now+1s, 2); +sql select 1 from (select tags t from stb34 order by t) +print $rows +if $rows != 1 then + return -1 +endi + +sql select count(*) from (select tags t from stb34) +if $data00 != 1 then + return -1 +endi + + +sql select 1 from (select tags ts from stb34) +print $rows +if $rows != 2 then + return -1 +endi + +sql select count(*) from (select tags ts from stb34) +if $data00 != 2 then + return -1 +endi + +sql select tags 2 from stb34 +if $rows != 1 then + return -1 +endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim index 6f78829db7..c70f2dc20a 100644 --- a/tests/script/tsim/sma/rsmaPersistenceRecovery.sim +++ b/tests/script/tsim/sma/rsmaPersistenceRecovery.sim @@ -5,7 +5,7 @@ sleep 50 sql connect #todo wait for streamState checkpoint -return 1 +#return 1 print =============== create database with retentions sql create database d0 retentions -:7d,5m:21d,15m:365d; diff --git a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim index 3b3cd01521..8b1720d213 100644 --- a/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim +++ b/tests/script/tsim/sync/vnodesnapshot-rsma-test.sim @@ -114,7 +114,7 @@ endi vg_ready: print ====> create stable/child table -sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) rollup(sum) watermark 3s,3s max_delay 3s,3s +sql create table stb (ts timestamp, c1 float, c2 float, c3 double) tags (t1 int) rollup(sum) watermark 3s,3s max_delay 3s,3s sql show stables if $rows != 1 then @@ -167,9 +167,6 @@ system sh/exec.sh -n dnode4 -s start sleep 3000 - - - print =============== query data of level 1 sql connect sql use db @@ -181,12 +178,21 @@ if $rows != 100 then return -1 endi +print =============== sleep 5s to wait the result +sleep 5000 + print =============== query data of level 2 sql select * from ct1 where ts > now - 10d -print rows of level 2: $rows print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows != 100 then + print rows of level 2: $rows +endi print =============== query data of level 3 sql select * from ct1 -print rows of level 3: $rows print $data00 $data01 $data02 +print $data10 $data11 $data12 +if $rows != 100 then + print rows of level 3: $rows +endi \ No newline at end of file diff --git a/tests/script/win-test-file b/tests/script/win-test-file index 4ff4b52f7e..fe5f5c39e3 100644 --- a/tests/script/win-test-file +++ b/tests/script/win-test-file @@ -320,6 +320,7 @@ ./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim +./test.sh -f tsim/sync/vnodesnapshot-rsma-test.sim ./test.sh -f tsim/valgrind/checkError1.sim ./test.sh -f tsim/valgrind/checkError2.sim ./test.sh -f tsim/valgrind/checkError3.sim diff --git a/tests/system-test/0-others/delete_check.py b/tests/system-test/0-others/delete_check.py new file mode 100644 index 0000000000..ec589295e4 --- /dev/null +++ b/tests/system-test/0-others/delete_check.py @@ -0,0 +1,154 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * +from util.autogen import * + +import random +import time +import traceback +import os +from os import path + + +class TDTestCase: + # init + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + # autoGen + self.autoGen = AutoGen() + # init cluster path + selfPath = os.path.dirname(os.path.realpath(__file__)) + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + self.projDir = f"{projPath}sim/" + tdLog.info(f" init projPath={self.projDir}") + + def compactDatbase(self): + # compact database + tdSql.execute(f"compact database {self.dbname}", show=True) + waitSeconds = 20 + if self.waitTranslation(waitSeconds) == False: + tdLog.exit(f"translation can not finish after wait {waitSeconds} seconds") + return + + # check tsdb folder empty + def check_tsdb_dir(self, tsdbDir): + for vfile in os.listdir(tsdbDir): + fileName, ext = os.path.splitext(vfile) + pathFile = os.path.join(tsdbDir, vfile) + ext = ext.lower() + tdLog.info(f"check exist file {pathFile} ...") + if ext == ".head" or ext == ".data" or ext == ".stt" or ext == ".sma": + tdLog.info(f"found {pathFile} not to be deleted ...") + real = True + for i in range(50): + tdLog.info(f"i={i} compact and try again ...") + self.compactDatbase() + if os.path.exists(pathFile) is False: + real = False + break + else: + time.sleep(0.5) + tdLog.info(f"file real exist {pathFile} , sleep 500ms and try") + + if real is False: + continue + fileStat = os.stat(pathFile) + tdLog.exit(f" check file can not be deleted. file={pathFile} file size={fileStat.st_size}") + + return True + + # check vnode tsdb folder empty + def check_filedelete(self): + # put all vnode to list + for dnode in os.listdir(self.projDir): + vnodesDir = self.projDir + f"{dnode}/data/vnode/" + if os.path.isdir(vnodesDir) == False or dnode[:5] != "dnode": + continue + print(f"vnodesDir={vnodesDir}") + # enum all vnode + for vnode in os.listdir(vnodesDir): + vnodeDir = path.join(vnodesDir, vnode) + print(f"vnodeDir={vnodeDir}") + if os.path.isdir(vnodesDir): + tsdbDir = path.join(vnodeDir, "tsdb") + if path.exists(tsdbDir) : + self.check_tsdb_dir(tsdbDir) + + def waitTranslation(self, waitSeconds): + # wait end + for i in range(waitSeconds): + sql ="show transactions;" + rows = tdSql.query(sql) + if rows == 0: + return True + tdLog.info(f"i={i} wait for translation finish ...") + time.sleep(1) + + return False + + # run + def run(self): + # seed + random.seed(int(time.time())) + self.dbname = "deletecheck" + stbname = "meters" + childname= "d" + child_cnt = 2 + batch_size = 8000 + insert_rows = 100015 + start_ts = 1600000000000 + + self.autoGen.create_db(self.dbname) + + loop = 3 + for i in range(loop): + self.autoGen.create_stable(stbname, 4, 10, 4, 8) + self.autoGen.create_child(stbname, childname, child_cnt) + self.autoGen.set_batch_size(batch_size) + self.autoGen.insert_data(insert_rows) + self.autoGen.set_start_ts(start_ts) + + if i % 2 == 1: + tdSql.execute(f"flush database {self.dbname}", show=True) + + # drop stable + tdSql.execute(f"drop table {self.dbname}.{stbname} ", show = True) + + self.compactDatbase() + + # check file delete + self.check_filedelete() + tdLog.info(f"loop = {i+1} / {loop} check file delete ok after drop table successfully.") + + start_ts += i*100000000 + + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/0-others/view/non_marterial_view/test_view.py b/tests/system-test/0-others/view/non_marterial_view/test_view.py new file mode 100644 index 0000000000..afb2476305 --- /dev/null +++ b/tests/system-test/0-others/view/non_marterial_view/test_view.py @@ -0,0 +1,591 @@ + +import taos +import os +import sys +import time +from pathlib import Path +sys.path.append(os.path.dirname(Path(__file__).resolve().parent.parent.parent) + "/7-tmq") + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +from util.sqlset import * +from tmqCommon import * + +class TDTestCase: + """This test case is used to veirfy the tmq consume data from non marterial view + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.setsql = TDSetSql() + + # db info + self.dbname = "view_db" + self.stbname = 'stb' + self.ctbname_list = ["ct1", "ct2"] + self.stable_column_dict = { + 'ts': 'timestamp', + 'col1': 'float', + 'col2': 'int', + } + self.tag_dict = { + 'ctbname': 'binary(10)' + } + + def prepare_data(self, conn=None): + """Create the db and data for test + """ + tdLog.debug("Start to prepare the data") + if not conn: + conn = tdSql + # create datebase + conn.execute(f"create database {self.dbname}") + conn.execute(f"use {self.dbname}") + time.sleep(2) + + # create stable + conn.execute(self.setsql.set_create_stable_sql(self.stbname, self.stable_column_dict, self.tag_dict)) + tdLog.debug("Create stable {} successfully".format(self.stbname)) + + # create child tables + for ctname in self.ctbname_list: + conn.execute(f"create table {ctname} using {self.stbname} tags('{ctname}');") + tdLog.debug("Create child table {} successfully".format(ctname)) + + # insert data into child tables + conn.execute(f"insert into {ctname} values(now, 1.1, 1)(now+1s, 2.2, 2)(now+2s, 3.3, 3)(now+3s, 4.4, 4)(now+4s, 5.5, 5)(now+5s, 6.6, 6)(now+6s, 7.7, 7)(now+7s, 8.8, 8)(now+8s, 9.9, 9)(now+9s, 10.1, 10);)") + tdLog.debug(f"Insert into data to {ctname} successfully") + + def prepare_tmq_data(self, para_dic): + tdLog.debug("Start to prepare the tmq data") + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, para_dic["dbName"], para_dic["dropFlag"], vgroups=para_dic["vgroups"], replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=para_dic["dbName"], stbname=para_dic["stbName"], column_elm_list=para_dic['colSchema'], tag_elm_list=para_dic['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=para_dic["dbName"], stbname=para_dic["stbName"],tag_elm_list=para_dic['tagSchema'], count=para_dic["ctbNum"], default_ctbname_prefix=para_dic['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql, para_dic["dbName"], para_dic["ctbPrefix"], para_dic["ctbNum"], para_dic["rowsPerTbl"], para_dic["batchNum"], para_dic["startTs"]) + tdLog.debug("Finish to prepare the tmq data") + + def check_view_num(self, num): + tdSql.query("show views;") + rows = tdSql.queryRows + assert(rows == num) + tdLog.debug(f"Verify the view number successfully") + + def create_user(self, username, password): + tdSql.execute(f"create user {username} pass '{password}';") + tdLog.debug("Create user {} with password {} successfully".format(username, password)) + + def check_permissions(self, username, db_name, permission_dict, view_name=None): + """ + :param permission_dict: {'db': ["read", "write], 'view': ["read", "write", "alter"]} + """ + tdSql.query("select * from information_schema.ins_user_privileges;") + for item in permission_dict.keys(): + if item == "db": + for permission in permission_dict[item]: + assert((username, permission, db_name, "", "", "") in tdSql.queryResult) + tdLog.debug(f"Verify the {item} {db_name} {permission} permission successfully") + elif item == "view": + for permission in permission_dict[item]: + assert((username, permission, db_name, view_name, "", "view") in tdSql.queryResult) + tdLog.debug(f"Verify the {item} {db_name} {view_name} {permission} permission successfully") + else: + raise Exception(f"Invalid permission type: {item}") + + def test_create_view_from_one_database(self): + """This test case is used to verify the create view from one database + """ + self.prepare_data() + tdSql.execute(f"create view v1 as select * from {self.stbname};") + self.check_view_num(1) + tdSql.error(f'create view v1 as select * from {self.stbname};', expectErrInfo='view already exists in db') + tdSql.error(f'create view db2.v2 as select * from {self.stbname};', expectErrInfo='Fail to get table info, error: Database not exist') + tdSql.error(f'create view v2 as select c2 from {self.stbname};', expectErrInfo='Invalid column name: c2') + tdSql.error(f'create view v2 as select ts, col1 from tt1;', expectErrInfo='Fail to get table info, error: Table does not exist') + + tdSql.execute(f"drop database {self.dbname}") + tdLog.debug("Finish test case 'test_create_view_from_one_database'") + + def test_create_view_from_multi_database(self): + """This test case is used to verify the create view from multi database + """ + self.prepare_data() + tdSql.execute(f"create view v1 as select * from view_db.{self.stbname};") + self.check_view_num(1) + + self.dbname = "view_db2" + self.prepare_data() + tdSql.execute(f"create view v1 as select * from view_db2.{self.stbname};") + tdSql.execute(f"create view v2 as select * from view_db.v1;") + self.check_view_num(2) + + self.dbname = "view_db" + tdSql.execute(f"drop database view_db;") + tdSql.execute(f"drop database view_db2;") + tdLog.debug("Finish test case 'test_create_view_from_multi_database'") + + def test_create_view_name_params(self): + """This test case is used to verify the create view with different view name params + """ + self.prepare_data() + tdSql.execute(f"create view v1 as select * from {self.stbname};") + self.check_view_num(1) + tdSql.error(f"create view v/2 as select * from {self.stbname};", expectErrInfo='syntax error near "/2 as select * from stb;"') + tdSql.execute(f"create view v2 as select ts, col1 from {self.stbname};") + self.check_view_num(2) + view_name_192_characters = "rzuoxoIXilAGgzNjYActiQwgzZK7PZYpDuaOe1lSJMFMVYXaexh1OfMmk3LvJcQbTeXXW7uGJY8IHuweHF73VHgoZgf0waO33YpZiTKfDQbdWtN4YmR2eWjL84ZtkfjM4huCP6lCysbDMj8YNwWksTdUq70LIyNhHp2V8HhhxyYSkREYFLJ1kOE78v61MQT6" + tdSql.execute(f"create view {view_name_192_characters} as select * from {self.stbname};") + self.check_view_num(3) + tdSql.error(f"create view {view_name_192_characters}1 as select * from {self.stbname};", expectErrInfo='Invalid identifier name: rzuoxoixilaggznjyactiqwgzzk7pzypduaoe1lsjmfmvyxaexh1ofmmk3lvjcqbtexxw7ugjy8ihuwehf73vhgozgf0wao33ypzitkfdqbdwtn4ymr2ewjl84ztkfjm4hucp6lcysbdmj8ynwwkstduq70liynhhp2v8hhhxyyskreyflj1koe78v61mqt61 as select * from stb;') + tdSql.execute(f"drop database {self.dbname}") + tdLog.debug("Finish test case 'test_create_view_name_params'") + + def test_create_view_query(self): + """This test case is used to verify the create view with different data type in query + """ + self.prepare_data() + # add different data type table + tdSql.execute(f"create table tb (ts timestamp, c1 int, c2 int unsigned, c3 bigint, c4 bigint unsigned, c5 float, c6 double, c7 binary(16), c8 smallint, c9 smallint unsigned, c10 tinyint, c11 tinyint unsigned, c12 bool, c13 varchar(16), c14 nchar(8), c15 geometry(21), c16 varbinary(16));") + tdSql.execute(f"create view v1 as select ts, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15, c16 from tb;") + # check data type in create view sql + tdSql.query("desc v1;") + res = tdSql.queryResult + data_type_list = [res[index][1] for index in range(len(res))] + tdLog.debug(data_type_list) + assert('TIMESTAMP' in data_type_list and 'INT' in data_type_list and 'INT UNSIGNED' in data_type_list and 'BIGINT' in data_type_list and 'BIGINT UNSIGNED' in data_type_list and 'FLOAT' in data_type_list and 'DOUBLE' in data_type_list and 'VARCHAR' in data_type_list and 'SMALLINT' in data_type_list and 'SMALLINT UNSIGNED' in data_type_list and 'TINYINT' in data_type_list and 'TINYINT UNSIGNED' in data_type_list and 'BOOL' in data_type_list and 'VARCHAR' in data_type_list and 'NCHAR' in data_type_list and 'GEOMETRY' in data_type_list and 'VARBINARY' in data_type_list) + tdSql.execute("create view v2 as select * from tb where c1 >5 and c7 like '%ab%';") + self.check_view_num(2) + tdSql.error("create view v3 as select * from tb where c1 like '%ab%';", expectErrInfo='Invalid value type') + tdSql.execute("create view v3 as select first(ts), sum(c1) from tb group by c2 having avg(c4) > 0;") + tdSql.execute("create view v4 as select _wstart,sum(c6) from tb interval(10s);") + tdSql.execute("create view v5 as select * from tb join v2 on tb.ts = v2.ts;") + tdSql.execute("create view v6 as select * from (select ts, c1, c2 from (select * from v2));") + self.check_view_num(6) + for v in ['v1', 'v2', 'v3', 'v4', 'v5', 'v6']: + tdSql.execute(f"drop view {v};") + tdSql.execute(f"drop database {self.dbname}") + tdLog.debug("Finish test case 'test_create_view_query'") + + def test_show_view(self): + """This test case is used to verify the show view + """ + self.prepare_data() + tdSql.execute(f"create view v1 as select * from {self.ctbname_list[0]};") + + # query from show sql + tdSql.query("show views;") + res = tdSql.queryResult + assert(res[0][0] == 'v1' and res[0][1] == 'view_db' and res[0][2] == 'root' and res[0][4] == 'NORMAL' and res[0][5] == 'select * from ct1;') + + # show create sql + tdSql.query("show create view v1;") + res = tdSql.queryResult + assert(res[0][1] == 'CREATE VIEW `view_db`.`v1` AS select * from ct1;') + + # query from desc results + tdSql.query("desc view_db.v1;") + res = tdSql.queryResult + assert(res[0][1] == 'TIMESTAMP' and res[1][1] == 'FLOAT' and res[2][1] == 'INT') + + # query from system table + tdSql.query("select * from information_schema.ins_views;") + res = tdSql.queryResult + assert(res[0][0] == 'v1' and res[0][1] == 'view_db' and res[0][2] == 'root' and res[0][4] == 'NORMAL' and res[0][5] == 'select * from ct1;') + tdSql.error("show db3.views;", expectErrInfo='Database not exist') + tdSql.error("desc viewx;", expectErrInfo='Table does not exist') + tdSql.error(f"show create view {self.dbname}.viewx;", expectErrInfo='view not exists in db') + tdSql.execute(f"drop database {self.dbname}") + tdSql.error("show views;", expectErrInfo='Database not exist') + tdLog.debug("Finish test case 'test_show_view'") + + def test_drop_view(self): + """This test case is used to verify the drop view + """ + self.prepare_data() + self.dbname = "view_db2" + self.prepare_data() + tdSql.execute("create view view_db.v1 as select * from view_db.stb;") + tdSql.execute("create view view_db2.v1 as select * from view_db2.stb;") + # delete view without database name + tdSql.execute("drop view v1;") + # delete view with database name + tdSql.execute("drop view view_db.v1;") + # delete non exist view + tdSql.error("drop view view_db.v11;", expectErrInfo='view not exists in db') + tdSql.execute("drop database view_db") + tdSql.execute("drop database view_db2;") + self.dbname = "view_db" + tdLog.debug("Finish test case 'test_drop_view'") + + def test_view_permission_db_all_view_all(self): + """This test case is used to verify the view permission with db all and view all, + the time sleep to wait the permission take effect + """ + self.prepare_data() + username = "view_test" + password = "test" + self.create_user(username, password) + # grant all db permission to user + tdSql.execute("grant all on view_db.* to view_test;") + + conn = taos.connect(user=username, password=password) + conn.execute(f"use {self.dbname};") + conn.execute("create view v1 as select * from stb;") + res = conn.query("show views;") + assert(len(res.fetch_all()) == 1) + tdLog.debug(f"Verify the show view permission of user '{username}' with db all and view all successfully") + self.check_permissions("view_test", "view_db", {"db": ["read", "write"], "view": ["read", "write", "alter"]}, "v1") + tdLog.debug(f"Verify the view permission from system table successfully") + time.sleep(2) + conn.execute("drop view v1;") + tdSql.execute("revoke all on view_db.* from view_test;") + tdSql.execute(f"drop database {self.dbname};") + time.sleep(1) + + # prepare data by user 'view_test' + self.prepare_data(conn) + + conn.execute("create view v1 as select * from stb;") + res = conn.query("show views;") + assert(len(res.fetch_all()) == 1) + tdLog.debug(f"Verify the view permission of user '{username}' with db all and view all successfully") + self.check_permissions("view_test", "view_db", {"db": ["read", "write"], "view": ["read", "write", "alter"]}, "v1") + tdLog.debug(f"Verify the view permission from system table successfully") + time.sleep(2) + conn.execute("drop view v1;") + tdSql.execute("revoke all on view_db.* from view_test;") + tdSql.execute("revoke all on view_db.v1 from view_test;") + tdSql.execute(f"drop database {self.dbname}") + tdSql.execute("drop user view_test;") + tdLog.debug("Finish test case 'test_view_permission_db_all_view_all'") + + def test_view_permission_db_write_view_all(self): + """This test case is used to verify the view permission with db write and view all + """ + username = "view_test" + password = "test" + self.create_user(username, password) + conn = taos.connect(user=username, password=password) + self.prepare_data(conn) + conn.execute("create view v1 as select * from stb;") + tdSql.execute("revoke read on view_db.* from view_test;") + self.check_permissions("view_test", "view_db", {"db": ["write"], "view": ["read", "write", "alter"]}, "v1") + # create view permission error + try: + conn.execute("create view v2 as select * from v1;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + # query from view permission error + try: + conn.query("select * from v1;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + # view query permission + res = conn.query("show views;") + assert(len(res.fetch_all()) == 1) + time.sleep(2) + conn.execute("drop view v1;") + tdSql.execute("revoke write on view_db.* from view_test;") + tdSql.execute(f"drop database {self.dbname}") + tdSql.execute("drop user view_test;") + tdLog.debug("Finish test case 'test_view_permission_db_write_view_all'") + + def test_view_permission_db_write_view_read(self): + """This test case is used to verify the view permission with db write and view read + """ + username = "view_test" + password = "test" + self.create_user(username, password) + conn = taos.connect(user=username, password=password) + self.prepare_data() + + tdSql.execute("create view v1 as select * from stb;") + tdSql.execute("grant write on view_db.* to view_test;") + tdSql.execute("grant read on view_db.v1 to view_test;") + + conn.execute(f"use {self.dbname};") + time.sleep(2) + res = conn.query("select * from v1;") + assert(len(res.fetch_all()) == 20) + + conn.execute("create view v2 as select * from v1;") + # create view from super table of database + try: + conn.execute("create view v3 as select * from stb;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + time.sleep(2) + conn.execute("drop view v2;") + try: + conn.execute("drop view v1;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + tdSql.execute("revoke read on view_db.v1 from view_test;") + tdSql.execute("revoke write on view_db.* from view_test;") + tdSql.execute(f"drop database {self.dbname}") + tdSql.execute("drop user view_test;") + tdLog.debug("Finish test case 'test_view_permission_db_write_view_read'") + + def test_view_permission_db_write_view_alter(self): + """This test case is used to verify the view permission with db write and view alter + """ + username = "view_test" + password = "test" + self.create_user(username, password) + conn = taos.connect(user=username, password=password) + self.prepare_data() + + tdSql.execute("create view v1 as select * from stb;") + tdSql.execute("grant write on view_db.* to view_test;") + tdSql.execute("grant alter on view_db.v1 to view_test;") + try: + conn.execute(f"use {self.dbname};") + conn.execute("select * from v1;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + time.sleep(2) + conn.execute("drop view v1;") + tdSql.execute("revoke write on view_db.* from view_test;") + tdSql.execute(f"drop database {self.dbname}") + tdSql.execute("drop user view_test;") + tdLog.debug("Finish test case 'test_view_permission_db_write_view_alter'") + + def test_view_permission_db_read_view_all(self): + """This test case is used to verify the view permission with db read and view all + """ + username = "view_test" + password = "test" + self.create_user(username, password) + conn = taos.connect(user=username, password=password) + self.prepare_data() + + tdSql.execute("create view v1 as select * from stb;") + tdSql.execute("grant read on view_db.* to view_test;") + tdSql.execute("grant all on view_db.v1 to view_test;") + try: + conn.execute(f"use {self.dbname};") + conn.execute("create view v2 as select * from v1;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + time.sleep(2) + res = conn.query("select * from v1;") + assert(len(res.fetch_all()) == 20) + conn.execute("drop view v1;") + tdSql.execute("revoke read on view_db.* from view_test;") + tdSql.execute(f"drop database {self.dbname}") + tdSql.execute("drop user view_test;") + tdLog.debug("Finish test case 'test_view_permission_db_read_view_all'") + + def test_view_permission_db_read_view_alter(self): + """This test case is used to verify the view permission with db read and view alter + """ + username = "view_test" + password = "test" + self.create_user(username, password) + conn = taos.connect(user=username, password=password) + self.prepare_data() + + tdSql.execute("create view v1 as select * from stb;") + tdSql.execute("grant read on view_db.* to view_test;") + tdSql.execute("grant alter on view_db.v1 to view_test;") + try: + conn.execute(f"use {self.dbname};") + conn.execute("select * from v1;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + + time.sleep(2) + conn.execute("drop view v1;") + tdSql.execute("revoke read on view_db.* from view_test;") + tdSql.execute(f"drop database {self.dbname}") + tdSql.execute("drop user view_test;") + tdLog.debug("Finish test case 'test_view_permission_db_read_view_alter'") + + def test_view_permission_db_read_view_read(self): + """This test case is used to verify the view permission with db read and view read + """ + username = "view_test" + password = "test" + self.create_user(username, password) + conn = taos.connect(user=username, password=password) + self.prepare_data() + + tdSql.execute("create view v1 as select * from stb;") + tdSql.execute("grant read on view_db.* to view_test;") + tdSql.execute("grant read on view_db.v1 to view_test;") + conn.execute(f"use {self.dbname};") + time.sleep(2) + res = conn.query("select * from v1;") + assert(len(res.fetch_all()) == 20) + try: + conn.execute("drop view v1;") + except Exception as ex: + assert("[0x2644]: Permission denied or target object not exist" in str(ex)) + tdSql.execute("revoke read on view_db.* from view_test;") + tdSql.execute("revoke read on view_db.v1 from view_test;") + tdSql.execute(f"drop database {self.dbname}") + tdSql.execute("drop user view_test;") + tdLog.debug("Finish test case 'test_view_permission_db_read_view_read'") + + def test_query_from_view(self): + """This test case is used to verify the query from view + """ + self.prepare_data() + view_name_list = [] + + # common query from super table + tdSql.execute(f"create view v1 as select * from {self.stbname};") + tdSql.query(f"select * from v1;") + rows = tdSql.queryRows + assert(rows == 20) + view_name_list.append("v1") + tdLog.debug("Verify the query from super table successfully") + + # common query from child table + tdSql.execute(f"create view v2 as select * from {self.ctbname_list[0]};") + tdSql.query(f"select * from v2;") + rows = tdSql.queryRows + assert(rows == 10) + view_name_list.append("v2") + tdLog.debug("Verify the query from child table successfully") + + # join query + tdSql.execute(f"create view v3 as select * from {self.stbname} join {self.ctbname_list[1]} on {self.ctbname_list[1]}.ts = {self.stbname}.ts;") + tdSql.query(f"select * from v3;") + rows = tdSql.queryRows + assert(rows == 10) + view_name_list.append("v3") + tdLog.debug("Verify the join query successfully") + + # group by query + tdSql.execute(f"create view v4 as select count(*) from {self.stbname} group by tbname;") + tdSql.query(f"select * from v4;") + rows = tdSql.queryRows + assert(rows == 2) + res = tdSql.queryResult + assert(res[0][0] == 10) + view_name_list.append("v4") + tdLog.debug("Verify the group by query successfully") + + # partition by query + tdSql.execute(f"create view v5 as select sum(col1) from {self.stbname} where col2 > 4 partition by tbname interval(3s);") + tdSql.query(f"select * from v5;") + rows = tdSql.queryRows + assert(rows >= 4) + view_name_list.append("v5") + tdLog.debug("Verify the partition by query successfully") + + # query from nested view + tdSql.execute(f"create view v6 as select * from v5;") + tdSql.query(f"select * from v6;") + rows = tdSql.queryRows + assert(rows >= 4) + view_name_list.append("v6") + tdLog.debug("Verify the query from nested view successfully") + + # delete view + for view in view_name_list: + tdSql.execute(f"drop view {view};") + tdLog.debug(f"Drop view {view} successfully") + tdSql.execute(f"drop database {self.dbname}") + tdLog.debug("Finish test case 'test_query_from_view'") + + def test_tmq_from_view(self): + """This test case is used to verify the tmq consume data from view + """ + # params for db + paraDict = {'dbName': 'view_db', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + # topic info + topic_name_list = ['topic1'] + view_name_list = ['view1'] + expectRowsList = [] + + self.prepare_tmq_data(paraDict) + + # init consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + queryString = "select * from %s.%s"%(paraDict['dbName'], paraDict['stbName']) + tdSql.execute(f"create view {view_name_list[0]} as {queryString}") + sqlString = "create topic %s as %s" %(topic_name_list[0], "select * from %s"%view_name_list[0]) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 1 + topicList = topic_name_list[0] + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + ifcheckdata = 1 + ifManualCommit = 1 + tmqCom.insertConsumerInfo(consumerId, expectrowcnt, topicList, keyList, ifcheckdata, ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'], paraDict["dbName"], paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + tmqCom.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topic_name_list)): + tdSql.query("drop topic %s"%topic_name_list[i]) + for i in range(len(view_name_list)): + tdSql.query("drop view %s"%view_name_list[i]) + + # drop database + tdSql.execute(f"drop database {paraDict['dbName']}") + tdSql.execute("drop database cdb;") + tdLog.debug("Finish test case 'test_tmq_from_view'") + + def run(self): + self.test_create_view_from_one_database() + self.test_create_view_from_multi_database() + self.test_create_view_name_params() + self.test_create_view_query() + self.test_show_view() + self.test_drop_view() + self.test_view_permission_db_all_view_all() + self.test_view_permission_db_write_view_all() + self.test_view_permission_db_write_view_read() + self.test_view_permission_db_write_view_alter() + self.test_view_permission_db_read_view_all() + self.test_view_permission_db_read_view_alter() + self.test_view_permission_db_read_view_read() + self.test_query_from_view() + self.test_tmq_from_view() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/1-insert/insert_stb.py b/tests/system-test/1-insert/insert_stb.py index 605bee7d8b..2cdd3ea7c2 100644 --- a/tests/system-test/1-insert/insert_stb.py +++ b/tests/system-test/1-insert/insert_stb.py @@ -176,7 +176,7 @@ class TDTestCase: tdSql.checkData(3, 2, 10) tdSql.checkData(4, 0, datetime.datetime(2021, 4, 19, 8, 0, 13)) tdSql.checkData(4, 1, 1) - tdSql.checkData(4, 2, None) + tdSql.checkData(4, 2, 1) def run_insert_stb(self): print("running {}".format('insert_stb')) diff --git a/tests/system-test/1-insert/test_ts4219.py b/tests/system-test/1-insert/test_ts4219.py new file mode 100644 index 0000000000..e6447d77ae --- /dev/null +++ b/tests/system-test/1-insert/test_ts4219.py @@ -0,0 +1,27 @@ +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import tdDnodes +from math import inf + + +class TDTestCase: + def init(self, conn, logSql, replicaVer=1): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), True) + + def prepare_data(self): + tdSql.execute("create database db;") + tdSql.execute("use db;") + tdSql.execute("create stable st(ts timestamp, c1 int, c2 float) tags(groupname binary(32));") + + def run(self): + tdSql.error("insert into ct1 using st tags('group name 1') values(now, 1, 1.1)(now+1s, 2, 2.2) ct1 using st tags('group 1) values(now+2s, 3, 3.3); ") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/last_cache_scan.py b/tests/system-test/2-query/last_cache_scan.py new file mode 100644 index 0000000000..0f0936ebab --- /dev/null +++ b/tests/system-test/2-query/last_cache_scan.py @@ -0,0 +1,334 @@ +import taos +import sys +import time +import socket +import os +import threading +import math +from datetime import datetime + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +# from tmqCommon import * + +COMPARE_DATA = 0 +COMPARE_LEN = 1 + +class TDTestCase: + def __init__(self): + self.vgroups = 4 + self.ctbNum = 10 + self.rowsPerTbl = 10000 + self.duraion = '1h' + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=2,replica=1, duration:str='1d'): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d duration %s CACHEMODEL 'both'"%(dbName, vgroups, replica, duration)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, paraDict): + colString = tdCom.gen_column_type_str(colname_prefix=paraDict["colPrefix"], column_elm_list=paraDict["colSchema"]) + tagString = tdCom.gen_tag_type_str(tagname_prefix=paraDict["tagPrefix"], tag_elm_list=paraDict["tagSchema"]) + sqlString = f"create table if not exists %s.%s (%s) tags (%s)"%(paraDict["dbName"], paraDict["stbName"], colString, tagString) + tdLog.debug("%s"%(sqlString)) + tsql.execute(sqlString) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1,ctbStartIdx=0): + for i in range(ctbNum): + sqlString = "create table %s.%s%d using %s.%s tags(%d, 'tb%d', 'tb%d', %d, %d, %d)" % \ + (dbName,ctbPrefix,i+ctbStartIdx,dbName,stbName,(i+ctbStartIdx) % 5,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx,i+ctbStartIdx) + tsql.execute(sqlString) + + tdLog.debug("complete to create %d child tables by %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs,tsStep): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + for i in range(ctbNum): + rowsBatched = 0 + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (i < ctbNum/2): + sql += "(%d, %d, %d, %d,%d,%d,%d,true,'binary%d', 'nchar%d', %d) "%(startTs + j*tsStep, j%1000, j%500, j%1000, j%5000, j%5400, j%128, j%10000, j%1000, startTs+j*tsStep+1000) + else: + sql += "(%d, %d, NULL, %d,NULL,%d,%d,true,'binary%d', 'nchar%d', %d) "%(startTs + j*tsStep, j%1000, j%500, j%1000, j%128, j%10000, j%1000, startTs + j*tsStep + 1000) + rowsBatched += 1 + if ((rowsBatched == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsBatched = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + if sql != pre_insert: + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def prepareTestEnv(self): + tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") + paraDict = {'dbName': 'test', + 'dropFlag': 1, + 'vgroups': 2, + 'stbName': 'meters', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, + {'type': 'BIGINT', 'count':1}, + {'type': 'FLOAT', 'count':1}, + {'type': 'DOUBLE', 'count':1}, + {'type': 'smallint', 'count':1}, + {'type': 'tinyint', 'count':1}, + {'type': 'bool', 'count':1}, + {'type': 'binary', 'len':10, 'count':1}, + {'type': 'nchar', 'len':10, 'count':1}, + {'type': 'timestamp', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1},{'type': 'nchar', 'len':20, 'count':1},{'type': 'binary', 'len':20, 'count':1},{'type': 'BIGINT', 'count':1},{'type': 'smallint', 'count':1},{'type': 'DOUBLE', 'count':1}], + 'ctbPrefix': 't', + 'ctbStartIdx': 0, + 'ctbNum': 100, + 'rowsPerTbl': 10000, + 'batchNum': 3000, + 'startTs': 1537146000000, + 'tsStep': 600000} + + paraDict['vgroups'] = self.vgroups + paraDict['ctbNum'] = self.ctbNum + paraDict['rowsPerTbl'] = self.rowsPerTbl + + tdLog.info("create database") + self.create_database(tsql=tdSql, dbName=paraDict["dbName"], dropFlag=paraDict["dropFlag"], vgroups=paraDict["vgroups"], replica=self.replicaVar, duration=self.duraion) + + tdLog.info("create stb") + self.create_stable(tsql=tdSql, paraDict=paraDict) + + tdLog.info("create child tables") + self.create_ctable(tsql=tdSql, dbName=paraDict["dbName"], \ + stbName=paraDict["stbName"],ctbPrefix=paraDict["ctbPrefix"],\ + ctbNum=paraDict["ctbNum"],ctbStartIdx=paraDict["ctbStartIdx"]) + self.insert_data(tsql=tdSql, dbName=paraDict["dbName"],\ + ctbPrefix=paraDict["ctbPrefix"],ctbNum=paraDict["ctbNum"],\ + rowsPerTbl=paraDict["rowsPerTbl"],batchNum=paraDict["batchNum"],\ + startTs=paraDict["startTs"],tsStep=paraDict["tsStep"]) + return + + def check_explain_res_has_row(self, plan_str_expect: str, rows, sql): + plan_found = False + for row in rows: + if str(row).find(plan_str_expect) >= 0: + tdLog.debug("plan: [%s] found in: [%s]" % (plan_str_expect, str(row))) + plan_found = True + break + if not plan_found: + tdLog.exit("plan: %s not found in res: [%s] in sql: %s" % (plan_str_expect, str(rows), sql)) + + def check_explain_res_no_row(self, plan_str_not_expect: str, res, sql): + for row in res: + if str(row).find(plan_str_not_expect) >= 0: + tdLog.exit('plan: [%s] found in: [%s] for sql: %s' % (plan_str_not_expect, str(row), sql)) + + def explain_sql(self, sql: str): + sql = "explain verbose true " + sql + tdSql.query(sql, queryTimes=1) + return tdSql.queryResult + + def explain_and_check_res(self, sqls, hasLastRowScanRes): + for sql, has_last in zip(sqls, hasLastRowScanRes): + res = self.explain_sql(sql) + if has_last == 1: + self.check_explain_res_has_row("Last Row Scan", res, sql) + else: + self.check_explain_res_no_row("Last Row Scan", res, sql) + + def format_sqls(self, sql_template, select_items): + sqls = [] + for item in select_items: + sqls.append(sql_template % item) + return sqls + + def query_check_one(self, sql, res_expect): + if res_expect is not None: + tdSql.query(sql, queryTimes=1) + tdSql.checkRows(1) + for i in range(0, tdSql.queryCols): + tdSql.checkData(0, i, res_expect[i]) + tdLog.info('%s check res col: %d succeed value: %s' % (sql, i, str(res_expect[i]))) + + def query_check_sqls(self, sqls, has_last_row_scan_res, res_expect): + for sql, has_last, res in zip(sqls, has_last_row_scan_res, res_expect): + if has_last == 1: + self.query_check_one(sql, res) + + def test_last_cache_scan(self): + sql_template = 'select %s from meters' + select_items = [ + "last(ts), ts", "last(ts), c1", "last(ts), c2", "last(ts), c3",\ + "last(ts), c4", "last(ts), tbname", "last(ts), t1", "last(ts), ts, ts"] + has_last_row_scan_res = [1, 0, 0, 0, 0, 0, 0, 1] + res_expect = [ + ["2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"], + None, None, None, None, None, None, + ["2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"] + ] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + self.query_check_sqls(sqls, has_last_row_scan_res, res_expect) + + select_items = ["last(c1),ts", "last(c1), c1", "last(c1), c2", "last(c1), c3",\ + "last(c1), c4", "last(c1), tbname", "last(c1), t1", "last(c1), ts, ts", "last(c1), c1, c1"] + has_last_row_scan_res = [1, 1, 0, 0, 0, 0, 0, 1, 1] + res_expect = [ + [999, "2018-11-25 19:30:00.000"], + [999, 999], None, None, None, None, None, + [999, "2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"], + [999,999,999] + ] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + self.query_check_sqls(sqls, has_last_row_scan_res, res_expect) + + sql_template = 'select %s from t1' + select_items = ["last(c4),ts", "last(c4), c1", "last(c4), c2", "last(c4), c3",\ + "last(c4), c4", "last(c4), tbname", "last(c4), t1"] + has_last_row_scan_res = [1, 0, 0, 0, 1, 0, 0] + res_expect = [ + [4999.000000000000000, "2018-11-25 19:30:00.000"], + None,None,None, + [4999.000000000000000, 4999.000000000000000] + ] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + self.query_check_sqls(sqls, has_last_row_scan_res, res_expect) + + sql_template = 'select %s from meters' + select_items = ["last(c8), ts", "last(c8), c1", "last(c8), c8", "last(c8), tbname", \ + "last(c8), t1", "last(c8), c8, c8", "last(c8), ts, ts"] + has_last_row_scan_res = [1, 0, 1, 0, 0, 1, 1] + res_expect = [ + ["binary9999", "2018-11-25 19:30:00.000"], + None, + ["binary9999", "binary9999"], + None, None, + ["binary9999", "binary9999", "binary9999"], + ["binary9999", "2018-11-25 19:30:00.000", "2018-11-25 19:30:00.000"] + ] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + self.query_check_sqls(sqls, has_last_row_scan_res, res_expect) + + # c2, c4 in last row of t5,t6,t7,t8,t9 will always be NULL + sql_template = 'select %s from t5' + select_items = ["last(c4), ts", "last(c4), c4", "last(c4), c4, c4", "last(c4), ts, ts"] + has_last_row_scan_res = [1,1,1,1] + + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + #for sql in sqls: + #tdSql.query(sql, queryTimes=1) + #tdSql.checkRows(0) + + sql_template = 'select %s from meters' + select_items = [ + "last_row(ts), last(ts)", + "last_row(c1), last(c1)", + "last_row(c1), c1,c3, ts" + ] + has_last_row_scan_res = [0,0,1] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + #res_expect = [None, None, [999, 999, 499, "2018-11-25 19:30:00.000"]] + #self.query_check_sqls(sqls, has_last_row_scan_res, res_expect) + + select_items = ["last(c10), c10", + "last(c10), ts", + "last(c10), c10, ts", + "last(c10), c10, ts, c10,ts", + "last(c10), ts, c1"] + has_last_row_scan_res = [1,1,1,1,0] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + res_expect = [ + ["2018-11-25 19:30:01.000", "2018-11-25 19:30:01.000"], + ["2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000"], + ["2018-11-25 19:30:01.000", "2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000"], + ["2018-11-25 19:30:01.000", "2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000", "2018-11-25 19:30:01.000", "2018-11-25 19:30:00.000"] + ] + self.query_check_sqls(sqls, has_last_row_scan_res, res_expect) + + sql = "select last(c1), c1, c1+1, c1+2, ts from meters" + res = self.explain_sql(sql) + self.check_explain_res_has_row("Last Row Scan", res, sql) + + tdSql.query(sql) + tdSql.checkRows(1) + tdSql.checkData(0, 0, 999) + tdSql.checkData(0, 1, 999) + tdSql.checkData(0, 2, 1000) + tdSql.checkData(0, 3, 1001) + tdSql.checkData(0, 4, "2018-11-25 19:30:00.000") + + sql_template = 'select %s from meters partition by tbname' + select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, tbname", "last(c10), c10, ts"] + has_last_row_scan_res = [1,1,1] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + tdSql.query(sqls[0], queryTimes=1) + tdSql.checkRows(10) + tdSql.checkData(0,0, '2018-11-25 19:30:00.000') + tdSql.checkData(0,1, '2018-11-25 19:30:01.000') + tdSql.checkData(0,2, '2018-11-25 19:30:01.000') + tdSql.checkData(0,3, '2018-11-25 19:30:00.000') + + tdSql.query(sqls[1], queryTimes=1) + tdSql.checkRows(10) + tdSql.checkData(0,0, '2018-11-25 19:30:00.000') + tdSql.checkData(0,1, '2018-11-25 19:30:00.000') + tdSql.checkData(0,2, '2018-11-25 19:30:01.000') + tdSql.checkData(0,3, '2018-11-25 19:30:01.000') + + sql_template = 'select %s from meters partition by t1' + select_items = ["ts, last(c10), c10, ts", "ts, ts, last(c10), c10, t1", "last(c10), c10, ts"] + has_last_row_scan_res = [1,1,1] + sqls = self.format_sqls(sql_template, select_items) + self.explain_and_check_res(sqls, has_last_row_scan_res) + tdSql.query(sqls[0], queryTimes=1) + tdSql.checkRows(5) + tdSql.checkData(0,0, '2018-11-25 19:30:00.000') + tdSql.checkData(0,1, '2018-11-25 19:30:01.000') + tdSql.checkData(0,2, '2018-11-25 19:30:01.000') + tdSql.checkData(0,3, '2018-11-25 19:30:00.000') + + tdSql.query("select ts, last(c10), t1, t2 from meters partition by t1, t2") + tdSql.checkRows(10) + tdSql.checkData(0, 0, '2018-11-25 19:30:00.000') + tdSql.checkData(0, 1, '2018-11-25 19:30:01.000') + + def run(self): + self.prepareTestEnv() + #time.sleep(99999999) + self.test_last_cache_scan() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())