diff --git a/cmake/cmake.define b/cmake/cmake.define index 56b6b7e1de..44b36d0efa 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -170,7 +170,7 @@ ELSE () SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx2") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2") ENDIF() - MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2/AVX512) is ACTIVATED") + MESSAGE(STATUS "SIMD instructions (FMA/AVX/AVX2) is ACTIVATED") IF (COMPILER_SUPPORT_AVX512F AND COMPILER_SUPPORT_AVX512BMI) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mavx512f -mavx512vbmi") diff --git a/docs/en/13-operation/07-import.md b/docs/en/13-operation/07-import.md index e95824e927..be0b988fc0 100644 --- a/docs/en/13-operation/07-import.md +++ b/docs/en/13-operation/07-import.md @@ -59,4 +59,4 @@ Query OK, 9 row(s) affected (0.004763s) ## Import using taosdump -A convenient tool for importing and exporting data is provided by TDengine, `taosdump`, which can be used to export data from one TDengine cluster and import into another one. For the details of using `taosdump` please refer to [Tool for exporting and importing data: taosdump](/reference/taosdump). +A convenient tool for importing and exporting data is provided by TDengine, `taosdump`, which can be used to export data from one TDengine cluster and import into another one. For the details of using `taosdump` please refer to the taosdump documentation. diff --git a/docs/en/13-operation/08-export.md b/docs/en/13-operation/08-export.md index bffda36e23..580844cf08 100644 --- a/docs/en/13-operation/08-export.md +++ b/docs/en/13-operation/08-export.md @@ -19,4 +19,4 @@ The data of table or STable specified by `tb_name` will be exported into a file ## Export Using taosdump -With `taosdump`, you can choose to export the data of all databases, a database, a table or a STable, you can also choose to export the data within a time range, or even only export the schema definition of a table. For the details of using `taosdump` please refer to [Tool for exporting and importing data: taosdump](/reference/taosdump). +With `taosdump`, you can choose to export the data of all databases, a database, a table or a STable, you can also choose to export the data within a time range, or even only export the schema definition of a table. For the details of using `taosdump` please refer to the taosdump documentation. diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index b08216a9c4..f1be4c5fd3 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -11,8 +11,6 @@ The collection of the monitoring information is enabled by default, but can be d TDinsight is a complete solution which uses the monitoring database `log` mentioned previously, and Grafana, to monitor a TDengine cluster. -Please refer to [TDinsight Grafana Dashboard](../../reference/tdinsight) to learn more details about using TDinsight to monitor TDengine. - A script `TDinsight.sh` is provided to deploy TDinsight automatically. Download `TDinsight.sh` with the below command: diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index fd43dd67fa..f770ce0d5d 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -36,6 +36,7 @@ REST connection supports all platforms that can run Java. | taos-jdbcdriver version | major changes | TDengine version | | :---------------------: | :------------------------------------------------------------------------------------------------------------------------------------------------: | :--------------: | +| 3.2.7 | Support VARBINARY and GEOMETRY types, and add time zone support for native connections. Support websocket auto reconnection | 3.2.0.0 or later | | 3.2.5 | Subscription add committed() and assignment() method | 3.1.0.3 or later | | 3.2.4 | Subscription add the enable.auto.commit parameter and the unsubscribe() method in the WebSocket connection | - | | 3.2.3 | Fixed resultSet data parsing failure in some cases | - | @@ -178,7 +179,7 @@ Add following dependency in the `pom.xml` file of your Maven project: com.taosdata.jdbc taos-jdbcdriver - 3.2.2 + 3.2.7 ``` diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index 8e5ee178a4..4744e143fc 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -13,7 +13,7 @@ taosBenchmark (formerly taosdemo ) is a tool for testing the performance of TDen There are two ways to install taosBenchmark: -- Installing the official TDengine installer will automatically install taosBenchmark. +- Installing the official TDengine installer will automatically install taosBenchmark. - Compile taos-tools separately and install them. Please refer to the [taos-tools](https://github.com/taosdata/taos-tools) repository for details. diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx index 8475888df5..f7d1a2db7e 100644 --- a/docs/en/20-third-party/01-grafana.mdx +++ b/docs/en/20-third-party/01-grafana.mdx @@ -218,7 +218,7 @@ The example to query the average system memory usage for the specified interval ### Importing the Dashboard -You can install TDinsight dashboard in data source configuration page (like `http://localhost:3000/datasources/edit/1/dashboards`) as a monitoring visualization tool for TDengine cluster. Ensure that you use TDinsight for 3.x. Please note TDinsight for 3.x needs to configure and run taoskeeper correctly. Check the [TDinsight User Manual](/reference/tdinsight/) for the details. +You can install TDinsight dashboard in data source configuration page (like `http://localhost:3000/datasources/edit/1/dashboards`) as a monitoring visualization tool for TDengine cluster. Ensure that you use TDinsight for 3.x. Please note TDinsight for 3.x needs to configure and run taoskeeper correctly. ![TDengine Database Grafana plugine import dashboard](./import_dashboard.webp) diff --git a/docs/en/20-third-party/11-kafka.md b/docs/en/20-third-party/11-kafka.md index 64c0f0bd48..b865c00bc3 100644 --- a/docs/en/20-third-party/11-kafka.md +++ b/docs/en/20-third-party/11-kafka.md @@ -21,7 +21,7 @@ TDengine Source Connector is used to read data from TDengine in real-time and se 1. Linux operating system 2. Java 8 and Maven installed 3. Git/curl/vi is installed -4. TDengine is installed and started. +4. TDengine is installed and started. ## Install Kafka diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index 237e3ef8f9..e2a0a85b9a 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -36,6 +36,7 @@ REST 连接支持所有能运行 Java 的平台。 | taos-jdbcdriver 版本 | 主要变化 | TDengine 版本 | | :------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------: | :----------------: | +| 3.2.7 | 支持VARBINARY和GEOMETRY类型，增加native连接的时区设置支持。增加websocket自动重连功能。 | 3.2.0.0 及更高版本 | | 3.2.5 | 数据订阅增加 committed()、assignment() 方法 | 3.1.0.3 及更高版本 | | 3.2.4 | 数据订阅在 WebSocket 连接下增加 enable.auto.commit 参数，以及 unsubscribe() 方法。 | - | | 3.2.3 | 修复 ResultSet 在一些情况数据解析失败 | - | @@ -177,7 +178,7 @@ Maven 项目中，在 pom.xml 中添加以下依赖： com.taosdata.jdbc taos-jdbcdriver - 3.2.2 + 3.2.7 ``` @@ -1097,7 +1098,6 @@ TaosConsumer consumer = new TaosConsumer<>(config); - httpPoolSize: 同一个连接下最大并行请求数。仅在 WebSocket 连接下有效。 其他参数请参考：[Consumer 参数列表](../../develop/tmq#创建-consumer-以及consumer-group)， 注意TDengine服务端自3.2.0.0版本开始消息订阅中的auto.offset.reset默认值发生变化。 - #### 订阅消费数据 ```java diff --git a/docs/zh/08-connector/05-schemaless-api.mdx b/docs/zh/08-connector/_05-schemaless.mdx similarity index 100% rename from docs/zh/08-connector/05-schemaless-api.mdx rename to docs/zh/08-connector/_05-schemaless.mdx diff --git a/docs/zh/10-deployment/03-k8s.md b/docs/zh/10-deployment/03-k8s.md index 16e2be0dfd..31e909f02d 100644 --- a/docs/zh/10-deployment/03-k8s.md +++ b/docs/zh/10-deployment/03-k8s.md @@ -105,7 +105,7 @@ spec: # TZ for timezone settings, we recommend to always set it. - name: TZ value: "Asia/Shanghai" - # TAOS_ prefix will configured in taos.cfg, strip prefix and camelCase. + # Environment variables with prefix TAOS_ will be parsed and converted into corresponding parameter in taos.cfg. For example, serverPort in taos.cfg should be configured by TAOS_SERVER_PORT when using K8S to deploy - name: TAOS_SERVER_PORT value: "6030" # Must set if you want a cluster. diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index e9ca5405f4..bd33281bc0 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -53,7 +53,7 @@ database_option: { - 1：表示一阶段压缩。 - 2：表示两阶段压缩。 - DURATION：数据文件存储数据的时间跨度。可以使用加单位的表示形式，如 DURATION 100h、DURATION 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。不加时间单位时默认单位为天，如 DURATION 50 表示 50 天。 -- WAL_FSYNC_PERIOD：当 WAL 参数设置为 2 时，落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 +- WAL_FSYNC_PERIOD：当 WAL_LEVEL 参数设置为 2 时，用于设置落盘的周期。默认为 3000，单位毫秒。最小为 0，表示每次写入立即落盘；最大为 180000，即三分钟。 - MAXROWS：文件块中记录的最大条数，默认为 4096 条。 - MINROWS：文件块中记录的最小条数，默认为 100 条。 - KEEP：表示数据文件保存的天数，缺省值为 3650，取值范围 [1, 365000]，且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式，如 KEEP 100h、KEEP 10d 等，支持 m（分钟）、h（小时）和 d（天）三个单位。也可以不写单位，如 KEEP 50，此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间（多个以英文逗号分隔，最多 3 个，满足 keep 0 <= keep 1 <= keep 2，如 KEEP 100h,100d,3650d）; 社区版不支持多级存储功能（即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间）。 diff --git a/docs/zh/17-operation/04-import.md b/docs/zh/17-operation/04-import.md index 17945be595..e2c35b36c6 100644 --- a/docs/zh/17-operation/04-import.md +++ b/docs/zh/17-operation/04-import.md @@ -59,4 +59,4 @@ Query OK, 9 row(s) affected (0.004763s) ## taosdump 工具导入 -TDengine 提供了方便的数据库导入导出工具 taosdump。用户可以将 taosdump 从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见：[TDengine 数据备份工具: taosdump](/reference/taosdump)。 +TDengine 提供了方便的数据库导入导出工具 taosdump。用户可以将 taosdump 从一个系统导出的数据，导入到其他系统中。具体使用方法，请参考 taosdump 的相关文档。 diff --git a/docs/zh/17-operation/05-export.md b/docs/zh/17-operation/05-export.md index 44247e28bd..3d4425a792 100644 --- a/docs/zh/17-operation/05-export.md +++ b/docs/zh/17-operation/05-export.md @@ -17,5 +17,4 @@ select * from >> data.csv; ## 用 taosdump 导出数据 -利用 taosdump，用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表，所有数据或一时间段的数据，甚至仅仅表的定义。具体使用方法，请参见： -[TDengine 数据备份工具: taosdump](/reference/taosdump)。 +利用 taosdump，用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表，所有数据或一时间段的数据，甚至仅仅表的定义。具体使用方法，请参考 taosdump 的相关文档。 \ No newline at end of file diff --git a/docs/zh/20-third-party/01-grafana.mdx b/docs/zh/20-third-party/01-grafana.mdx index 9a1223ab9c..8e17ce4768 100644 --- a/docs/zh/20-third-party/01-grafana.mdx +++ b/docs/zh/20-third-party/01-grafana.mdx @@ -218,7 +218,7 @@ docker run -d \ ### 导入 Dashboard -在数据源配置页面，您可以为该数据源导入 TDinsight 面板，作为 TDengine 集群的监控可视化工具。如果 TDengine 服务端为 3.0 版本请选择 `TDinsight for 3.x` 导入。注意 TDinsight for 3.x 需要运行和配置 taoskeeper，相关使用说明请见 [TDinsight 用户手册](/reference/tdinsight/)。 +在数据源配置页面，您可以为该数据源导入 TDinsight 面板，作为 TDengine 集群的监控可视化工具。如果 TDengine 服务端为 3.0 版本请选择 `TDinsight for 3.x` 导入。注意 TDinsight for 3.x 需要运行和配置 taoskeeper。 ![TDengine Database Grafana plugine import dashboard](./import_dashboard.webp) diff --git a/docs/zh/21-tdinternal/07-tsz.md b/docs/zh/21-tdinternal/07-tsz.md new file mode 100644 index 0000000000..db1a340ab8 --- /dev/null +++ b/docs/zh/21-tdinternal/07-tsz.md @@ -0,0 +1,69 @@ +--- +title: TSZ 压缩算法 +description: TDengine 对浮点数进行高效压缩的算法 +--- + +TSZ 压缩算法是 TDengine 为浮点数据类型提供的可选压缩算法，可以实现浮点数有损至无损全状态压缩，相比默认压缩算法， TSZ 压缩算法压缩率更高，即使切至无损状态，压缩率也会比默认压缩高一倍。 + +## 适合场景 + +- TSZ 压缩算法是通过数据预测技术完成的压缩，所以更适合有规律变化的数据 +- TSZ 压缩时间会更长一些，如果您的服务器 CPU 空闲多，存储空间小的情况下适合选用 + +## 使用步骤 +- TDengine 支持版本为 3.2.0.0 或以上 +- 开启选项 + 在 taos.cfg 配置中增加以下内容，即可开启 TSZ 压缩算法，功能打开后，会替换默认算法。 + 以下表示字段类型是 float 及 double 类型都使用此压缩算法，也可以单独只配置一个 + +```sql + lossyColumns float|double +``` + +- 配置需重启服务生效 +- Taosd 日志输出以下内容，表明功能已生效： + +```sql + 02/22 10:49:27.607990 00002933 UTL lossyColumns float|double +``` + +## 配置参数 + +### fPrecision +FLOAT 类型精度控制： + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 float 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.00000001 | +| 缺省值 | 0.00000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + + + +### dPrecision +DOUBLE 类型精度控制： + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 设置 double 类型浮点数压缩精度 | +| 取值范围 | 0.1 ~ 0.0000000000000001 | +| 缺省值 | 0.0000000000000001 | +| 补充说明 | 小于此值的浮点数尾数部分将被截取 | + + +### ifAdtFse +TSZ 压缩中可选择的算法 FSE，默认为 HUFFMAN： + +| 属性 | 说明 | +| -------- | -------------------------------- | +| 适用范围 | 服务器端 | +| 含义 | 使用 FSE 算法替换 HUFFMAN 算法， FSE 算法压缩速度更快，但解压稍慢，追求压缩速度可选用此算法 | +| 取值范围 | 0：关闭 1：打开 | +| 缺省值 | 0：关闭 | + + +## 注意事项 +- 打开 TSZ 后生成的存储数据格式，回退至 3.2.0.0 之前的版本，数据将不能被识别 diff --git a/examples/JDBC/taosdemo/pom.xml b/examples/JDBC/taosdemo/pom.xml index ff64d3e1df..031d83b084 100644 --- a/examples/JDBC/taosdemo/pom.xml +++ b/examples/JDBC/taosdemo/pom.xml @@ -67,7 +67,7 @@ com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.2.7 diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8688e07932..58517a5db0 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -217,13 +217,13 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); void taosCleanupCfg(); -void taosCfgDynamicOptions(const char *option, const char *value); + +int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer); struct SConfig *taosGetCfg(); void taosSetAllDebugFlag(int32_t flag, bool rewrite); void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite); -int32_t taosApplyLocalCfg(SConfig *pCfg, char *name); void taosLocalCfgForbiddenToChange(char *name, bool *forbidden); int8_t taosGranted(); diff --git a/include/common/ttime.h b/include/common/ttime.h index 306b5105d0..1dfa609064 100644 --- a/include/common/ttime.h +++ b/include/common/ttime.h @@ -118,6 +118,13 @@ int32_t taosChar2Ts(const char* format, SArray** formats, const char* tsStr, int void TEST_ts2char(const char* format, int64_t ts, int32_t precision, char* out, int32_t outLen); int32_t TEST_char2ts(const char* format, int64_t* ts, int32_t precision, const char* tsStr); +/// @brief get offset seconds from zero timezone to input timezone +/// for +XX timezone, the offset to zero is negative value +/// @param tzStr timezonestr, eg: +0800, -0830, -08 +/// @param offset seconds, eg: +08 offset -28800, -01 offset 3600 +/// @return 0 success, other fail +int32_t offsetOfTimezone(char* tzStr, int64_t* offset); + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/storageapi.h b/include/libs/executor/storageapi.h index f5392f02b1..d5f1da957d 100644 --- a/include/libs/executor/storageapi.h +++ b/include/libs/executor/storageapi.h @@ -119,6 +119,7 @@ typedef struct SRowBuffPos { bool beFlushed; bool beUsed; bool needFree; + bool beUpdated; } SRowBuffPos; // tq @@ -387,11 +388,13 @@ typedef struct SStateStore { int32_t (*streamStateStateAddIfNotExist)(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); int32_t (*streamStateSessionGetKeyByRange)(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); + int32_t (*streamStateSessionAllocWinBuffByNextPosition)(SStreamState* pState, SStreamStateCur* pCur, + const SSessionKey* pKey, void** pVal, int32_t* pVLen); SUpdateInfo* (*updateInfoInit)(int64_t interval, int32_t precision, int64_t watermark, bool igUp); - TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); - bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); - bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); + TSKEY (*updateInfoFillBlockData)(SUpdateInfo* pInfo, SSDataBlock* pBlock, int32_t primaryTsCol); + bool (*updateInfoIsUpdated)(SUpdateInfo* pInfo, uint64_t tableId, TSKEY ts); + bool (*updateInfoIsTableInserted)(SUpdateInfo* pInfo, int64_t tbUid); void (*updateInfoDestroy)(SUpdateInfo* pInfo); void (*windowSBfDelete)(SUpdateInfo *pInfo, uint64_t count); void (*windowSBfAdd)(SUpdateInfo *pInfo, uint64_t count); diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index b7f100733b..d0a2b311ee 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -57,6 +57,8 @@ int32_t streamStateSessionDel(SStreamState* pState, const SSessionKey* key); int32_t streamStateSessionClear(SStreamState* pState); int32_t streamStateSessionGetKVByCur(SStreamStateCur* pCur, SSessionKey* pKey, void** pVal, int32_t* pVLen); int32_t streamStateSessionGetKeyByRange(SStreamState* pState, const SSessionKey* range, SSessionKey* curKey); +int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur, + const SSessionKey* pKey, void** pVal, int32_t* pVLen); SStreamStateCur* streamStateSessionSeekKeyNext(SStreamState* pState, const SSessionKey* key); SStreamStateCur* streamStateSessionSeekKeyCurrentPrev(SStreamState* pState, const SSessionKey* key); @@ -66,6 +68,7 @@ SStreamStateCur* streamStateSessionSeekKeyCurrentNext(SStreamState* pState, cons int32_t streamStateStateAddIfNotExist(SStreamState* pState, SSessionKey* key, char* pKeyData, int32_t keyDataLen, state_key_cmpr_fn fn, void** pVal, int32_t* pVLen); +// fill int32_t streamStateFillPut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); int32_t streamStateFillGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); int32_t streamStateFillDel(SStreamState* pState, const SWinKey* key); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 1505b42524..3ce2de476e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -825,6 +825,7 @@ int32_t streamMetaGetNumOfTasks(SStreamMeta* pMeta); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); int32_t streamMetaReopen(SStreamMeta* pMeta); +void streamMetaInitBackend(SStreamMeta* pMeta); int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaLoadAllTasks(SStreamMeta* pMeta); int64_t streamMetaGetLatestCheckpointId(SStreamMeta* pMeta); diff --git a/include/libs/stream/tstreamFileState.h b/include/libs/stream/tstreamFileState.h index c1974df7de..2a129c1830 100644 --- a/include/libs/stream/tstreamFileState.h +++ b/include/libs/stream/tstreamFileState.h @@ -76,8 +76,10 @@ int32_t getRowStateRowSize(SStreamFileState* pFileState); int32_t getSessionWinResultBuff(SStreamFileState* pFileState, SSessionKey* pKey, TSKEY gap, void** pVal, int32_t* pVLen); int32_t putSessionWinResultBuff(SStreamFileState* pFileState, SRowBuffPos* pPos); int32_t getSessionFlushedBuff(SStreamFileState* pFileState, SSessionKey* pKey, void** pVal, int32_t* pVLen); -int32_t deleteSessionWinStateBuffFn(void* pBuff, const void *key, size_t keyLen); +int32_t deleteSessionWinStateBuffFn(void* pBuff, const void* key, size_t keyLen); int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffPos* pPos); +int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, + const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen); SRowBuffPos* createSessionWinBuff(SStreamFileState* pFileState, SSessionKey* pKey, void* p, int32_t* pVLen); int32_t recoverSesssion(SStreamFileState* pFileState, int64_t ckId); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 7665550153..a3ee294338 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -172,7 +172,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0X0231) // mnode-common -// #define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) // 2.x +#define TSDB_CODE_MND_REQ_REJECTED TAOS_DEF_ERROR_CODE(0, 0x0300) // #define TSDB_CODE_MND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0301) // 2.x // #define TSDB_CODE_MND_ACTION_NEED_REPROCESSEDTAOS_DEF_ERROR_CODE(0, 0x0302) // 2.x #define TSDB_CODE_MND_NO_RIGHTS TAOS_DEF_ERROR_CODE(0, 0x0303) @@ -640,6 +640,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SCH_IGNORE_ERROR TAOS_DEF_ERROR_CODE(0, 0x2503) #define TSDB_CODE_SCH_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x2504) #define TSDB_CODE_SCH_JOB_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x2505) +#define TSDB_CODE_SCH_JOB_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2506) //parser #define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600) diff --git a/include/util/tcompression.h b/include/util/tcompression.h index 3a3d13117e..ab0c22fc9b 100644 --- a/include/util/tcompression.h +++ b/include/util/tcompression.h @@ -30,6 +30,10 @@ extern "C" { #define INT64MASK(_x) ((((uint64_t)1) << _x) - 1) #define INT32MASK(_x) (((uint32_t)1 << _x) - 1) #define INT8MASK(_x) (((uint8_t)1 << _x) - 1) + +#define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode +#define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode + // Compression algorithm #define NO_COMPRESSION 0 #define ONE_STAGE_COMP 1 @@ -129,6 +133,12 @@ int32_t tsCompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32 int32_t nBuf); int32_t tsDecompressBigint(void *pIn, int32_t nIn, int32_t nEle, void *pOut, int32_t nOut, uint8_t cmprAlg, void *pBuf, int32_t nBuf); +// for internal usage +int32_t getWordLength(char type); + +int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type); +int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output); +int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output); /************************************************************************* * STREAM COMPRESSION diff --git a/include/util/tdef.h b/include/util/tdef.h index 1b56b5b623..69d0c1126d 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -34,7 +34,6 @@ extern "C" { // Bytes for each type. extern const int32_t TYPE_BYTES[21]; -// TODO: replace and remove code below #define CHAR_BYTES sizeof(char) #define SHORT_BYTES sizeof(int16_t) #define INT_BYTES sizeof(int32_t) diff --git a/include/util/tunit.h b/include/util/tunit.h new file mode 100644 index 0000000000..de37c85929 --- /dev/null +++ b/include/util/tunit.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UNIT_H_ +#define _TD_UNIT_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int64_t taosStrHumanToInt64(const char* str); +void taosInt64ToHumanStr(int64_t val, char* outStr); + +int32_t taosStrHumanToInt32(const char* str); +void taosInt32ToHumanStr(int32_t val, char* outStr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UNIT_H_*/ diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 0a11ef3a53..02ebb182fa 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -34,6 +34,7 @@ benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" xname="taosx" +keeperName="taoskeeper" clientName2="taos" serverName2="${clientName2}d" @@ -42,6 +43,7 @@ productName2="TDengine" emailName2="taosdata.com" xname2="${clientName2}x" adapterName2="${clientName2}adapter" +keeperName2="${clientName2}keeper" explorerName="${clientName2}-explorer" benchmarkName2="${clientName2}Benchmark" @@ -154,7 +156,7 @@ interactiveFqdn=yes # [yes | no] verType=server # [server | client] initType=systemd # [systemd | service | ...] -while getopts "hv:e:i:" arg; do +while getopts "hv:e:" arg; do case $arg in e) #echo "interactiveFqdn=$OPTARG" @@ -164,10 +166,6 @@ while getopts "hv:e:i:" arg; do #echo "verType=$OPTARG" verType=$(echo $OPTARG) ;; - i) - #echo "initType=$OPTARG" - initType=$(echo $OPTARG) - ;; h) echo "Usage: $(basename $0) -v [server | client] -e [yes | no]" exit 0 @@ -218,6 +216,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${demoName2} || : ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : + ${csudo}rm -f ${bin_link_dir}/${keeperName2} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -231,6 +230,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${demoName2} || : [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : + [ -x ${install_main_dir}/bin/${keeperName2} ] && ${csudo}ln -sf ${install_main_dir}/bin/${keeperName2} ${bin_link_dir}/${keeperName2} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : if [ "$clientName2" == "${clientName}" ]; then [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : @@ -373,42 +373,56 @@ function add_newHostname_to_hosts() { return fi done - ${csudo}echo "127.0.0.1 $1" >>/etc/hosts || : + + if grep -q "127.0.0.1 $1" /etc/hosts; then + return + else + ${csudo}chmod 666 /etc/hosts + ${csudo}echo "127.0.0.1 $1" >>/etc/hosts + fi } function set_hostname() { - echo -e -n "${GREEN}Please enter one hostname(must not be 'localhost')${NC}:" - read newHostname + echo -e -n "${GREEN}Host name or IP (assigned to this machine) which can be accessed by your tools or apps (must not be 'localhost')${NC}" + read -e -p " : " -i "$(hostname)" newHostname while true; do - if [[ ! -z "$newHostname" && "$newHostname" != "localhost" ]]; then + if [ -z "$newHostname" ]; then + newHostname=$(hostname) + break + elif [ "$newHostname" != "localhost" ]; then break else - read -p "Please enter one hostname(must not be 'localhost'):" newHostname + echo -e -n "${GREEN}Host name or IP (assigned to this machine) which can be accessed by your tools or apps (must not be 'localhost')${NC}" + read -e -p " : " -i "$(hostname)" newHostname fi done - ${csudo}hostname $newHostname || : - retval=$(echo $?) - if [[ $retval != 0 ]]; then - echo - echo "set hostname fail!" - return - fi + # ${csudo}hostname $newHostname || : + # retval=$(echo $?) + # if [[ $retval != 0 ]]; then + # echo + # echo "set hostname fail!" + # return + # fi - #ubuntu/centos /etc/hostname - if [[ -e /etc/hostname ]]; then - ${csudo}echo $newHostname >/etc/hostname || : - fi + # #ubuntu/centos /etc/hostname + # if [[ -e /etc/hostname ]]; then + # ${csudo}echo $newHostname >/etc/hostname || : + # fi - #debian: #HOSTNAME=yourname - if [[ -e /etc/sysconfig/network ]]; then - ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : - fi + # #debian: #HOSTNAME=yourname + # if [[ -e /etc/sysconfig/network ]]; then + # ${csudo}sed -i -r "s/#*\s*(HOSTNAME=\s*).*/\1$newHostname/" /etc/sysconfig/network || : + # fi - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile2} + if [ -f ${cfg_install_dir}/${configFile2} ]; then + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${cfg_install_dir}/${configFile2} + else + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$newHostname/" ${script_dir}/cfg/${configFile2} + fi serverFqdn=$newHostname - if [[ -e /etc/hosts ]]; then + if [[ -e /etc/hosts ]] && [[ ! $newHostname =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then add_newHostname_to_hosts $newHostname fi } @@ -439,7 +453,12 @@ function set_ipAsFqdn() { echo -e -n "${GREEN}Unable to get local ip, use 127.0.0.1${NC}" localFqdn="127.0.0.1" # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} + + if [ -f ${cfg_install_dir}/${configFile2} ]; then + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} + else + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${script_dir}/cfg/${configFile2} + fi serverFqdn=$localFqdn echo return @@ -460,8 +479,12 @@ function set_ipAsFqdn() { if [[ $retval != 0 ]]; then read -p "Please choose an IP from local IP list:" localFqdn else - # Write the local FQDN to configuration file - ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} + # Write the local FQDN to configuration file + if [ -f ${cfg_install_dir}/${configFile2} ]; then + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${cfg_install_dir}/${configFile2} + else + ${csudo}sed -i -r "s/#*\s*(fqdn\s*).*/\1$localFqdn/" ${script_dir}/cfg/${configFile2} + fi serverFqdn=$localFqdn break fi @@ -476,37 +499,13 @@ function local_fqdn_check() { echo echo -e -n "System hostname is: ${GREEN}$serverFqdn${NC}" echo - if [[ "$serverFqdn" == "" ]] || [[ "$serverFqdn" == "localhost" ]]; then - echo -e -n "${GREEN}It is strongly recommended to configure a hostname for this machine ${NC}" - echo - - while true; do - read -r -p "Set hostname now? [Y/n] " input - if [ ! -n "$input" ]; then - set_hostname - break - else - case $input in - [yY][eE][sS] | [yY]) - set_hostname - break - ;; - - [nN][oO] | [nN]) - set_ipAsFqdn - break - ;; - - *) - echo "Invalid input..." - ;; - esac - fi - done - fi + set_hostname } function install_adapter_config() { + if [ -f ${script_dir}/cfg/${adapterName}.toml ]; then + ${csudo}sed -i -r "s/localhost/${serverFqdn}/g" ${script_dir}/cfg/${adapterName}.toml + fi if [ ! -f "${cfg_install_dir}/${adapterName}.toml" ]; then ${csudo}mkdir -p ${cfg_install_dir} [ -f ${script_dir}/cfg/${adapterName}.toml ] && ${csudo}cp ${script_dir}/cfg/${adapterName}.toml ${cfg_install_dir} @@ -523,13 +522,38 @@ function install_adapter_config() { } -function install_config() { +function install_keeper_config() { + if [ -f ${script_dir}/cfg/${keeperName2}.toml ]; then + ${csudo}sed -i -r "s/127.0.0.1/${serverFqdn}/g" ${script_dir}/cfg/${keeperName2}.toml + fi + if [ -f "${configDir}/keeper.toml" ]; then + echo "The file keeper.toml will be renamed to ${keeperName2}.toml" + ${csudo}cp ${script_dir}/cfg/${keeperName2}.toml ${configDir}/${keeperName2}.toml.new + ${csudo}mv ${configDir}/keeper.toml ${configDir}/${keeperName2}.toml + elif [ -f "${configDir}/${keeperName2}.toml" ]; then + # "taoskeeper.toml exists,new config is taoskeeper.toml.new" + ${csudo}cp ${script_dir}/cfg/${keeperName2}.toml ${configDir}/${keeperName2}.toml.new + else + ${csudo}cp ${script_dir}/cfg/${keeperName2}.toml ${configDir}/${keeperName2}.toml + fi + command -v systemctl >/dev/null 2>&1 && ${csudo}systemctl daemon-reload >/dev/null 2>&1 || true +} + +function install_config() { if [ ! -f "${cfg_install_dir}/${configFile2}" ]; then ${csudo}mkdir -p ${cfg_install_dir} - [ -f ${script_dir}/cfg/${configFile2} ] && ${csudo}cp ${script_dir}/cfg/${configFile2} ${cfg_install_dir} + if [ -f ${script_dir}/cfg/${configFile2} ]; then + ${csudo} echo "monitor 1" >> ${script_dir}/cfg/${configFile2} + ${csudo} echo "monitorFQDN ${serverFqdn}" >> ${script_dir}/cfg/${configFile2} + ${csudo} echo "audit 1" >> ${script_dir}/cfg/${configFile2} + ${csudo}cp ${script_dir}/cfg/${configFile2} ${cfg_install_dir} + fi ${csudo}chmod 644 ${cfg_install_dir}/* else + ${csudo} echo "monitor 1" >> ${script_dir}/cfg/${configFile2} + ${csudo} echo "monitorFQDN ${serverFqdn}" >> ${script_dir}/cfg/${configFile2} + ${csudo} echo "audit 1" >> ${script_dir}/cfg/${configFile2} ${csudo}cp -f ${script_dir}/cfg/${configFile2} ${cfg_install_dir}/${configFile2}.new fi @@ -537,6 +561,8 @@ function install_config() { [ ! -z $1 ] && return 0 || : # only install client + + if ((${update_flag} == 1)); then return 0 fi @@ -554,7 +580,11 @@ function install_config() { read firstEp while true; do if [ ! -z "$firstEp" ]; then - ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile2} + if [ -f ${cfg_install_dir}/${configFile2} ]; then + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${cfg_install_dir}/${configFile2} + else + ${csudo}sed -i -r "s/#*\s*(firstEp\s*).*/\1$firstEp/" ${script_dir}/cfg/${configFile2} + fi break else break @@ -606,7 +636,10 @@ function install_data() { function install_connector() { if [ -d "${script_dir}/connector/" ]; then - ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector" + ${csudo}cp -rf ${script_dir}/connector/ ${install_main_dir}/ || echo "failed to copy connector" + ${csudo}cp ${script_dir}/start-all.sh ${install_main_dir}/ || echo "failed to copy start-all.sh" + ${csudo}cp ${script_dir}/stop-all.sh ${install_main_dir}/ || echo "failed to copy stop-all.sh" + ${csudo}cp ${script_dir}/README.md ${install_main_dir}/ || echo "failed to copy README.md" fi } @@ -622,6 +655,14 @@ function install_web() { fi } +function install_taosx() { + if [ -f "${script_dir}/taosx/install_taosx.sh" ]; then + cd ${script_dir}/taosx + chmod a+x install_taosx.sh + bash install_taosx.sh -e $serverFqdn + fi +} + function clean_service_on_sysvinit() { if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then ${csudo}service ${serverName2} stop || : @@ -701,30 +742,7 @@ function clean_service_on_systemd() { ${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${tarbitratord_service_config} - - if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - x_service_config="${service_config_dir}/${xName2}.service" - if [ -e "$x_service_config" ]; then - if systemctl is-active --quiet ${xName2}; then - echo "${productName2} ${xName2} is running, stopping it..." - ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${x_service_config} - fi - - explorer_service_config="${service_config_dir}/${explorerName2}.service" - if [ -e "$explorer_service_config" ]; then - if systemctl is-active --quiet ${explorerName2}; then - echo "${productName2} ${explorerName2} is running, stopping it..." - ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${explorer_service_config} - ${csudo}rm -f /etc/${clientName2}/explorer.toml - fi - fi + ${csudo}rm -f ${tarbitratord_service_config} } function install_service_on_systemd() { @@ -745,15 +763,27 @@ function install_service_on_systemd() { ${csudo}systemctl daemon-reload ${csudo}systemctl enable ${serverName2} - ${csudo}systemctl daemon-reload } function install_adapter_service() { if ((${service_mod} == 0)); then - [ -f ${script_dir}/cfg/${adapterName}.service ] && - ${csudo}cp ${script_dir}/cfg/${adapterName}.service \ + [ -f ${script_dir}/cfg/${adapterName2}.service ] && + ${csudo}cp ${script_dir}/cfg/${adapterName2}.service \ ${service_config_dir}/ || : + + ${csudo}systemctl enable ${adapterName2} + ${csudo}systemctl daemon-reload + fi +} + +function install_keeper_service() { + if ((${service_mod} == 0)); then + [ -f ${script_dir}/cfg/${clientName2}keeper.service ] && + ${csudo}cp ${script_dir}/cfg/${clientName2}keeper.service \ + ${service_config_dir}/ || : + + ${csudo}systemctl enable ${clientName2}keeper ${csudo}systemctl daemon-reload fi } @@ -872,7 +902,7 @@ function updateProduct() { tar -zxf ${tarName} install_jemalloc - echo -e "${GREEN}Start to update ${productName2}...${NC}" + echo "Start to update ${productName2}..." # Stop the service if running if ps aux | grep -v grep | grep ${serverName2} &>/dev/null; then if ((${service_mod} == 0)); then @@ -890,9 +920,11 @@ function updateProduct() { install_log install_header install_lib + install_config if [ "$verMode" == "cluster" ]; then install_connector + install_taosx fi install_examples @@ -900,54 +932,71 @@ function updateProduct() { if [ -z $1 ]; then install_bin install_service - install_adapter_service - install_config + install_adapter_service install_adapter_config + install_keeper_service + install_keeper_config openresty_work=false echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t\t: edit ${cfg_install_dir}/${configFile2}" [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml" + if [ "$verMode" == "cluster" ]; then + echo -e "${GREEN_DARK}To configure ${clientName2}-explorer ${NC}\t: edit ${configDir}/explorer.toml" + fi if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t\t: ${csudo}systemctl start ${serverName2}${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}service ${serverName2} start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t\t: ${csudo}service ${serverName2} start${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t\t: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ./${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t\t: ./${serverName2}${NC}" [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t\t: ${clientName2}adapter ${NC}" fi - echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}" - - if [ ${openresty_work} = 'true' ]; then - echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" - else - echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}" + echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t\t: sudo systemctl enable ${clientName2}keeper ${NC}" + if [ "$verMode" == "cluster" ];then + echo -e "${GREEN_DARK}To start ${clientName2}x ${NC}\t\t\t: sudo systemctl start ${clientName2}x ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}-explorer ${NC}\t\t: sudo systemctl start ${clientName2}-explorer ${NC}" fi - if ((${prompt_force} == 1)); then - echo "" - echo -e "${RED}Please run '${serverName2} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" - fi + # if [ ${openresty_work} = 'true' ]; then + # echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell OR from ${GREEN_UNDERLINE}http://127.0.0.1:${web_port}${NC}" + # else + # echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t\t: use ${GREEN_UNDERLINE}${clientName2} -h $serverFqdn${NC} in shell${NC}" + # fi + + # if ((${prompt_force} == 1)); then + # echo "" + # echo -e "${RED}Please run '${serverName2} --force-keep-file' at first time for the exist ${productName2} $exist_version!${NC}" + # fi + echo - echo -e "\033[44;32;1m${productName2} is updated successfully!${NC}" - echo -e "\033[44;32;1mTo manage ${productName2} instance, view documentation or explorer features, please install ${clientName2}Explorer ${NC}" + echo "${productName2} is updated successfully!" + echo + if [ "$verMode" == "cluster" ];then + echo -e "\033[44;32;1mTo start all the components : ./start-all.sh${NC}" + fi + echo -e "\033[44;32;1mTo access ${productName2} : ${clientName2} -h $serverFqdn${NC}" + if [ "$verMode" == "cluster" ];then + echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}" + echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs${NC}" + fi else - install_bin - install_config + install_bin echo echo -e "\033[44;32;1m${productName2} client is updated successfully!${NC}" fi - rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") + cd $script_dir + rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } function installProduct() { @@ -958,7 +1007,7 @@ function installProduct() { fi tar -zxf ${tarName} - echo -e "${GREEN}Start to install ${productName2}...${NC}" + echo "Start to install ${productName2}..." install_main_path @@ -972,9 +1021,11 @@ function installProduct() { install_jemalloc #install_avro lib #install_avro lib64 + install_config if [ "$verMode" == "cluster" ]; then install_connector + install_taosx fi install_examples install_web @@ -984,62 +1035,80 @@ function installProduct() { install_service install_adapter_service install_adapter_config + install_keeper_service + install_keeper_config openresty_work=false - install_config # Ask if to start the service echo - echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t: edit ${cfg_install_dir}/${configFile2}" + echo -e "${GREEN_DARK}To configure ${productName2} ${NC}\t\t: edit ${cfg_install_dir}/${configFile2}" [ -f ${configDir}/${clientName2}adapter.toml ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ echo -e "${GREEN_DARK}To configure ${clientName2}Adapter ${NC}\t: edit ${configDir}/${clientName2}adapter.toml" + if [ "$verMode" == "cluster" ]; then + echo -e "${GREEN_DARK}To configure ${clientName2}-explorer ${NC}\t: edit ${configDir}/explorer.toml" + fi if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}systemctl start ${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t\t: ${csudo}systemctl start ${serverName2}${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}systemctl start ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t\t: ${csudo}systemctl start ${clientName2}adapter ${NC}" elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${csudo}service ${serverName2} start${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t\t: ${csudo}service ${serverName2} start${NC}" [ -f ${service_config_dir}/${clientName2}adapter.service ] && [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${csudo}service ${clientName2}adapter start${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t\t: ${csudo}service ${clientName2}adapter start${NC}" else - echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t: ${serverName2}${NC}" + echo -e "${GREEN_DARK}To start ${productName2} ${NC}\t\t: ${serverName2}${NC}" [ -f ${installDir}/bin/${clientName2}adapter ] && \ - echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t: ${clientName2}adapter ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}Adapter ${NC}\t\t: ${clientName2}adapter ${NC}" fi - echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t: sudo systemctl enable ${clientName2}keeper ${NC}" - - if [ ! -z "$firstEp" ]; then - tmpFqdn=${firstEp%%:*} - substr=":" - if [[ $firstEp =~ $substr ]]; then - tmpPort=${firstEp#*:} - else - tmpPort="" - fi - if [[ "$tmpPort" != "" ]]; then - echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" - else - echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" - fi - echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" - echo - elif [ ! -z "$serverFqdn" ]; then - echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}" - echo + echo -e "${GREEN_DARK}To enable ${clientName2}keeper ${NC}\t\t: sudo systemctl enable ${clientName2}keeper ${NC}" + + if [ "$verMode" == "cluster" ];then + echo -e "${GREEN_DARK}To start ${clientName2}x ${NC}\t\t\t: sudo systemctl start ${clientName2}x ${NC}" + echo -e "${GREEN_DARK}To start ${clientName2}-explorer ${NC}\t\t: sudo systemctl start ${clientName2}-explorer ${NC}" fi - echo -e "\033[44;32;1m${productName2} is installed successfully!${NC}" - echo -e "\033[44;32;1mTo manage ${productName2} instance, view documentation or explorer features, please install ${clientName2}Explorer ${NC}" + # if [ ! -z "$firstEp" ]; then + # tmpFqdn=${firstEp%%:*} + # substr=":" + # if [[ $firstEp =~ $substr ]]; then + # tmpPort=${firstEp#*:} + # else + # tmpPort="" + # fi + # if [[ "$tmpPort" != "" ]]; then + # echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t\t: ${clientName2} -h $tmpFqdn -P $tmpPort${GREEN_DARK} to login into cluster, then${NC}" + # else + # echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t\t: ${clientName2} -h $tmpFqdn${GREEN_DARK} to login into cluster, then${NC}" + # fi + # echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + # echo + # elif [ ! -z "$serverFqdn" ]; then + # echo -e "${GREEN_DARK}To access ${productName2} ${NC}\t\t: ${clientName2} -h $serverFqdn${GREEN_DARK} to login into ${productName2} server${NC}" + # echo + # fi + echo + echo "${productName2} is installed successfully!" + echo + if [ "$verMode" == "cluster" ];then + echo -e "\033[44;32;1mTo start all the components : sudo ./start-all.sh${NC}" + fi + echo -e "\033[44;32;1mTo access ${productName2} : ${clientName2} -h $serverFqdn${NC}" + if [ "$verMode" == "cluster" ];then + echo -e "\033[44;32;1mTo access the management system : http://$serverFqdn:6060${NC}" + echo -e "\033[44;32;1mTo read the user manual : http://$serverFqdn:6060/docs-en${NC}" + fi echo else # Only install client install_bin - install_config + echo echo -e "\033[44;32;1m${productName2} client is installed successfully!${NC}" fi - + + cd $script_dir touch ~/.${historyFile} rm -rf $(tar -tf ${tarName} | grep -Ev "^\./$|^\/") } @@ -1071,3 +1140,5 @@ elif [ "$verType" == "client" ]; then else echo "please input correct verType" fi + + diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index c8baab8269..6643363339 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -129,6 +129,7 @@ function install_bin() { if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/${demoName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${demoName2} ${bin_link_dir}/${demoName2} || : [ -x ${install_main_dir}/bin/${benchmarkName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName2} ${bin_link_dir}/${benchmarkName2} || : + [ -x ${install_main_dir}/bin/${dumpName2} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName2} ${bin_link_dir}/${dumpName2} || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo}ln -sf ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/${uninstallScript2} || : fi diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 243efd693e..f46a9adeff 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -24,10 +24,12 @@ clientName2="${12}" productName="TDengine" clientName="taos" benchmarkName="taosBenchmark" +dumpName="taosdump" configFile="taos.cfg" tarName="package.tar.gz" benchmarkName2="${clientName2}Benchmark" +dumpName2="${clientName2}dump" if [ "$osType" != "Darwin" ]; then script_dir="$(dirname $(readlink -f $0))" @@ -71,6 +73,7 @@ if [ "$osType" != "Darwin" ]; then else bin_files="${build_dir}/bin/${clientName} \ ${build_dir}/bin/${benchmarkName} \ + ${build_dir}/bin/${dumpName} \ ${script_dir}/remove_client.sh \ ${script_dir}/set_core.sh \ ${script_dir}/get_client.sh" diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 42465b8783..4b0faaa958 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -42,7 +42,7 @@ release_dir="${top_dir}/release" #package_name='linux' if [ "$verMode" == "cluster" ]; then - install_dir="${release_dir}/${productName2}-enterprise-server-${version}" + install_dir="${release_dir}/${productName2}-enterprise-${version}" elif [ "$verMode" == "cloud" ]; then install_dir="${release_dir}/${productName2}-cloud-server-${version}" else @@ -92,14 +92,10 @@ else ${build_dir}/bin/tdengine-datasource.zip.md5" fi - [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" - explorer_bin_files=$(find ${build_dir}/bin/ -name '*-explorer') bin_files="${build_dir}/bin/${serverName} \ ${build_dir}/bin/${clientName} \ ${taostools_bin_files} \ - ${taosx_bin} \ - ${explorer_bin_files} \ ${build_dir}/bin/${clientName}adapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ @@ -284,8 +280,13 @@ if [ "$pagMode" == "lite" ]; then fi chmod a+x ${install_dir}/install.sh -if [[ $dbName == "taos" ]]; then - # Copy example code +if [[ $dbName == "taos" ]]; then + cp ${top_dir}/../enterprise/packaging/start-all.sh ${install_dir} + cp ${top_dir}/../enterprise/packaging/stop-all.sh ${install_dir} + cp ${top_dir}/../enterprise/packaging/README.md ${install_dir} + chmod a+x ${install_dir}/start-all.sh + chmod a+x ${install_dir}/stop-all.sh + # Copy example code mkdir -p ${install_dir}/examples examples_dir="${top_dir}/examples" cp -r ${examples_dir}/c ${install_dir}/examples @@ -330,8 +331,8 @@ fi mkdir -p ${install_dir}/driver && cp ${lib_files} ${install_dir}/driver && echo "${versionComp}" >${install_dir}/driver/vercomp.txt [ -f ${wslib_files} ] && cp ${wslib_files} ${install_dir}/driver || : -# Copy connector -if [ "$verMode" == "cluster" ]; then +# Copy connector && taosx +if [ "$verMode" == "cluster" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then @@ -364,8 +365,13 @@ if [ "$verMode" == "cluster" ]; then git clone --depth 1 https://github.com/taosdata/taos-connector-rust ${install_dir}/connector/rust rm -rf ${install_dir}/connector/rust/.git ||: - # cp -r ${connector_dir}/python ${install_dir}/connector - # cp -r ${connector_dir}/nodejs ${install_dir}/connector + # copy taosx + if [ -d ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ]; then + cp -r ${top_dir}/../enterprise/src/plugins/taosx/release/taosx ${install_dir} + cp ${top_dir}/../enterprise/packaging/install_taosx.sh ${install_dir}/taosx + cp ${top_dir}/../enterprise/src/plugins/taosx/packaging/uninstall.sh ${install_dir}/taosx + sed -i 's/target=\"\"/target=\"taosx\"/g' ${install_dir}/taosx/uninstall.sh + fi fi fi diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 97bffb0535..1ec83b7b0d 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -63,6 +63,10 @@ service_config_dir="/etc/systemd/system" taos_service_name=${serverName2} taosadapter_service_name="${clientName2}adapter" tarbitrator_service_name="tarbitratord" + +config_dir="/etc/${clientName2}" + + csudo="" if command -v sudo >/dev/null; then csudo="sudo " @@ -113,8 +117,10 @@ function clean_bin() { # Remove link ${csudo}rm -f ${bin_link_dir}/${clientName} || : ${csudo}rm -f ${bin_link_dir}/${serverName} || : + echo "${serverName} is removed successfully" ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/${adapterName2} || : + echo "${adapterName2} is removed successfully" ${csudo}rm -f ${bin_link_dir}/${benchmarkName2} || : ${csudo}rm -f ${bin_link_dir}/${demoName2} || : ${csudo}rm -f ${bin_link_dir}/${dumpName2} || : @@ -175,7 +181,7 @@ function clean_log() { function clean_service_on_systemd() { taosd_service_config="${service_config_dir}/${taos_service_name}.service" if systemctl is-active --quiet ${taos_service_name}; then - echo "${productName2} ${serverName2} is running, stopping it..." + echo "${taos_service_name} is running, stopping it..." ${csudo}systemctl stop ${taos_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${taos_service_name} &>/dev/null || echo &>/dev/null @@ -183,7 +189,7 @@ function clean_service_on_systemd() { taosadapter_service_config="${service_config_dir}/${clientName2}adapter.service" if systemctl is-active --quiet ${taosadapter_service_name}; then - echo "${productName2} ${clientName2}Adapter is running, stopping it..." + echo "${clientName2}Adapter is running, stopping it..." ${csudo}systemctl stop ${taosadapter_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${taosadapter_service_name} &>/dev/null || echo &>/dev/null @@ -196,33 +202,11 @@ function clean_service_on_systemd() { fi ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null - if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then - x_service_config="${service_config_dir}/${xName2}.service" - if [ -e "$x_service_config" ]; then - if systemctl is-active --quiet ${xName2}; then - echo "${productName2} ${xName2} is running, stopping it..." - ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${x_service_config} - fi - - explorer_service_config="${service_config_dir}/${explorerName2}.service" - if [ -e "$explorer_service_config" ]; then - if systemctl is-active --quiet ${explorerName2}; then - echo "${productName2} ${explorerName2} is running, stopping it..." - ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${explorer_service_config} - ${csudo}rm -f /etc/${clientName2}/explorer.toml - fi - fi } function clean_service_on_sysvinit() { if ps aux | grep -v grep | grep ${serverName} &>/dev/null; then - echo "${productName2} ${serverName2} is running, stopping it..." + echo "${serverName2} is running, stopping it..." ${csudo}service ${serverName} stop || : fi @@ -284,6 +268,97 @@ function clean_service() { fi } +function remove_data_and_config() { + data_dir=`grep dataDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}` + if [ X"$data_dir" == X"" ]; then + data_dir="/var/lib/taos" + fi + log_dir=`grep logDir /etc/taos/taos.cfg | grep -v '#' | tail -n 1 | awk {'print $2'}` + if [ X"$log_dir" == X"" ]; then + log_dir="/var/log/taos" + fi + [ -d "${config_dir}" ] && ${csudo}rm -rf ${config_dir}/* + [ -d "${data_dir}" ] && ${csudo}rm -rf ${data_dir}/* + [ -d "${log_dir}" ] && ${csudo}rm -rf ${log_dir}/* +} + +_kill_service_of() { + _service=$1 + pid=$(ps -ef | grep "$_service" | grep -v "grep" | awk '{print $2}') + if [ -n "$pid" ]; then + ${csudo}kill -9 $pid || : + fi +} + +_clean_service_on_systemd_of() { + _service=$1 + _service_config="${service_config_dir}/${_service}.service" + if systemctl is-active --quiet ${_service}; then + echo "taoskeeper is running, stopping it..." + ${csudo}systemctl stop ${_service} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${_service} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${_service_config} +} +_clean_service_on_sysvinit_of() { + _service=$1 + if pidof ${_service} &>/dev/null; then + echo "${_service} is running, stopping it..." + ${csudo}service ${_service} stop || : + fi + if ((${initd_mod} == 1)); then + if [ -e ${service_config_dir}/${_service} ]; then + ${csudo}chkconfig --del ${_service} || : + fi + elif ((${initd_mod} == 2)); then + if [ -e ${service_config_dir}/${_service} ]; then + ${csudo}insserv -r ${_service} || : + fi + elif ((${initd_mod} == 3)); then + if [ -e ${service_config_dir}/${_service} ]; then + ${csudo}update-rc.d -f ${_service} remove || : + fi + fi + + ${csudo}rm -f ${service_config_dir}/${_service} || : + + if $(which init &>/dev/null); then + ${csudo}init q || : + fi +} + +_clean_service_of() { + _service=$1 + if ((${service_mod} == 0)); then + _clean_service_on_systemd_of $_service + elif ((${service_mod} == 1)); then + _clean_service_on_sysvinit_of $_service + else + _kill_service_of $_service + fi +} + +remove_taoskeeper() { + # remove taoskeeper bin + _clean_service_of taoskeeper + [ -e "${bin_link_dir}/taoskeeper" ] && ${csudo}rm -rf ${bin_link_dir}/taoskeeper + [ -e "${installDir}/taoskeeper" ] && ${csudo}rm -rf ${installDir}/taoskeeper + [ -e "${cfg_link_dir}/metrics.toml" ] || ${csudo}rm -rf ${cfg_link_dir}/metrics.toml + echo "taosKeeper is removed successfully!" +} + +function uninstall_taosx() { + if [ -f ${installDir}/uninstall.sh ]; then + cd ${installDir} + bash uninstall.sh + fi +} + +if [ "$verMode" == "cluster" ]; then + uninstall_taosx +fi + +remove_taoskeeper # Stop service and disable booting start. clean_service # Remove binary file and links @@ -322,5 +397,13 @@ if [ "$osType" = "Darwin" ]; then ${csudo}rm -rf /Applications/TDengine.app fi -echo -e "${GREEN}${productName2} is removed successfully!${NC}" +echo +echo "Do you want to remove all the data, log and configuration files? [y/n]" +read answer +if [ X$answer == X"y" ] || [ X$answer == X"Y" ]; then + remove_data_and_config +fi + +echo +echo "${productName2} is removed successfully!" echo diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index b36ef20b53..da24bc0a3b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -776,7 +776,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { } else { tscInfo("set cfg:%s to %s", pItem->name, str); if (TSDB_OPTION_SHELL_ACTIVITY_TIMER == option || TSDB_OPTION_USE_ADAPTER == option) { - code = taosApplyLocalCfg(pCfg, pItem->name); + code = taosCfgDynamicOptions(pCfg, pItem->name, false); } } diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index 03cd19fa40..1f9d3c6d8c 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -203,7 +203,7 @@ static char* processCreateStb(SMqMetaRsp* metaRsp) { goto _err; } string = buildCreateTableJson(&req.schemaRow, &req.schemaTag, req.name, req.suid, TSDB_SUPER_TABLE); - _err: +_err: uDebug("create stable return, sql json:%s", string); tDecoderClear(&coder); return string; @@ -224,7 +224,7 @@ static char* processAlterStb(SMqMetaRsp* metaRsp) { goto _err; } string = buildAlterSTableJson(req.alterOriData, req.alterOriDataLen); - _err: +_err: uDebug("alter stable return, sql json:%s", string); tDecoderClear(&coder); return string; @@ -375,7 +375,7 @@ static char* processCreateTable(SMqMetaRsp* metaRsp) { } } - _exit: +_exit: uDebug("create table return, sql json:%s", string); for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; @@ -416,7 +416,7 @@ static char* processAutoCreateTable(STaosxRsp* rsp) { } } string = buildCreateCTableJson(pCreateReq, rsp->createTableNum); - _exit: +_exit: uDebug("auto created table return, sql json:%s", string); for (int i = 0; i < rsp->createTableNum; i++) { tDecoderClear(&decoder[i]); @@ -549,7 +549,7 @@ static char* processAlterTable(SMqMetaRsp* metaRsp) { } string = cJSON_PrintUnformatted(json); - _exit: +_exit: uDebug("alter table return, sql json:%s", string); cJSON_Delete(json); tDecoderClear(&decoder); @@ -585,7 +585,7 @@ static char* processDropSTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "tableName", tableName); string = cJSON_PrintUnformatted(json); - _exit: +_exit: uDebug("processDropSTable return, sql json:%s", string); cJSON_Delete(json); tDecoderClear(&decoder); @@ -624,7 +624,7 @@ static char* processDeleteTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "sql", sqlJson); string = cJSON_PrintUnformatted(json); - _exit: +_exit: uDebug("processDeleteTable return, sql json:%s", string); cJSON_Delete(json); tDecoderClear(&coder); @@ -669,7 +669,7 @@ static char* processDropTable(SMqMetaRsp* metaRsp) { cJSON_AddItemToObject(json, "tableNameList", tableNameList); string = cJSON_PrintUnformatted(json); - _exit: +_exit: uDebug("processDropTable return, json sql:%s", string); cJSON_Delete(json); tDecoderClear(&decoder); @@ -765,7 +765,7 @@ static int32_t taosCreateStb(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: uDebug(LOG_ID_TAG" create stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); destroyRequest(pRequest); tFreeSMCreateStbReq(&pReq); @@ -869,7 +869,7 @@ static int32_t taosDropStb(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; taosMemoryFree(pCmdMsg.pMsg); - end: +end: uDebug(LOG_ID_TAG" drop stable return, msg:%s", LOG_ID_VALUE, tstrerror(code)); destroyRequest(pRequest); tDecoderClear(&coder); @@ -1023,7 +1023,7 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG" create table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; @@ -1175,7 +1175,7 @@ static int32_t taosDropTable(TAOS* taos, void* meta, int32_t metaLen) { } code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG" drop table return, msg:%s", LOG_ID_VALUE, tstrerror(code)); taosHashCleanup(pVgroupHashmap); destroyRequest(pRequest); @@ -1250,7 +1250,7 @@ static int32_t taosDeleteData(TAOS* taos, void* meta, int32_t metaLen) { } taos_free_result(res); - end: +end: uDebug("connId:0x%"PRIx64" delete data sql:%s, code:%s", *(int64_t*)taos, sql, tstrerror(code)); tDecoderClear(&coder); terrno = code; @@ -1368,7 +1368,7 @@ static int32_t taosAlterTable(TAOS* taos, void* meta, int32_t metaLen) { code = handleAlterTbExecRes(pRes->res, pCatalog); } } - end: +end: uDebug(LOG_ID_TAG " alter table return, meta:%p, len:%d, msg:%s", LOG_ID_VALUE, meta, metaLen, tstrerror(code)); taosArrayDestroy(pArray); if (pVgData) taosMemoryFreeClear(pVgData->pData); @@ -1459,7 +1459,7 @@ int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pDat launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " write raw block with field return, msg:%s", LOG_ID_VALUE, tstrerror(code)); taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); @@ -1543,7 +1543,7 @@ int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const cha launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " write raw block return, msg:%s", LOG_ID_VALUE, tstrerror(code)); taosMemoryFreeClear(pTableMeta); qDestroyQuery(pQuery); @@ -1669,7 +1669,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " write raw data return, msg:%s", LOG_ID_VALUE, tstrerror(code)); tDeleteMqDataRsp(&rspObj.rsp); tDecoderClear(&decoder); @@ -1841,7 +1841,7 @@ static int32_t tmqWriteRawMetaDataImpl(TAOS* taos, void* data, int32_t dataLen) launchQueryImpl(pRequest, pQuery, true, NULL); code = pRequest->code; - end: +end: uDebug(LOG_ID_TAG " write raw metadata return, msg:%s", LOG_ID_VALUE, tstrerror(code)); tDeleteSTaosxRsp(&rspObj.rsp); tDecoderClear(&decoder); @@ -1984,7 +1984,7 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { return tmqWriteRawMetaDataImpl(taos, raw.raw, raw.raw_len); } - end: +end: terrno = TSDB_CODE_INVALID_PARA; return terrno; } diff --git a/source/common/src/cos.c b/source/common/src/cos.c index ea41afd8fb..f36ab42ada 100644 --- a/source/common/src/cos.c +++ b/source/common/src/cos.c @@ -497,12 +497,6 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, 0, &putObjectHandler, &data); } while (S3_status_is_retryable(data.status) && should_retry()); - if (data.infileFD) { - taosCloseFile(&data.infileFD); - } else if (data.gb) { - growbuffer_destroy(data.gb); - } - if (data.status != S3StatusOK) { s3PrintError(__func__, data.status, data.err_msg); code = TAOS_SYSTEM_ERROR(EIO); @@ -519,9 +513,14 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { manager.gb = 0; // div round up - int seq; - uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 8; - int totalSeq = ((contentLength + chunk_size - 1) / chunk_size); + int seq; + uint64_t chunk_size = MULTIPART_CHUNK_SIZE >> 7; + int totalSeq = (contentLength + chunk_size - 1) / chunk_size; + const int max_part_num = 1000; + if (totalSeq > max_part_num) { + chunk_size = (contentLength + max_part_num - contentLength % max_part_num) / max_part_num; + totalSeq = (contentLength + chunk_size - 1) / chunk_size; + } MultipartPartData partData; memset(&partData, 0, sizeof(MultipartPartData)); @@ -622,6 +621,12 @@ int32_t s3PutObjectFromFile2(const char *file, const char *object) { for (i = 0; i < manager.next_etags_pos; i++) { taosMemoryFree(manager.etags[i]); } + if (data.infileFD) { + taosCloseFile(&data.infileFD); + } else if (data.gb) { + growbuffer_destroy(data.gb); + } + growbuffer_destroy(manager.gb); taosMemoryFree(manager.etags); } @@ -719,6 +724,8 @@ static SArray *getListByPrefix(const char *prefix) { } else { s3PrintError(__func__, data.status, data.err_msg); } + + taosArrayDestroyEx(data.objectArray, s3FreeObjectKey); return NULL; } diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 47988b0de9..40d1f92d5d 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -21,6 +21,7 @@ #include "tgrant.h" #include "tlog.h" #include "tmisce.h" +#include "tunit.h" #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" @@ -94,8 +95,8 @@ int32_t tsMonitorMaxLogs = 100; bool tsMonitorComp = false; // audit -bool tsEnableAudit = true; -bool tsEnableAuditCreateTable = true; +bool tsEnableAudit = true; +bool tsEnableAuditCreateTable = true; // telem #ifdef TD_ENTERPRISE @@ -507,8 +508,8 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = tsNumOfCores / 2; tsNumOfTaskQueueThreads = TMAX(tsNumOfTaskQueueThreads, 4); - if (tsNumOfTaskQueueThreads >= 10) { - tsNumOfTaskQueueThreads = 10; + if (tsNumOfTaskQueueThreads >= 50) { + tsNumOfTaskQueueThreads = 50; } if (cfgAddInt32(pCfg, "numOfTaskQueueThreads", tsNumOfTaskQueueThreads, 4, 1024, CFG_SCOPE_CLIENT, CFG_DYN_NONE) != 0) return -1; @@ -722,7 +723,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddBool(pCfg, "disableStream", tsDisableStream, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddInt64(pCfg, "streamBufferSize", tsStreamBufferSize, 0, INT64_MAX, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; - if (cfgAddInt64(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, + if (cfgAddInt32(pCfg, "checkpointInterval", tsStreamCheckpointInterval, 60, 1200, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; if (cfgAddFloat(pCfg, "streamSinkDataRate", tsSinkDataRate, 0.1, 5, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; @@ -753,6 +754,10 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddString(pCfg, "s3BucketName", tsS3BucketName, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0) return -1; if (cfgAddInt32(pCfg, "s3BlockSize", tsS3BlockSize, -1, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; + if (tsS3BlockSize > -1 && tsS3BlockSize < 1024) { + uError("failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", tsS3BlockSize); + return -1; + } if (cfgAddInt32(pCfg, "s3BlockCacheSize", tsS3BlockCacheSize, 4, 1024 * 1024, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1; @@ -1196,302 +1201,6 @@ static int32_t taosSetReleaseCfg(SConfig *pCfg) { return 0; } int32_t taosSetReleaseCfg(SConfig *pCfg); #endif -int32_t taosApplyLocalCfg(SConfig *pCfg, char *name) { - int32_t len = strlen(name); - char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; - strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); - bool matchItem = true; - - switch (lowcaseName[0]) { - case 'a': { - if (strcasecmp("asyncLog", name) == 0) { - tsAsyncLog = cfgGetItem(pCfg, "asyncLog")->bval; - } else if (strcasecmp("assert", name) == 0) { - tsAssert = cfgGetItem(pCfg, "assert")->bval; - } else { - matchItem = false; - } - break; - } - case 'c': { - if (strcasecmp("compressMsgSize", name) == 0) { - tsCompressMsgSize = cfgGetItem(pCfg, "compressMsgSize")->i32; - } else if (strcasecmp("countAlwaysReturnValue", name) == 0) { - tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32; - } else if (strcasecmp("cDebugFlag", name) == 0) { - cDebugFlag = cfgGetItem(pCfg, "cDebugFlag")->i32; - } else if (strcasecmp("crashReporting", name) == 0) { - tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; - } else { - matchItem = false; - } - break; - } - case 'd': { - if (strcasecmp("dDebugFlag", name) == 0) { - dDebugFlag = cfgGetItem(pCfg, "dDebugFlag")->i32; - } else if (strcasecmp("debugFlag", name) == 0) { - int32_t flag = cfgGetItem(pCfg, "debugFlag")->i32; - taosSetAllDebugFlag(flag, true); - } else { - matchItem = false; - } - break; - } - case 'e': { - if (strcasecmp("enableCoreFile", name) == 0) { - bool enableCore = cfgGetItem(pCfg, "enableCoreFile")->bval; - taosSetCoreDump(enableCore); - } else if (strcasecmp("enableQueryHb", name) == 0) { - tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; - } else { - matchItem = false; - } - break; - } - case 'f': { - if (strcasecmp("fqdn", name) == 0) { - tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); - tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - - char defaultFirstEp[TSDB_EP_LEN] = {0}; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - - SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); - SEp firstEp = {0}; - taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); - cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); - } else if (strcasecmp("firstEp", name) == 0) { - tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); - tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - - char defaultFirstEp[TSDB_EP_LEN] = {0}; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - - SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); - SEp firstEp = {0}; - taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); - cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); - } else if (strcasecmp("fsDebugFlag", name) == 0) { - fsDebugFlag = cfgGetItem(pCfg, "fsDebugFlag")->i32; - } else { - matchItem = false; - } - break; - } - case 'i': { - if (strcasecmp("idxDebugFlag", name) == 0) { - idxDebugFlag = cfgGetItem(pCfg, "idxDebugFlag")->i32; - } else { - matchItem = false; - } - break; - } - case 'j': { - if (strcasecmp("jniDebugFlag", name) == 0) { - jniDebugFlag = cfgGetItem(pCfg, "jniDebugFlag")->i32; - } else { - matchItem = false; - } - break; - } - case 'k': { - if (strcasecmp("keepColumnName", name) == 0) { - tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; - } else if (strcasecmp("keepAliveIdle", name) == 0) { - tsKeepAliveIdle = cfgGetItem(pCfg, "keepAliveIdle")->i32; - } else { - matchItem = false; - } - break; - } - case 'l': { - if (strcasecmp("locale", name) == 0) { - const char *locale = cfgGetItem(pCfg, "locale")->str; - const char *charset = cfgGetItem(pCfg, "charset")->str; - taosSetSystemLocale(locale, charset); - osSetSystemLocale(locale, charset); - } else if (strcasecmp("logDir", name) == 0) { - tstrncpy(tsLogDir, cfgGetItem(pCfg, "logDir")->str, PATH_MAX); - taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); - } else if (strcasecmp("logKeepDays", name) == 0) { - tsLogKeepDays = cfgGetItem(pCfg, "logKeepDays")->i32; - } else { - matchItem = false; - } - break; - } - case 'm': { - switch (lowcaseName[1]) { - case 'a': { - if (strcasecmp("maxInsertBatchRows", name) == 0) { - tsMaxInsertBatchRows = cfgGetItem(pCfg, "maxInsertBatchRows")->i32; - } else if (strcasecmp("maxRetryWaitTime", name) == 0) { - tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; - } else { - matchItem = false; - } - break; - } - case 'e': { - if (strcasecmp("metaCacheMaxSize", name) == 0) { - atomic_store_32(&tsMetaCacheMaxSize, cfgGetItem(pCfg, "metaCacheMaxSize")->i32); - } else { - matchItem = false; - } - break; - } - case 'i': { - if (strcasecmp("minimalTmpDirGB", name) == 0) { - tsTempSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalTmpDirGB")->fval) * 1024 * 1024 * 1024); - } else if (strcasecmp("minimalDataDirGB", name) == 0) { - tsDataSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalDataDirGB")->fval) * 1024 * 1024 * 1024); - } else if (strcasecmp("minSlidingTime", name) == 0) { - tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32; - } else if (strcasecmp("minIntervalTime", name) == 0) { - tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32; - } else if (strcasecmp("minimalLogDirGB", name) == 0) { - tsLogSpace.reserved = (int64_t)(((double)cfgGetItem(pCfg, "minimalLogDirGB")->fval) * 1024 * 1024 * 1024); - } else { - matchItem = false; - } - break; - } - default: - terrno = TSDB_CODE_CFG_NOT_FOUND; - return -1; - } - break; - } - case 'n': { - if (strcasecmp("numOfLogLines", name) == 0) { - tsNumOfLogLines = cfgGetItem(pCfg, "numOfLogLines")->i32; - } else { - matchItem = false; - } - break; - } - case 'q': { - if (strcasecmp("querySmaOptimize", name) == 0) { - tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; - } else if (strcasecmp("queryPolicy", name) == 0) { - tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; - } else if (strcasecmp("qDebugFlag", name) == 0) { - qDebugFlag = cfgGetItem(pCfg, "qDebugFlag")->i32; - } else if (strcasecmp("queryPlannerTrace", name) == 0) { - tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; - } else if (strcasecmp("queryNodeChunkSize", name) == 0) { - tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; - } else if (strcasecmp("queryUseNodeAllocator", name) == 0) { - tsQueryUseNodeAllocator = cfgGetItem(pCfg, "queryUseNodeAllocator")->bval; - } else { - matchItem = false; - } - break; - } - case 'r': { - if (strcasecmp("rpcDebugFlag", name) == 0) { - rpcDebugFlag = cfgGetItem(pCfg, "rpcDebugFlag")->i32; - } else { - matchItem = false; - } - break; - } - case 's': { - if (strcasecmp("secondEp", name) == 0) { - SConfigItem *pSecondpItem = cfgGetItem(pCfg, "secondEp"); - SEp secondEp = {0}; - taosGetFqdnPortFromEp(strlen(pSecondpItem->str) == 0 ? tsFirst : pSecondpItem->str, &secondEp); - snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); - cfgSetItem(pCfg, "secondEp", tsSecond, pSecondpItem->stype); - } else if (strcasecmp("smlChildTableName", name) == 0) { - tstrncpy(tsSmlChildTableName, cfgGetItem(pCfg, "smlChildTableName")->str, TSDB_TABLE_NAME_LEN); - } else if (strcasecmp("smlAutoChildTableNameDelimiter", name) == 0) { - tstrncpy(tsSmlAutoChildTableNameDelimiter, cfgGetItem(pCfg, "smlAutoChildTableNameDelimiter")->str, - TSDB_TABLE_NAME_LEN); - } else if (strcasecmp("smlTagName", name) == 0) { - tstrncpy(tsSmlTagName, cfgGetItem(pCfg, "smlTagName")->str, TSDB_COL_NAME_LEN); - // } else if (strcasecmp("smlDataFormat", name) == 0) { - // tsSmlDataFormat = cfgGetItem(pCfg, "smlDataFormat")->bval; - // } else if (strcasecmp("smlBatchSize", name) == 0) { - // tsSmlBatchSize = cfgGetItem(pCfg, "smlBatchSize")->i32; - } else if (strcasecmp("smlTsDefaultName", name) == 0) { - tstrncpy(tsSmlTsDefaultName, cfgGetItem(pCfg, "smlTsDefaultName")->str, TSDB_COL_NAME_LEN); - } else if (strcasecmp("smlDot2Underline", name) == 0) { - tsSmlDot2Underline = cfgGetItem(pCfg, "smlDot2Underline")->bval; - } else if (strcasecmp("shellActivityTimer", name) == 0) { - tsShellActivityTimer = cfgGetItem(pCfg, "shellActivityTimer")->i32; - } else if (strcasecmp("serverPort", name) == 0) { - tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); - tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; - snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); - - char defaultFirstEp[TSDB_EP_LEN] = {0}; - snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); - - SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); - SEp firstEp = {0}; - taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); - snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); - cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); - } else if (strcasecmp("smaDebugFlag", name) == 0) { - smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; - } else if (strcasecmp("slowLogThreshold", name) == 0) { - tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; - } else if (strcasecmp("slowLogScope", name) == 0) { - if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { - return -1; - } - } else { - matchItem = false; - } - break; - } - case 't': { - if (strcasecmp("timezone", name) == 0) { - SConfigItem *pItem = cfgGetItem(pCfg, "timezone"); - osSetTimezone(pItem->str); - uDebug("timezone format changed from %s to %s", pItem->str, tsTimezoneStr); - cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype); - } else if (strcasecmp("tempDir", name) == 0) { - tstrncpy(tsTempDir, cfgGetItem(pCfg, "tempDir")->str, PATH_MAX); - taosExpandDir(tsTempDir, tsTempDir, PATH_MAX); - if (taosMulMkDir(tsTempDir) != 0) { - uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); - return -1; - } - } else if (strcasecmp("telemetryServer", name) == 0) { - tstrncpy(tsTelemServer, cfgGetItem(pCfg, "telemetryServer")->str, TSDB_FQDN_LEN); - } else if (strcasecmp("tmrDebugFlag", name) == 0) { - tmrDebugFlag = cfgGetItem(pCfg, "tmrDebugFlag")->i32; - } else { - matchItem = false; - } - break; - } - case 'u': { - if (strcasecmp("uDebugFlag", name) == 0) { - uDebugFlag = cfgGetItem(pCfg, "uDebugFlag")->i32; - } else if (strcasecmp("useAdapter", name) == 0) { - tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; - } else { - matchItem = false; - } - break; - } - default: - terrno = TSDB_CODE_CFG_NOT_FOUND; - return -1; - } - - if (!matchItem) terrno = TSDB_CODE_CFG_NOT_FOUND; - return matchItem ? 0 : -1; -} - int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { if (tsCfg == NULL) osDefaultInit(); @@ -1629,49 +1338,98 @@ void taosCleanupCfg() { tsCfg = NULL; } } + typedef struct { const char *optionName; void *optionVar; } OptionNameAndVar; -void taosCfgDynamicOptions(const char *option, const char *value) { - if (strncasecmp(option, "debugFlag", 9) == 0) { - int32_t flag = atoi(value); - taosSetAllDebugFlag(flag, true); - return; +static int32_t taosCfgSetOption(OptionNameAndVar *pOptions, int32_t optionSize, SConfigItem *pItem, bool isDebugflag) { + terrno = TSDB_CODE_INVALID_CFG; + char *name = pItem->name; + for (int32_t d = 0; d < optionSize; ++d) { + const char *optName = pOptions[d].optionName; + int32_t optLen = strlen(optName); + if (strncasecmp(name, optName, optLen) != 0) continue; + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: { + int32_t flag = pItem->i32; + bool *pVar = pOptions[d].optionVar; + uInfo("%s set from %d to %d", optName, *pVar, flag); + *pVar = flag; + terrno = TSDB_CODE_SUCCESS; + } break; + case CFG_DTYPE_INT32: { + int32_t flag = pItem->i32; + int32_t *pVar = pOptions[d].optionVar; + uInfo("%s set from %d to %d", optName, *pVar, flag); + *pVar = flag; + + if (isDebugflag) { + taosSetDebugFlag(pOptions[d].optionVar, optName, flag, true); + } + terrno = TSDB_CODE_SUCCESS; + } break; + case CFG_DTYPE_INT64: { + int64_t flag = pItem->i64; + int64_t *pVar = pOptions[d].optionVar; + uInfo("%s set from %" PRId64 " to %" PRId64, optName, *pVar, flag); + *pVar = flag; + terrno = TSDB_CODE_SUCCESS; + } break; + case CFG_DTYPE_FLOAT: + case CFG_DTYPE_DOUBLE: { + float flag = pItem->fval; + float *pVar = pOptions[d].optionVar; + uInfo("%s set from %f to %f", optName, *pVar, flag); + *pVar = flag; + terrno = TSDB_CODE_SUCCESS; + } break; + default: + terrno = TSDB_CODE_INVALID_CFG; + break; + } + + break; } - if (strcasecmp(option, "resetlog") == 0) { + return terrno == TSDB_CODE_SUCCESS ? 0 : -1; +} + +static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) { + terrno = TSDB_CODE_SUCCESS; + + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { + uError("failed to config:%s, not support", name); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + if (strncasecmp(name, "debugFlag", 9) == 0) { + int32_t flag = pItem->i32; + taosSetAllDebugFlag(flag, true); + return 0; + } + + if (strcasecmp(name, "resetlog") == 0) { taosResetLog(); cfgDumpCfg(tsCfg, 0, false); - return; + return 0; } - { // 'bool/int32_t/int64_t' variables with general modification function - const int32_t nDebugFlag = 20; - static OptionNameAndVar options[] = { - {"dDebugFlag", &dDebugFlag}, - {"vDebugFlag", &vDebugFlag}, - {"mDebugFlag", &mDebugFlag}, - {"wDebugFlag", &wDebugFlag}, - {"sDebugFlag", &sDebugFlag}, - {"tsdbDebugFlag", &tsdbDebugFlag}, - {"tqDebugFlag", &tqDebugFlag}, - {"fsDebugFlag", &fsDebugFlag}, - {"udfDebugFlag", &udfDebugFlag}, - {"smaDebugFlag", &smaDebugFlag}, - {"idxDebugFlag", &idxDebugFlag}, - {"tdbDebugFlag", &tdbDebugFlag}, - {"tmrDebugFlag", &tmrDebugFlag}, - {"uDebugFlag", &uDebugFlag}, - {"smaDebugFlag", &smaDebugFlag}, - {"rpcDebugFlag", &rpcDebugFlag}, - {"qDebugFlag", &qDebugFlag}, - {"metaDebugFlag", &metaDebugFlag}, - {"jniDebugFlag", &jniDebugFlag}, - {"stDebugFlag", &stDebugFlag}, - {"sndDebugFlag", &sndDebugFlag}, + { // 'bool/int32_t/int64_t/float/double' variables with general modification function + static OptionNameAndVar debugOptions[] = { + {"dDebugFlag", &dDebugFlag}, {"vDebugFlag", &vDebugFlag}, {"mDebugFlag", &mDebugFlag}, + {"wDebugFlag", &wDebugFlag}, {"sDebugFlag", &sDebugFlag}, {"tsdbDebugFlag", &tsdbDebugFlag}, + {"tqDebugFlag", &tqDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, {"udfDebugFlag", &udfDebugFlag}, + {"smaDebugFlag", &smaDebugFlag}, {"idxDebugFlag", &idxDebugFlag}, {"tdbDebugFlag", &tdbDebugFlag}, + {"tmrDebugFlag", &tmrDebugFlag}, {"uDebugFlag", &uDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, + {"rpcDebugFlag", &rpcDebugFlag}, {"qDebugFlag", &qDebugFlag}, {"metaDebugFlag", &metaDebugFlag}, + {"jniDebugFlag", &jniDebugFlag}, {"stDebugFlag", &stDebugFlag}, {"sndDebugFlag", &sndDebugFlag}, + }; + static OptionNameAndVar options[] = { {"audit", &tsEnableAudit}, {"asynclog", &tsAsyncLog}, {"disableStream", &tsDisableStream}, @@ -1704,54 +1462,241 @@ void taosCfgDynamicOptions(const char *option, const char *value) { {"supportVnodes", &tsNumOfSupportVnodes}, }; - int32_t optionSize = tListLen(options); - for (int32_t d = 0; d < optionSize; ++d) { - const char *optName = options[d].optionName; - int32_t optLen = strlen(optName); - if (strncasecmp(option, optName, optLen) != 0) continue; - - SConfig *pCfg = taosGetCfg(); - SConfigItem *pItem = NULL; - - pItem = cfgGetItem(pCfg, optName); - if (!pItem || (pItem->dynScope & CFG_DYN_SERVER) == 0) { - uError("failed to config:%s, not support", optName); - break; - } - - switch (pItem->dtype) { - case CFG_DTYPE_BOOL: { - int32_t flag = atoi(value); - bool *pVar = options[d].optionVar; - uInfo("%s set from %d to %d", optName, *pVar, flag); - *pVar = flag; - } break; - case CFG_DTYPE_INT32: { - int32_t flag = atoi(value); - int32_t *pVar = options[d].optionVar; - uInfo("%s set from %d to %d", optName, *pVar, flag); - *pVar = flag; - - if (d < nDebugFlag) { - // debug flags - taosSetDebugFlag(options[d].optionVar, optName, flag, true); - } - } break; - case CFG_DTYPE_INT64: { - int64_t flag = atoll(value); - int64_t *pVar = options[d].optionVar; - uInfo("%s set from %" PRId64 " to %" PRId64, optName, *pVar, flag); - *pVar = flag; - } break; - default: - break; - } - - return; + if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { + taosCfgSetOption(options, tListLen(options), pItem, false); } } - uError("failed to cfg dynamic option:%s value:%s", option, value); + return terrno == TSDB_CODE_SUCCESS ? 0 : -1; +} + +static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) { + terrno = TSDB_CODE_SUCCESS; + + SConfigItem *pItem = cfgGetItem(pCfg, name); + if (!pItem || (pItem->dynScope & CFG_DYN_CLIENT) == 0) { + uError("failed to config:%s, not support", name); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + bool matched = false; + + int32_t len = strlen(name); + char lowcaseName[CFG_NAME_MAX_LEN + 1] = {0}; + strntolower(lowcaseName, name, TMIN(CFG_NAME_MAX_LEN, len)); + switch (lowcaseName[0]) { + case 'd': { + if (strcasecmp("debugFlag", name) == 0) { + int32_t flag = pItem->i32; + taosSetAllDebugFlag(flag, true); + matched = true; + } + break; + } + case 'e': { + if (strcasecmp("enableCoreFile", name) == 0) { + bool enableCore = pItem->bval; + taosSetCoreDump(enableCore); + uInfo("%s set to %d", name, enableCore); + matched = true; + } + break; + } + case 'f': { + if (strcasecmp("fqdn", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + uInfo("localEp set to '%s', tsFirst set to '%s'", tsLocalEp, tsFirst); + matched = true; + } else if (strcasecmp("firstEp", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + uInfo("localEp set to '%s', tsFirst set to '%s'", tsLocalEp, tsFirst); + matched = true; + } + break; + } + case 'l': { + if (strcasecmp("locale", name) == 0) { + const char *locale = cfgGetItem(pCfg, "locale")->str; + const char *charset = cfgGetItem(pCfg, "charset")->str; + taosSetSystemLocale(locale, charset); + osSetSystemLocale(locale, charset); + uInfo("locale set to '%s', charset set to '%s'", locale, charset); + matched = true; + } else if (strcasecmp("logDir", name) == 0) { + uInfo("%s set from '%s' to '%s'", name, tsLogDir, pItem->str); + tstrncpy(tsLogDir, pItem->str, PATH_MAX); + taosExpandDir(tsLogDir, tsLogDir, PATH_MAX); + matched = true; + } + break; + } + case 'm': { + if (strcasecmp("metaCacheMaxSize", name) == 0) { + atomic_store_32(&tsMetaCacheMaxSize, pItem->i32); + uInfo("%s set to %d", name, atomic_load_32(&tsMetaCacheMaxSize)); + matched = true; + } else if (strcasecmp("minimalTmpDirGB", name) == 0) { + tsTempSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); + uInfo("%s set to %"PRId64, name, tsTempSpace.reserved); + matched = true; + } else if (strcasecmp("minimalDataDirGB", name) == 0) { + tsDataSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); + uInfo("%s set to %"PRId64, name, tsDataSpace.reserved); + matched = true; + } else if (strcasecmp("minimalLogDirGB", name) == 0) { + tsLogSpace.reserved = (int64_t)(((double)pItem->fval) * 1024 * 1024 * 1024); + uInfo("%s set to %"PRId64, name, tsLogSpace.reserved); + matched = true; + } + break; + } + case 's': { + if (strcasecmp("secondEp", name) == 0) { + SEp secondEp = {0}; + taosGetFqdnPortFromEp(strlen(pItem->str) == 0 ? tsFirst : pItem->str, &secondEp); + snprintf(tsSecond, sizeof(tsSecond), "%s:%u", secondEp.fqdn, secondEp.port); + cfgSetItem(pCfg, "secondEp", tsSecond, pItem->stype); + uInfo("%s set to %s", name, tsSecond); + matched = true; + } else if (strcasecmp("smlChildTableName", name) == 0) { + uInfo("%s set from %s to %s", name, tsSmlChildTableName, pItem->str); + tstrncpy(tsSmlChildTableName, pItem->str, TSDB_TABLE_NAME_LEN); + matched = true; + } else if (strcasecmp("smlAutoChildTableNameDelimiter", name) == 0) { + uInfo("%s set from %s to %s", name, tsSmlAutoChildTableNameDelimiter, pItem->str); + tstrncpy(tsSmlAutoChildTableNameDelimiter, pItem->str, TSDB_TABLE_NAME_LEN); + matched = true; + } else if (strcasecmp("smlTagName", name) == 0) { + uInfo("%s set from %s to %s", name, tsSmlTagName, pItem->str); + tstrncpy(tsSmlTagName, pItem->str, TSDB_COL_NAME_LEN); + matched = true; + } else if (strcasecmp("smlTsDefaultName", name) == 0) { + uInfo("%s set from %s to %s", name, tsSmlTsDefaultName, pItem->str); + tstrncpy(tsSmlTsDefaultName, pItem->str, TSDB_COL_NAME_LEN); + matched = true; + } else if (strcasecmp("serverPort", name) == 0) { + tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); + tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; + snprintf(tsLocalEp, sizeof(tsLocalEp), "%s:%u", tsLocalFqdn, tsServerPort); + + char defaultFirstEp[TSDB_EP_LEN] = {0}; + snprintf(defaultFirstEp, TSDB_EP_LEN, "%s:%u", tsLocalFqdn, tsServerPort); + + SConfigItem *pFirstEpItem = cfgGetItem(pCfg, "firstEp"); + SEp firstEp = {0}; + taosGetFqdnPortFromEp(strlen(pFirstEpItem->str) == 0 ? defaultFirstEp : pFirstEpItem->str, &firstEp); + snprintf(tsFirst, sizeof(tsFirst), "%s:%u", firstEp.fqdn, firstEp.port); + cfgSetItem(pCfg, "firstEp", tsFirst, pFirstEpItem->stype); + uInfo("localEp set to '%s', tsFirst set to '%s'", tsLocalEp, tsFirst); + matched = true; + } else if (strcasecmp("slowLogScope", name) == 0) { + if (taosSetSlowLogScope(pItem->str)) { + return -1; + } + uInfo("%s set to %s", name, pItem->str); + matched = true; + } + break; + } + case 't': { + if (strcasecmp("timezone", name) == 0) { + osSetTimezone(pItem->str); + uInfo("%s set from %s to %s", name, tsTimezoneStr, pItem->str); + cfgSetItem(pCfg, "timezone", tsTimezoneStr, pItem->stype); + matched = true; + } else if (strcasecmp("tempDir", name) == 0) { + uInfo("%s set from %s to %s", name, tsTempDir, pItem->str); + tstrncpy(tsTempDir, pItem->str, PATH_MAX); + taosExpandDir(tsTempDir, tsTempDir, PATH_MAX); + if (taosMulMkDir(tsTempDir) != 0) { + uError("failed to create tempDir:%s since %s", tsTempDir, terrstr()); + return -1; + } + matched = true; + } else if (strcasecmp("telemetryServer", name) == 0) { + uInfo("%s set from %s to %s", name, pItem->str, tsTelemServer); + tstrncpy(tsTelemServer, pItem->str, TSDB_FQDN_LEN); + matched = true; + } + break; + } + default: + terrno = TSDB_CODE_CFG_NOT_FOUND; + break; + } + + if (matched) goto _out; + + { // 'bool/int32_t/int64_t/float/double' variables with general modification function + static OptionNameAndVar debugOptions[] = { + {"cDebugFlag", &cDebugFlag}, {"dDebugFlag", &dDebugFlag}, {"fsDebugFlag", &fsDebugFlag}, + {"idxDebugFlag", &idxDebugFlag}, {"jniDebugFlag", &jniDebugFlag}, {"qDebugFlag", &qDebugFlag}, + {"rpcDebugFlag", &rpcDebugFlag}, {"smaDebugFlag", &smaDebugFlag}, {"tmrDebugFlag", &tmrDebugFlag}, + {"uDebugFlag", &uDebugFlag}, + }; + + static OptionNameAndVar options[] = { + {"asyncLog", &tsAsyncLog}, + {"assert", &tsAssert}, + {"compressMsgSize", &tsCompressMsgSize}, + {"countAlwaysReturnValue", &tsCountAlwaysReturnValue}, + {"crashReporting", &tsEnableCrashReport}, + {"enableCoreFile", &tsAsyncLog}, + {"enableQueryHb", &tsEnableQueryHb}, + {"keepColumnName", &tsKeepColumnName}, + {"keepAliveIdle", &tsKeepAliveIdle}, + {"logKeepDays", &tsLogKeepDays}, + {"maxInsertBatchRows", &tsMaxInsertBatchRows}, + {"maxRetryWaitTime", &tsMaxRetryWaitTime}, + {"minSlidingTime", &tsMinSlidingTime}, + {"minIntervalTime", &tsMinIntervalTime}, + {"numOfLogLines", &tsNumOfLogLines}, + {"querySmaOptimize", &tsQuerySmaOptimize}, + {"queryPolicy", &tsQueryPolicy}, + {"queryPlannerTrace", &tsQueryPlannerTrace}, + {"queryNodeChunkSize", &tsQueryNodeChunkSize}, + {"queryUseNodeAllocator", &tsQueryUseNodeAllocator}, + {"smlDot2Underline", &tsSmlDot2Underline}, + {"shellActivityTimer", &tsShellActivityTimer}, + {"slowLogThreshold", &tsSlowLogThreshold}, + {"useAdapter", &tsUseAdapter}, + }; + + if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) { + taosCfgSetOption(options, tListLen(options), pItem, false); + } + } + +_out: + return terrno == TSDB_CODE_SUCCESS ? 0 : -1; +} + +int32_t taosCfgDynamicOptions(SConfig *pCfg, char *name, bool forServer) { + if (forServer) return taosCfgDynamicOptionsForServer(pCfg, name); + return taosCfgDynamicOptionsForClient(pCfg, name); } void taosSetDebugFlag(int32_t *pFlagPtr, const char *flagName, int32_t flagVal, bool rewrite) { diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index a701c88a24..227de7f5fc 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -194,6 +194,14 @@ int32_t parseTimezone(char* str, int64_t* tzOffset) { return 0; } +int32_t offsetOfTimezone(char* tzStr, int64_t* offset) { + if (tzStr && (tzStr[0] == 'z' || tzStr[0] == 'Z')) { + *offset = 0; + return 0; + } + return parseTimezone(tzStr, offset); +} + /* * rfc3339 format: * 2013-04-12T15:52:01+08:00 diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 991f17f326..ac94390619 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -218,7 +218,10 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); - taosCfgDynamicOptions(cfgReq.config, cfgReq.value); + + SConfig *pCfg = taosGetCfg(); + cfgSetItem(pCfg, cfgReq.config, cfgReq.value, CFG_STYPE_ALTER_CMD); + taosCfgDynamicOptions(pCfg, cfgReq.config, true); return 0; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index f4108b52c6..e224aceec2 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -27,6 +27,7 @@ #include "mndUser.h" #include "mndVgroup.h" #include "tmisce.h" +#include "tunit.h" #define TSDB_DNODE_VER_NUMBER 2 #define TSDB_DNODE_RESERVE_SIZE 64 @@ -1316,8 +1317,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag); if (code < 0) return code; - if (flag > 1024 * 1024 || (flag > -1 && flag < 4) || flag < -1) { - mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [4, 1024 * 1024]", + if (flag > 1024 * 1024 || (flag > -1 && flag < 1024) || flag < -1) { + mError("dnode:%d, failed to config s3blocksize since value:%d. Valid range: -1 or [1024, 1024 * 1024]", cfgReq.dnodeId, flag); terrno = TSDB_CODE_INVALID_CFG; tFreeSMCfgDnodeReq(&cfgReq); @@ -1328,7 +1329,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); #endif } else { - mndMCfg2DCfg(&cfgReq, &dcfgReq); + if (mndMCfg2DCfg(&cfgReq, &dcfgReq)) goto _err_out; if (strlen(dcfgReq.config) > TSDB_DNODE_CONFIG_LEN) { mError("dnode:%d, failed to config since config is too long", cfgReq.dnodeId); terrno = TSDB_CODE_INVALID_CFG; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 1b21d4a017..82d0074fbd 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -882,7 +882,6 @@ static int32_t mndCreateStb(SMnode *pMnode, SRpcMsg *pReq, SMCreateStbReq *pCrea if (mndAcquireGlobalIdx(pMnode, fullIdxName, SDB_IDX, &idx) == 0 && idx.pIdx != NULL) { terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; mndReleaseIdx(pMnode, idx.pIdx); - goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 1c9c3887a2..66e3982cba 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -678,7 +678,7 @@ _OVER: return -1; } -static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { +static int32_t mndPersistTaskDropReq(SMnode* pMnode, STrans *pTrans, SStreamTask *pTask) { SVDropStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVDropStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -690,7 +690,12 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { pReq->streamId = pTask->id.streamId; STransAction action = {0}; - initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &pTask->info.epSet, 0); + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + + // The epset of nodeId of this task may have been expired now, let's use the newest epset from mnode. + initTransAction(&action, pReq, sizeof(SVDropStreamTaskReq), TDMT_STREAM_TASK_DROP, &epset, 0); if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; @@ -706,7 +711,7 @@ int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (mndPersistTaskDropReq(pTrans, pTask) < 0) { + if (mndPersistTaskDropReq(pMnode, pTrans, pTask) < 0) { return -1; } } @@ -1085,9 +1090,10 @@ static int32_t mndAddStreamCheckpointToTrans(STrans *pTrans, SStreamObj *pStream STransAction action = {0}; SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + initTransAction(&action, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY); - mndReleaseVgroup(pMnode, pVgObj); if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(buf); @@ -1283,12 +1289,13 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { if (pStream == NULL) { if (dropReq.igNotExists) { - mInfo("stream:%s, not exist, ignore not exist is set", dropReq.name); + mInfo("stream:%s not exist, ignore not exist is set", dropReq.name); sdbRelease(pMnode->pSdb, pStream); tFreeSMDropStreamReq(&dropReq); return 0; } else { terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; + mError("stream:%s not exist failed to drop", dropReq.name); tFreeSMDropStreamReq(&dropReq); return -1; } @@ -1660,7 +1667,7 @@ static void mndCancelGetNextStreamTask(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } -static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { +static int32_t mndPauseStreamTask(SMnode* pMnode, STrans *pTrans, SStreamTask *pTask) { SVPauseStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVPauseStreamTaskReq)); if (pReq == NULL) { mError("failed to malloc in pause stream, size:%" PRIzu ", code:%s", sizeof(SVPauseStreamTaskReq), @@ -1673,8 +1680,12 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + STransAction action = {0}; - initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &pTask->info.epSet, 0); + initTransAction(&action, pReq, sizeof(SVPauseStreamTaskReq), TDMT_STREAM_TASK_PAUSE, &epset, 0); if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; @@ -1682,7 +1693,7 @@ static int32_t mndPauseStreamTask(STrans *pTrans, SStreamTask *pTask) { return 0; } -int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { +int32_t mndPauseAllStreamTasks(SMnode* pMnode, STrans *pTrans, SStreamObj *pStream) { SArray *tasks = pStream->tasks; int32_t size = taosArrayGetSize(tasks); @@ -1691,7 +1702,7 @@ int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (mndPauseStreamTask(pTrans, pTask) < 0) { + if (mndPauseStreamTask(pMnode, pTrans, pTask) < 0) { return -1; } @@ -1768,7 +1779,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { } // pause all tasks - if (mndPauseAllStreamTasks(pTrans, pStream) < 0) { + if (mndPauseAllStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to pause task since %s", pauseReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); @@ -1795,7 +1806,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) { return TSDB_CODE_ACTION_IN_PROGRESS; } -static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t igUntreated) { +static int32_t mndResumeStreamTask(STrans *pTrans, SMnode* pMnode, SStreamTask *pTask, int8_t igUntreated) { SVResumeStreamTaskReq *pReq = taosMemoryCalloc(1, sizeof(SVResumeStreamTaskReq)); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1806,8 +1817,12 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t ig pReq->streamId = pTask->id.streamId; pReq->igUntreated = igUntreated; + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + STransAction action = {0}; - initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &pTask->info.epSet, 0); + initTransAction(&action, pReq, sizeof(SVResumeStreamTaskReq), TDMT_STREAM_TASK_RESUME, &epset, 0); if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; @@ -1815,14 +1830,14 @@ static int32_t mndResumeStreamTask(STrans *pTrans, SStreamTask *pTask, int8_t ig return 0; } -int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUntreated) { +int32_t mndResumeAllStreamTasks(STrans *pTrans, SMnode* pMnode, SStreamObj *pStream, int8_t igUntreated) { int32_t size = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < size; i++) { SArray *pTasks = taosArrayGetP(pStream->tasks, i); int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (mndResumeStreamTask(pTrans, pTask, igUntreated) < 0) { + if (mndResumeStreamTask(pTrans, pMnode, pTask, igUntreated) < 0) { return -1; } @@ -1831,7 +1846,6 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUn } } } - // pStream->pHTasksList is null return 0; } @@ -1884,7 +1898,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) { } // resume all tasks - if (mndResumeAllStreamTasks(pTrans, pStream, pauseReq.igUntreated) < 0) { + if (mndResumeAllStreamTasks(pTrans, pMnode, pStream, pauseReq.igUntreated) < 0) { mError("stream:%s, failed to drop task since %s", pauseReq.name, terrstr()); sdbRelease(pMnode->pSdb, pStream); mndTransDrop(pTrans); @@ -2557,8 +2571,12 @@ int32_t createStreamResetStatusTrans(SMnode *pMnode, SStreamObj *pStream) { pReq->taskId = pTask->id.taskId; pReq->streamId = pTask->id.streamId; + SVgObj *pVgObj = mndAcquireVgroup(pMnode, pTask->info.nodeId); + SEpSet epset = mndGetVgroupEpset(pMnode, pVgObj); + mndReleaseVgroup(pMnode, pVgObj); + STransAction action = {0}; - initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &pTask->info.epSet, 0); + initTransAction(&action, pReq, sizeof(SVResetStreamTaskReq), TDMT_VND_STREAM_TASK_RESET, &epset, 0); if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); taosWUnLockLatch(&pStream->lock); diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index af4ccd4ad0..01ffa92513 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -2176,7 +2176,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { code = mndRedistributeVgroup(pMnode, pReq, pDb, pVgroup, pNew1, pOld1, pNew2, pOld2, pNew3, pOld3); } else { - terrno = TSDB_CODE_MND_INVALID_REPLICA; + terrno = TSDB_CODE_MND_REQ_REJECTED; goto _OVER; } diff --git a/source/dnode/snode/src/snodeInitApi.c b/source/dnode/snode/src/snodeInitApi.c index 389137f630..570feffc14 100644 --- a/source/dnode/snode/src/snodeInitApi.c +++ b/source/dnode/snode/src/snodeInitApi.c @@ -71,6 +71,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition; pStore->updateInfoInit = updateInfoInit; pStore->updateInfoFillBlockData = updateInfoFillBlockData; diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index ec69ae5ca7..ca9d22a987 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -831,7 +831,7 @@ struct SDiskDataBuilder { SBlkInfo bi; }; -typedef struct SLDataIter { +struct SLDataIter { SRBTreeNode node; SSttBlk *pSttBlk; int64_t cid; // for debug purpose @@ -845,7 +845,7 @@ typedef struct SLDataIter { SSttBlockLoadInfo *pBlockLoadInfo; bool ignoreEarlierTs; struct SSttFileReader *pReader; -} SLDataIter; +}; #define tMergeTreeGetRow(_t) (&((_t)->pIter->rInfo.row)) diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 521686d023..8ee0edcb31 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -1100,10 +1100,6 @@ static void doStartFillhistoryStep2(SStreamTask* pTask, SStreamTask* pStreamTask } } -static void ddxx() { - -} - // this function should be executed by only one thread, so we set an sentinel to protect this function int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { SStreamScanHistoryReq* pReq = (SStreamScanHistoryReq*)pMsg->pCont; @@ -1150,7 +1146,9 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) { tqDebug("s-task:%s continue exec scan-history(step1), original step1 startTs:%" PRId64 ", already elapsed:%.2fs", id, pTask->execInfo.step1Start, pTask->execInfo.step1El); } else { - tqDebug("s-task:%s already in step2, no need to scan-history data, step2 starTs:%"PRId64, id, pTask->execInfo.step2Start); + tqDebug("s-task:%s already in step2, no need to scan-history data, step2 startTs:%" PRId64, id, + pTask->execInfo.step2Start); + atomic_store_32(&pTask->status.inScanHistorySentinel, 0); streamMetaReleaseTask(pMeta, pTask); return 0; @@ -1969,6 +1967,8 @@ int32_t tqProcessTaskUpdateReq(STQ* pTq, SRpcMsg* pMsg) { return -1; } + streamMetaInitBackend(pMeta); + if (streamMetaLoadAllTasks(pTq->pStreamMeta) < 0) { tqError("vgId:%d failed to load stream tasks", vgId); streamMetaWUnLock(pMeta); diff --git a/source/dnode/vnode/src/tq/tqStreamStateSnap.c b/source/dnode/vnode/src/tq/tqStreamStateSnap.c index 41392ba27b..7a8147f83b 100644 --- a/source/dnode/vnode/src/tq/tqStreamStateSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamStateSnap.c @@ -169,10 +169,15 @@ int32_t streamStateSnapWriterClose(SStreamStateWriter* pWriter, int8_t rollback) } int32_t streamStateRebuildFromSnap(SStreamStateWriter* pWriter, int64_t chkpId) { tqDebug("vgId:%d, vnode %s start to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); + + streamMetaWLock(pWriter->pTq->pStreamMeta); int32_t code = streamMetaReopen(pWriter->pTq->pStreamMeta); if (code == 0) { + streamMetaInitBackend(pWriter->pTq->pStreamMeta); code = streamStateLoadTasks(pWriter); } + + streamMetaWUnLock(pWriter->pTq->pStreamMeta); tqDebug("vgId:%d, vnode %s succ to rebuild stream-state", TD_VID(pWriter->pTq->pVnode), STREAM_STATE_TRANSFER); taosMemoryFree(pWriter); return code; diff --git a/source/dnode/vnode/src/tq/tqStreamTask.c b/source/dnode/vnode/src/tq/tqStreamTask.c index 9f00f9cc10..e069d6c4b7 100644 --- a/source/dnode/vnode/src/tq/tqStreamTask.c +++ b/source/dnode/vnode/src/tq/tqStreamTask.c @@ -145,7 +145,6 @@ int32_t tqRestartStreamTasks(STQ* pTq) { } streamMetaWLock(pMeta); - code = streamMetaReopen(pMeta); if (code != TSDB_CODE_SUCCESS) { tqError("vgId:%d failed to reopen stream meta", vgId); @@ -154,9 +153,10 @@ int32_t tqRestartStreamTasks(STQ* pTq) { return code; } + streamMetaInitBackend(pMeta); int64_t el = taosGetTimestampMs() - st; - tqInfo("vgId:%d close&reload state elapsed time:%.3fms", vgId, el/1000.); + tqInfo("vgId:%d close&reload state elapsed time:%.3fs", vgId, el/1000.); code = streamMetaLoadAllTasks(pTq->pStreamMeta); if (code != TSDB_CODE_SUCCESS) { @@ -169,12 +169,15 @@ int32_t tqRestartStreamTasks(STQ* pTq) { if (vnodeIsRoleLeader(pTq->pVnode) && !tsDisableStream) { tqInfo("vgId:%d restart all stream tasks after all tasks being updated", vgId); tqResetStreamTaskStatus(pTq); + + streamMetaWUnLock(pMeta); tqStartStreamTasks(pTq); } else { + streamMetaResetStartInfo(&pMeta->startInfo); + streamMetaWUnLock(pMeta); tqInfo("vgId:%d, follower node not start stream tasks", vgId); } - streamMetaWUnLock(pMeta); code = terrno; return code; } diff --git a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c index e122cf19d3..f22ecc3daf 100644 --- a/source/dnode/vnode/src/tq/tqStreamTaskSnap.c +++ b/source/dnode/vnode/src/tq/tqStreamTaskSnap.c @@ -212,9 +212,7 @@ int32_t streamTaskSnapWriterClose(SStreamTaskWriter* pWriter, int8_t rollback) { taosMemoryFree(pWriter); goto _err; } - taosWUnLockLatch(&pTq->pStreamMeta->lock); - taosMemoryFree(pWriter); return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit2.c b/source/dnode/vnode/src/tsdb/tsdbCommit2.c index 22fb3b84ec..48b622e324 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit2.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit2.c @@ -409,7 +409,6 @@ static int32_t tsdbCommitFileSetBegin(SCommitter2 *committer) { extern int32_t tsS3UploadDelaySec; long s3Size(const char *object_name); int32_t nlevel = tfsGetLevel(committer->tsdb->pVnode->pTfs); - committer->ctx->skipTsRow = false; if (tsS3Enabled && nlevel > 1 && committer->ctx->fset) { STFileObj *fobj = committer->ctx->fset->farr[TSDB_FTYPE_DATA]; if (fobj && fobj->f->did.level == nlevel - 1) { @@ -670,4 +669,4 @@ _exit: tsdbInfo("vgId:%d %s done", TD_VID(pTsdb->pVnode), __func__); } return code; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 02ef75ae86..a3b3ffc1c8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -882,8 +882,38 @@ int32_t tsdbFSEditCommit(STFileSystem *fs) { int32_t numFile = TARRAY2_SIZE(lvl->fobjArr); if (numFile >= sttTrigger) { // launch merge - code = tsdbSchedMerge(fs->tsdb, fset->fid); - TSDB_CHECK_CODE(code, lino, _exit); + bool skipMerge = false; + { + int32_t now = taosGetTimestampSec(); + + extern int8_t tsS3Enabled; + extern int32_t tsS3UploadDelaySec; + long s3Size(const char *object_name); + int32_t nlevel = tfsGetLevel(fs->tsdb->pVnode->pTfs); + if (tsS3Enabled && nlevel > 1) { + STFileObj *fobj = fset->farr[TSDB_FTYPE_DATA]; + if (fobj && fobj->f->did.level == nlevel - 1) { + // if exists on s3 or local mtime < committer->ctx->now - tsS3UploadDelay + const char *object_name = taosDirEntryBaseName((char *)fobj->fname); + + if (taosCheckExistFile(fobj->fname)) { + int32_t mtime = 0; + taosStatFile(fobj->fname, NULL, &mtime, NULL); + if (mtime < now - tsS3UploadDelaySec) { + skipMerge = true; + } + } else if (s3Size(object_name) > 0) { + skipMerge = true; + } + } + // new fset can be written with ts data + } + } + + if (!skipMerge) { + code = tsdbSchedMerge(fs->tsdb, fset->fid); + TSDB_CHECK_CODE(code, lino, _exit); + } } if (numFile >= sttTrigger * BLOCK_COMMIT_FACTOR) { diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index cc05b8ee18..8cd9304188 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -50,6 +50,7 @@ void remove_file(const char *fname, bool last_level) { long s3_size = tsS3Enabled ? s3Size(object_name) : 0; if (!strncmp(fname + strlen(fname) - 5, ".data", 5) && s3_size > 0) { s3DeleteObjects(&object_name, 1); + tsdbInfo("file:%s is removed from s3", fname); } else { tsdbError("file:%s remove failed", fname); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 4682c47bd1..6169014d9f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -524,7 +524,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN int64_t st = taosGetTimestampUs(); // clear info for the new file - cleanupInfoFoxNextFileset(pReader->status.pTableMap); + cleanupInfoForNextFileset(pReader->status.pTableMap); int32_t k = 0; int32_t numOfTables = tSimpleHashGetSize(pReader->status.pTableMap); @@ -1444,7 +1444,9 @@ static bool nextRowFromSttBlocks(SLastBlockReader* pLastBlockReader, STableBlock } } -static void doPinSttBlock(SLastBlockReader* pLastBlockReader) { tMergeTreePinSttBlock(&pLastBlockReader->mergeTree); } +static void doPinSttBlock(SLastBlockReader* pLastBlockReader) { + tMergeTreePinSttBlock(&pLastBlockReader->mergeTree); +} static void doUnpinSttBlock(SLastBlockReader* pLastBlockReader) { tMergeTreeUnpinSttBlock(&pLastBlockReader->mergeTree); @@ -1454,7 +1456,6 @@ static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLas STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader, bool* copied) { int32_t code = TSDB_CODE_SUCCESS; - *copied = false; // avoid the fetch next row replace the referenced stt block in buffer @@ -1540,12 +1541,6 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - - int32_t code = tsdbRowMergerInit(pMerger, ps); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("failed to init row merger, code:%s", tstrerror(code)); - return code; - } } int64_t minKey = 0; @@ -1688,7 +1683,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, tsdbTrace("fRow ptr:%p, %d, uid:%" PRIu64 ", ts:%" PRId64 " %s", pRow->pBlockData, pRow->iRow, pLastBlockReader->uid, fRow.pBlockData->aTSKEY[fRow.iRow], pReader->idStr); - // only last block exists + // only stt block exists if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { code = tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader, &copied); if (code) { @@ -1767,12 +1762,6 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (ps == NULL) { return terrno; } - - int32_t code = tsdbRowMergerInit(pMerger, ps); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("failed to init row merger, code:%s", tstrerror(code)); - return code; - } } if (hasDataInFileBlock(pBlockData, pDumpInfo)) { @@ -1874,12 +1863,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (ps == NULL) { return terrno; } - - code = tsdbRowMergerInit(pMerger, ps); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("failed to init row merger, code:%s", tstrerror(code)); - return code; - } } int64_t minKey = 0; @@ -2202,12 +2185,6 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc if (ps == NULL) { return terrno; } - - code = tsdbRowMergerInit(pMerger, ps); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("failed to init row merger, code:%s", tstrerror(code)); - return code; - } } if (copied) { diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c index af7cae33fc..305399e0af 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.c @@ -245,7 +245,7 @@ static void doCleanupInfoForNextFileset(STableBlockScanInfo* pScanInfo) { pScanInfo->sttKeyInfo.status = STT_FILE_READER_UNINIT; } -void cleanupInfoFoxNextFileset(SSHashObj* pTableMap) { +void cleanupInfoForNextFileset(SSHashObj* pTableMap) { STableBlockScanInfo** p = NULL; int32_t iter = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h index 47585ea6e3..60e6e6960a 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReadUtil.h +++ b/source/dnode/vnode/src/tsdb/tsdbReadUtil.h @@ -248,7 +248,7 @@ SSHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, SBlockInfoBuf* pBuf void clearBlockScanInfo(STableBlockScanInfo* p); void destroyAllBlockScanInfo(SSHashObj* pTableMap); void resetAllDataBlockScanInfo(SSHashObj* pTableMap, int64_t ts, int32_t step); -void cleanupInfoFoxNextFileset(SSHashObj* pTableMap); +void cleanupInfoForNextFileset(SSHashObj* pTableMap); int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables); void clearBlockScanInfoBuf(SBlockInfoBuf* pBuf); void* getPosInBlockInfoBuf(SBlockInfoBuf* pBuf, int32_t index); diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index 194c2784f4..f6888ba9cb 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -74,6 +74,8 @@ static int32_t tsdbDoCopyFile(SRTNer *rtner, const STFileObj *from, const STFile if (fdFrom == NULL) code = terrno; TSDB_CHECK_CODE(code, lino, _exit); + tsdbInfo("vgId: %d, open tofile: %s size: %" PRId64, TD_VID(rtner->tsdb->pVnode), fname, from->f->size); + fdTo = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (fdTo == NULL) code = terrno; TSDB_CHECK_CODE(code, lino, _exit); @@ -101,7 +103,7 @@ static int32_t tsdbCopyFileS3(SRTNer *rtner, const STFileObj *from, const STFile char fname[TSDB_FILENAME_LEN]; TdFilePtr fdFrom = NULL; - TdFilePtr fdTo = NULL; + // TdFilePtr fdTo = NULL; tsdbTFileName(rtner->tsdb, to, fname); @@ -333,6 +335,7 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { int32_t mtime = 0; taosStatFile(fobj->fname, NULL, &mtime, NULL); if (mtime < rtner->now - tsS3UploadDelaySec) { + tsdbInfo("file:%s size: %" PRId64 " do migrate s3", fobj->fname, fobj->f->size); code = tsdbMigrateDataFileS3(rtner, fobj, &did); TSDB_CHECK_CODE(code, lino, _exit); } @@ -356,6 +359,12 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { s3EvictCache(fobj->fname, fsize * 2); } */ + if (fobj->f->did.level > did.level) { + continue; + } + tsdbInfo("file:%s size: %" PRId64 " do migrate from %d to %d", fobj->fname, fobj->f->size, fobj->f->did.level, + did.level); + code = tsdbDoMigrateFileObj(rtner, fobj, &did); TSDB_CHECK_CODE(code, lino, _exit); //} @@ -375,14 +384,6 @@ static int32_t tsdbDoRetentionOnFileSet(SRTNer *rtner, STFileSet *fset) { _exit: if (code) { - if (TARRAY2_DATA(rtner->fopArr)) { - TARRAY2_DESTROY(rtner->fopArr, NULL); - } - TFileSetArray **fsetArr = &rtner->fsetArr; - if (fsetArr[0]) { - tsdbFSDestroyCopySnapshot(&rtner->fsetArr); - } - TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } return code; @@ -437,13 +438,19 @@ static int32_t tsdbDoRetentionAsync(void *arg) { _exit: if (code) { + if (TARRAY2_DATA(rtner->fopArr)) { + TARRAY2_DESTROY(rtner->fopArr, NULL); + } + TFileSetArray **fsetArr = &rtner->fsetArr; + if (fsetArr[0]) { + tsdbFSDestroyCopySnapshot(&rtner->fsetArr); + } + TSDB_ERROR_LOG(TD_VID(rtner->tsdb->pVnode), lino, code); } return code; } - - int32_t tsdbRetention(STsdb *tsdb, int64_t now, int32_t sync) { int32_t code = 0; diff --git a/source/dnode/vnode/src/vnd/vnodeInitApi.c b/source/dnode/vnode/src/vnd/vnodeInitApi.c index a6dae429c3..a6673917bf 100644 --- a/source/dnode/vnode/src/vnd/vnodeInitApi.c +++ b/source/dnode/vnode/src/vnd/vnodeInitApi.c @@ -180,6 +180,7 @@ void initStateStoreAPI(SStateStore* pStore) { pStore->streamStateSessionGetKVByCur = streamStateSessionGetKVByCur; pStore->streamStateStateAddIfNotExist = streamStateStateAddIfNotExist; pStore->streamStateSessionGetKeyByRange = streamStateSessionGetKeyByRange; + pStore->streamStateSessionAllocWinBuffByNextPosition = streamStateSessionAllocWinBuffByNextPosition; pStore->updateInfoInit = updateInfoInit; pStore->updateInfoFillBlockData = updateInfoFillBlockData; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 899efc8e70..33b4114009 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -21,6 +21,8 @@ #include "cos.h" #include "vnode.h" #include "vnodeInt.h" +#include "audit.h" +#include "tstrbuild.h" static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessAlterStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp); @@ -1002,10 +1004,12 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq, taosMemoryFreeClear(*key); } - size_t len = 0; - char *keyJoined = taosStringBuilderGetResult(&sb, &len); + size_t len = 0; + char* keyJoined = taosStringBuilderGetResult(&sb, &len); - auditRecord(pOriginRpc, clusterId, "createTable", name.dbname, "", keyJoined, len); + if(pOriginRpc->info.conn.user != NULL && strlen(pOriginRpc->info.conn.user) > 0){ + auditRecord(pOriginRpc, clusterId, "createTable", name.dbname, "", keyJoined, len); + } taosStringBuilderDestroy(&sb); } @@ -1015,7 +1019,7 @@ _exit: pCreateReq = req.pReqs + iReq; taosMemoryFree(pCreateReq->sql); taosMemoryFree(pCreateReq->comment); - taosArrayDestroy(pCreateReq->ctb.tagName); + taosArrayDestroy(pCreateReq->ctb.tagName); } taosArrayDestroyEx(rsp.pArray, tFreeSVCreateTbRsp); taosArrayDestroy(tbUids); @@ -1227,11 +1231,13 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in size_t len = 0; char *keyJoined = taosStringBuilderGetResult(&sb, &len); - auditRecord(pOriginRpc, clusterId, "dropTable", name.dbname, "", keyJoined, len); + if(pOriginRpc->info.conn.user != NULL && strlen(pOriginRpc->info.conn.user) > 0){ + auditRecord(pOriginRpc, clusterId, "dropTable", name.dbname, "", keyJoined, len); + } taosStringBuilderDestroy(&sb); } - + _exit: taosArrayDestroy(tbUids); tdUidStoreFree(pStore); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 12063d4883..6bae0e1022 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -842,7 +842,7 @@ static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return terrno; } - if (taosApplyLocalCfg(tsCfg, pStmt->config)) { + if (taosCfgDynamicOptions(tsCfg, pStmt->config, false)) { return terrno; } diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index 331ce44366..cf717472b1 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -607,6 +607,31 @@ typedef struct SStreamStateAggOperatorInfo { bool recvGetAll; } SStreamStateAggOperatorInfo; +typedef struct SStreamEventAggOperatorInfo { + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + STimeWindowAggSupp twAggSup; + SSDataBlock* pDelRes; + SSHashObj* pSeDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; + bool ignoreExpiredData; + bool ignoreExpiredDataSaved; + SArray* pUpdated; + SSHashObj* pSeUpdated; + SSHashObj* pAllUpdated; + int64_t dataVersion; + bool isHistoryOp; + SArray* historyWins; + bool reCkBlock; + SSDataBlock* pCheckpointRes; + SFilterInfo* pStartCondInfo; + SFilterInfo* pEndCondInfo; +} SStreamEventAggOperatorInfo; + typedef struct SStreamPartitionOperatorInfo { SOptrBasicInfo binfo; SPartitionBySupporter partitionSup; @@ -765,9 +790,51 @@ void doClearBufferedBlocks(SStreamScanInfo* pInfo); uint64_t calcGroupId(char* pData, int32_t len); void streamOpReleaseState(struct SOperatorInfo* pOperator); void streamOpReloadState(struct SOperatorInfo* pOperator); +void destroyStreamAggSupporter(SStreamAggSupporter* pSup); +void clearGroupResInfo(SGroupResInfo* pGroupResInfo); +int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SFunctionStateStore* pStore); +int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, int32_t numOfOutput, int64_t gap, + SStreamState* pState, int32_t keySize, int16_t keyType, SStateStore* pStore, + SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr, + SStorageAPI* pApi); +void initDownStream(struct SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, uint16_t type, int32_t tsColIndex, + STimeWindowAggSupp* pTwSup); +void getMaxTsWins(const SArray* pAllWins, SArray* pMaxWins); +void initGroupResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); +void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey); +void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, SSHashObj* pMapDelete); +int32_t getAllSessionWindow(SSHashObj* pHashMap, SSHashObj* pStUpdated); +int32_t closeSessionWindow(SSHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SSHashObj* pClosed); +int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar); +int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2); +void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins); +int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, + int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, + struct SOperatorInfo* pOperator, int64_t winDelta); +int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo); +int32_t saveSessionOutputBuf(SStreamAggSupporter* pAggSup, SResultWindowInfo* pWinInfo); +int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated); +void saveDeleteRes(SSHashObj* pStDelete, SSessionKey key); +void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey); +void doBuildDeleteDataBlock(struct SOperatorInfo* pOp, SSHashObj* pStDeleted, SSDataBlock* pBlock, void** Ite); +void doBuildSessionResult(struct SOperatorInfo* pOperator, void* pState, SGroupResInfo* pGroupResInfo, SSDataBlock* pBlock); +void getSessionWindowInfoByKey(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SResultWindowInfo* pWinInfo); +void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, + SResultWindowInfo* pNextWin); +void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, + SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap); +int32_t releaseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI); +void resetWinRange(STimeWindow* winRange); + +int32_t encodeSSessionKey(void** buf, SSessionKey* key); +void* decodeSSessionKey(void* buf, SSessionKey* key); +int32_t encodeSResultWindowInfo(void** buf, SResultWindowInfo* key, int32_t outLen); +void* decodeSResultWindowInfo(void* buf, SResultWindowInfo* key, int32_t outLen); +int32_t encodeSTimeWindowAggSupp(void** buf, STimeWindowAggSupp* pTwAggSup); +void* decodeSTimeWindowAggSupp(void* buf, STimeWindowAggSupp* pTwAggSup); -int32_t encodeSTimeWindowAggSupp(void** buf, STimeWindowAggSupp* pTwAggSup); -void* decodeSTimeWindowAggSupp(void* buf, STimeWindowAggSupp* pTwAggSup); void destroyOperatorParamValue(void* pValues); int32_t mergeOperatorParams(SOperatorParam* pDst, SOperatorParam* pSrc); int32_t buildTableScanOperatorParam(SOperatorParam** ppRes, SArray* pUidList, int32_t srcOpType, bool tableSeq); diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index 13da9f7238..0858a7e595 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -152,6 +152,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle); + SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* physiNode, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/operator.c b/source/libs/executor/src/operator.c index 6f9aac7595..69a8acb3d7 100644 --- a/source/libs/executor/src/operator.c +++ b/source/libs/executor/src/operator.c @@ -519,6 +519,8 @@ SOperatorInfo* createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SR pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE == type) { pOptr = createStreamStateAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT == type) { + pOptr = createStreamEventAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, pHandle); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN == type) { pOptr = createMergeJoinOperatorInfo(ops, size, (SSortMergeJoinPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN == type) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c47e14ad0d..1c3db48972 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1127,7 +1127,8 @@ static bool isSessionWindow(SStreamScanInfo* pInfo) { } static bool isStateWindow(SStreamScanInfo* pInfo) { - return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE || + pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT; } static bool isIntervalWindow(SStreamScanInfo* pInfo) { @@ -2243,6 +2244,7 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pSup->pScanBlock); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); pInfo->pUpdateRes->info.type = STREAM_DELETE_DATA; + printSpecDataBlock(pInfo->pUpdateRes, getStreamOpName(pOperator->operatorType), "rebuild", GET_TASKID(pTaskInfo)); return pInfo->pUpdateRes; } diff --git a/source/libs/executor/src/streameventwindowoperator.c b/source/libs/executor/src/streameventwindowoperator.c new file mode 100644 index 0000000000..9b987ff1a4 --- /dev/null +++ b/source/libs/executor/src/streameventwindowoperator.c @@ -0,0 +1,751 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include "executorInt.h" +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "operator.h" +#include "querytask.h" +#include "tchecksum.h" +#include "tcommon.h" +#include "tcompare.h" +#include "tdatablock.h" +#include "tfill.h" +#include "tglobal.h" +#include "tlog.h" +#include "ttime.h" + +#define IS_FINAL_EVENT_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_EVENT) +#define STREAM_EVENT_OP_STATE_NAME "StreamEventHistoryState" +#define STREAM_EVENT_OP_CHECKPOINT_NAME "StreamEventOperator_Checkpoint" + +typedef struct SEventWinfowFlag { + bool startFlag; + bool endFlag; +} SEventWinfowFlag; + +typedef struct SEventWindowInfo { + SResultWindowInfo winInfo; + SEventWinfowFlag* pWinFlag; +} SEventWindowInfo; + +void destroyStreamEventOperatorInfo(void* param) { + SStreamEventAggOperatorInfo* pInfo = (SStreamEventAggOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + destroyStreamAggSupporter(&pInfo->streamAggSup); + clearGroupResInfo(&pInfo->groupResInfo); + cleanupExprSupp(&pInfo->scalarSupp); + if (pInfo->pChildren != NULL) { + int32_t size = taosArrayGetSize(pInfo->pChildren); + for (int32_t i = 0; i < size; i++) { + SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, i); + destroyOperator(pChild); + } + taosArrayDestroy(pInfo->pChildren); + } + colDataDestroy(&pInfo->twAggSup.timeWindowData); + blockDataDestroy(pInfo->pDelRes); + tSimpleHashCleanup(pInfo->pSeUpdated); + tSimpleHashCleanup(pInfo->pAllUpdated); + tSimpleHashCleanup(pInfo->pSeDeleted); + pInfo->pUpdated = taosArrayDestroy(pInfo->pUpdated); + cleanupGroupResInfo(&pInfo->groupResInfo); + + taosArrayDestroy(pInfo->historyWins); + blockDataDestroy(pInfo->pCheckpointRes); + + if (pInfo->pStartCondInfo != NULL) { + filterFreeInfo(pInfo->pStartCondInfo); + pInfo->pStartCondInfo = NULL; + } + + if (pInfo->pEndCondInfo != NULL) { + filterFreeInfo(pInfo->pEndCondInfo); + pInfo->pEndCondInfo = NULL; + } + + taosMemoryFreeClear(param); +} + +void setEventWindowFlag(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo) { + char* pFlagInfo = (char*)pWinInfo->winInfo.pStatePos->pRowBuff + (pAggSup->resultRowSize - pAggSup->stateKeySize); + pWinInfo->pWinFlag = (SEventWinfowFlag*) pFlagInfo; +} + +void setEventWindowInfo(SStreamAggSupporter* pAggSup, SSessionKey* pKey, SRowBuffPos* pPos, SEventWindowInfo* pWinInfo) { + pWinInfo->winInfo.sessionWin = *pKey; + pWinInfo->winInfo.pStatePos = pPos; + setEventWindowFlag(pAggSup, pWinInfo); +} + +int32_t getEndCondIndex(bool* pEnd, int32_t start, int32_t rows) { + for (int32_t i = start; i < rows; i++) { + if (pEnd[i]) { + return i; + } + } + return -1; +} + +void setEventOutputBuf(SStreamAggSupporter* pAggSup, TSKEY* pTs, uint64_t groupId, bool* pStart, bool* pEnd, int32_t index, int32_t rows, SEventWindowInfo* pCurWin, SSessionKey* pNextWinKey) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t size = pAggSup->resultRowSize; + TSKEY ts = pTs [index]; + bool start = pStart[index]; + bool end = pEnd[index]; + pCurWin->winInfo.sessionWin.groupId = groupId; + pCurWin->winInfo.sessionWin.win.skey = ts; + pCurWin->winInfo.sessionWin.win.ekey = ts; + SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyCurrentPrev(pAggSup->pState, &pCurWin->winInfo.sessionWin); + SSessionKey leftWinKey = {.groupId = groupId}; + void* pVal = NULL; + int32_t len = 0; + code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &leftWinKey, &pVal, &len); + if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &leftWinKey.win) ) { + bool inWin = isInTimeWindow(&leftWinKey.win, ts, 0); + setEventWindowInfo(pAggSup, &leftWinKey, pVal, pCurWin); + if(inWin || (pCurWin->pWinFlag->startFlag && !pCurWin->pWinFlag->endFlag) ) { + pCurWin->winInfo.isOutput = true; + goto _end; + } + } + pAggSup->stateStore.streamStateFreeCur(pCur); + pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->winInfo.sessionWin); + SSessionKey rightWinKey = {.groupId = groupId}; + code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, &rightWinKey, &pVal, &len); + bool inWin = isInTimeWindow(&rightWinKey.win, ts, 0); + if (code == TSDB_CODE_SUCCESS && inWinRange(&pAggSup->winRange, &rightWinKey.win) && (inWin || (start && !end))) { + int32_t endi = getEndCondIndex(pEnd, index, rows); + if (endi < 0 || pTs[endi] >= rightWinKey.win.skey) { + setEventWindowInfo(pAggSup, &rightWinKey, pVal, pCurWin); + pCurWin->winInfo.isOutput = true; + goto _end; + } + } + + SSessionKey winKey = {.win.skey = ts, .win.ekey = ts, .groupId = groupId}; + pAggSup->stateStore.streamStateSessionAllocWinBuffByNextPosition(pAggSup->pState, pCur, &winKey, &pVal, &len); + setEventWindowInfo(pAggSup, &winKey, pVal, pCurWin); + pCurWin->pWinFlag->startFlag = start; + pCurWin->pWinFlag->endFlag = end; + pCurWin->winInfo.isOutput = false; + +_end: + pAggSup->stateStore.streamStateCurNext(pAggSup->pState, pCur); + pNextWinKey->groupId = groupId; + code = pAggSup->stateStore.streamStateSessionGetKVByCur(pCur, pNextWinKey, NULL, 0); + if (code != TSDB_CODE_SUCCESS) { + SET_SESSION_WIN_KEY_INVALID(pNextWinKey); + } + if(pCurWin->winInfo.pStatePos->needFree) { + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, &pCurWin->winInfo.sessionWin); + } + pAggSup->stateStore.streamStateFreeCur(pCur); + qDebug("===stream===set event next win buff. skey:%" PRId64 ", endkey:%" PRId64, pCurWin->winInfo.sessionWin.win.skey, + pCurWin->winInfo.sessionWin.win.ekey); +} + +int32_t updateEventWindowInfo(SStreamAggSupporter* pAggSup, SEventWindowInfo* pWinInfo, SSessionKey* pNextWinKey, + TSKEY* pTsData, bool* starts, bool* ends, int32_t rows, int32_t start, SSHashObj* pResultRows, + SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool* pRebuild) { + *pRebuild = false; + if (!pWinInfo->pWinFlag->startFlag && !(starts[start]) ) { + return 1; + } + + TSKEY maxTs = INT64_MAX; + STimeWindow* pWin = &pWinInfo->winInfo.sessionWin.win; + if (pWinInfo->pWinFlag->endFlag) { + maxTs = pWin->ekey + 1; + } + if (!IS_INVALID_SESSION_WIN_KEY(*pNextWinKey)) { + maxTs = TMIN(maxTs, pNextWinKey->win.skey); + } + + for (int32_t i = start; i < rows; ++i) { + if (pTsData[i] >= maxTs) { + return i - 1 - start; + } + + if (pWin->skey > pTsData[i]) { + if (pStDeleted && pWinInfo->winInfo.isOutput) { + saveDeleteRes(pStDeleted, pWinInfo->winInfo.sessionWin); + } + removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); + pWin->skey = pTsData[i]; + pWinInfo->pWinFlag->startFlag = starts[i]; + } else if (pWin->skey == pTsData[i]) { + pWinInfo->pWinFlag->startFlag |= starts[i]; + } + + if (pWin->ekey < pTsData[i]) { + pWin->ekey = pTsData[i]; + pWinInfo->pWinFlag->endFlag = ends[i]; + } else if (pWin->ekey == pTsData[i]) { + pWinInfo->pWinFlag->endFlag |= ends[i]; + } + + memcpy(pWinInfo->winInfo.pStatePos->pKey, &pWinInfo->winInfo.sessionWin, sizeof(SSessionKey)); + + if (ends[i]) { + if (pWinInfo->pWinFlag->endFlag && pWin->skey <= pTsData[i] && pTsData[i] < pWin->ekey ) { + *pRebuild = true; + } + return i + 1 - start; + } + } + return rows - start; +} + +static int32_t compactEventWindow(SOperatorInfo* pOperator, SEventWindowInfo* pCurWin, SSHashObj* pStUpdated, + SSHashObj* pStDeleted, bool addGap) { + SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + int32_t winNum = 0; + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + while (1) { + if (!pCurWin->pWinFlag->startFlag || pCurWin->pWinFlag->endFlag) { + break; + } + SEventWindowInfo nextWinInfo = {0}; + getNextSessionWinInfo(pAggSup, pStUpdated, &pCurWin->winInfo, &nextWinInfo.winInfo); + if (!IS_VALID_SESSION_WIN(nextWinInfo.winInfo) || !inWinRange(&pAggSup->winRange, &nextWinInfo.winInfo.sessionWin.win)) { + releaseOutputBuf(pAggSup->pState, nextWinInfo.winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); + break; + } + setEventWindowFlag(pAggSup, &nextWinInfo); + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, &pCurWin->winInfo, &nextWinInfo.winInfo, pStUpdated, pStDeleted, false); + pCurWin->pWinFlag->endFlag = nextWinInfo.pWinFlag->endFlag; + winNum++; + } + return winNum; +} + +static bool isWindowIncomplete(SEventWindowInfo* pWinInfo) { + return !(pWinInfo->pWinFlag->startFlag && pWinInfo->pWinFlag->endFlag); +} + +bool doDeleteEventWindow(SStreamAggSupporter* pAggSup, SSHashObj* pSeUpdated, SSessionKey* pKey) { + pAggSup->stateStore.streamStateSessionDel(pAggSup->pState, pKey); + removeSessionResult(pAggSup, pSeUpdated, pAggSup->pResultRows, pKey); + return true; +} + +static void doStreamEventAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SSHashObj* pSeUpdated, + SSHashObj* pStDeleted) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + int32_t numOfOutput = pOperator->exprSupp.numOfExprs; + uint64_t groupId = pSDataBlock->info.id.groupId; + int64_t code = TSDB_CODE_SUCCESS; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t winRows = 0; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + SColumnInfoData* pColStart = NULL; + SColumnInfoData* pColEnd = NULL; + + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + pAggSup->winRange = pTaskInfo->streamInfo.fillHistoryWindow; + if (pAggSup->winRange.ekey <= 0) { + pAggSup->winRange.ekey = INT64_MAX; + } + + if (pSDataBlock->pDataBlock != NULL) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + } else { + return; + } + + SFilterColumnParam paramStart = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + code = filterSetDataFromSlotId(pInfo->pStartCondInfo, ¶mStart); + if (code != TSDB_CODE_SUCCESS) { + qError("set data from start slotId error."); + goto _end; + } + int32_t statusStart = 0; + filterExecute(pInfo->pStartCondInfo, pSDataBlock, &pColStart, NULL, paramStart.numOfCols, &statusStart); + + SFilterColumnParam paramEnd = {.numOfCols = taosArrayGetSize(pSDataBlock->pDataBlock), .pDataBlock = pSDataBlock->pDataBlock}; + code = filterSetDataFromSlotId(pInfo->pEndCondInfo, ¶mEnd); + if (code != TSDB_CODE_SUCCESS) { + qError("set data from end slotId error."); + goto _end; + } + + int32_t statusEnd = 0; + filterExecute(pInfo->pEndCondInfo, pSDataBlock, &pColEnd, NULL, paramEnd.numOfCols, &statusEnd); + + int32_t rows = pSDataBlock->info.rows; + blockDataEnsureCapacity(pAggSup->pScanBlock, rows); + for (int32_t i = 0; i < rows; i += winRows) { + if (pInfo->ignoreExpiredData && isOverdue(tsCols[i], &pInfo->twAggSup)) { + i++; + continue; + } + int32_t winIndex = 0; + bool allEqual = true; + SEventWindowInfo curWin = {0}; + SSessionKey nextWinKey = {0}; + setEventOutputBuf(pAggSup, tsCols, groupId, (bool*)pColStart->pData, (bool*)pColEnd->pData, i, rows, &curWin, &nextWinKey); + setSessionWinOutputInfo(pSeUpdated, &curWin.winInfo); + bool rebuild = false; + winRows = updateEventWindowInfo(pAggSup, &curWin, &nextWinKey, tsCols, (bool*)pColStart->pData, (bool*)pColEnd->pData, rows, i, + pAggSup->pResultRows, pSeUpdated, pStDeleted, &rebuild); + ASSERT(winRows >= 1); + if (rebuild) { + uint64_t uid = 0; + appendOneRowToStreamSpecialBlock(pAggSup->pScanBlock, &curWin.winInfo.sessionWin.win.skey, + &curWin.winInfo.sessionWin.win.ekey, &uid, &groupId, NULL); + tSimpleHashRemove(pSeUpdated, &curWin.winInfo.sessionWin, sizeof(SSessionKey)); + doDeleteEventWindow(pAggSup, pSeUpdated, &curWin.winInfo.sessionWin); + releaseOutputBuf(pAggSup->pState, curWin.winInfo.pStatePos, &pAPI->stateStore); + continue; + } + code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, + pOperator, 0); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); + } + compactEventWindow(pOperator, &curWin, pInfo->pSeUpdated, pInfo->pSeDeleted, false); + saveSessionOutputBuf(pAggSup, &curWin.winInfo); + + if (pInfo->isHistoryOp) { + saveResult(curWin.winInfo, pInfo->pAllUpdated); + } + + if (isWindowIncomplete(&curWin)) { + continue; + } + + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + code = saveResult(curWin.winInfo, pSeUpdated); + } + + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + SSessionKey key = {0}; + getSessionHashKey(&curWin.winInfo.sessionWin, &key); + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curWin.winInfo, sizeof(SResultWindowInfo)); + } + } + +_end: + colDataDestroy(pColStart); + taosMemoryFree(pColStart); + colDataDestroy(pColEnd); + taosMemoryFree(pColEnd); +} + +int32_t doStreamEventEncodeOpState(void** buf, int32_t len, SOperatorInfo* pOperator) { + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + if (!pInfo) { + return 0; + } + + void* pData = (buf == NULL) ? NULL : *buf; + + // 1.streamAggSup.pResultRows + int32_t tlen = 0; + int32_t mapSize = tSimpleHashGetSize(pInfo->streamAggSup.pResultRows); + tlen += taosEncodeFixedI32(buf, mapSize); + void* pIte = NULL; + size_t keyLen = 0; + int32_t iter = 0; + while ((pIte = tSimpleHashIterate(pInfo->streamAggSup.pResultRows, pIte, &iter)) != NULL) { + void* key = tSimpleHashGetKey(pIte, &keyLen); + tlen += encodeSSessionKey(buf, key); + tlen += encodeSResultWindowInfo(buf, pIte, pInfo->streamAggSup.resultRowSize); + } + + // 2.twAggSup + tlen += encodeSTimeWindowAggSupp(buf, &pInfo->twAggSup); + + // 3.dataVersion + tlen += taosEncodeFixedI32(buf, pInfo->dataVersion); + + // 4.checksum + if (buf) { + uint32_t cksum = taosCalcChecksum(0, pData, len - sizeof(uint32_t)); + tlen += taosEncodeFixedU32(buf, cksum); + } else { + tlen += sizeof(uint32_t); + } + + return tlen; +} + +void* doStreamEventDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperator) { + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + if (!pInfo) { + return buf; + } + + // 4.checksum + int32_t dataLen = len - sizeof(uint32_t); + void* pCksum = POINTER_SHIFT(buf, dataLen); + if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { + ASSERT(0); // debug + qError("stream event state is invalid"); + return buf; + } + + // 1.streamAggSup.pResultRows + int32_t mapSize = 0; + buf = taosDecodeFixedI32(buf, &mapSize); + for (int32_t i = 0; i < mapSize; i++) { + SSessionKey key = {0}; + SResultWindowInfo winfo = {0}; + buf = decodeSSessionKey(buf, &key); + buf = decodeSResultWindowInfo(buf, &winfo, pInfo->streamAggSup.resultRowSize); + tSimpleHashPut(pInfo->streamAggSup.pResultRows, &key, sizeof(SSessionKey), &winfo, sizeof(SResultWindowInfo)); + } + + // 2.twAggSup + buf = decodeSTimeWindowAggSupp(buf, &pInfo->twAggSup); + + // 3.dataVersion + buf = taosDecodeFixedI64(buf, &pInfo->dataVersion); + return buf; +} + +void doStreamEventSaveCheckpoint(SOperatorInfo* pOperator) { + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + int32_t len = doStreamEventEncodeOpState(NULL, 0, pOperator); + void* buf = taosMemoryCalloc(1, len); + void* pBuf = buf; + len = doStreamEventEncodeOpState(&pBuf, len, pOperator); + pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, + strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), buf, len); + taosMemoryFree(buf); +} + +static SSDataBlock* buildEventResult(SOperatorInfo* pOperator) { + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + doBuildDeleteDataBlock(pOperator, pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pDelRes; + } + + doBuildSessionResult(pOperator, pInfo->streamAggSup.pState, &pInfo->groupResInfo, pBInfo->pRes); + if (pBInfo->pRes->info.rows > 0) { + printDataBlock(pBInfo->pRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pBInfo->pRes; + } + return NULL; +} + +static SSDataBlock* doStreamEventAgg(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExprSupp* pSup = &pOperator->exprSupp; + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + qDebug("===stream=== stream event agg"); + if (pOperator->status == OP_RES_TO_RETURN) { + SSDataBlock* resBlock = buildEventResult(pOperator); + if (resBlock != NULL) { + return resBlock; + } + + if (pInfo->reCkBlock) { + pInfo->reCkBlock = false; + printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pCheckpointRes; + } + + setOperatorCompleted(pOperator); + return NULL; + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (!pInfo->pUpdated) { + pInfo->pUpdated = taosArrayInit(16, sizeof(SEventWindowInfo)); + } + if (!pInfo->pSeUpdated) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeUpdated = tSimpleHashInit(64, hashFn); + } + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + break; + } + printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "recv", GET_TASKID(pTaskInfo)); + + if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || + pBlock->info.type == STREAM_CLEAR) { + deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted); + continue; + } else if (pBlock->info.type == STREAM_GET_ALL) { + getAllSessionWindow(pInfo->streamAggSup.pResultRows, pInfo->pSeUpdated); + continue; + } else if (pBlock->info.type == STREAM_CREATE_CHILD_TABLE) { + return pBlock; + } else if (pBlock->info.type == STREAM_CHECKPOINT) { + pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); + doStreamEventSaveCheckpoint(pOperator); + pInfo->reCkBlock = true; + copyDataBlock(pInfo->pCheckpointRes, pBlock); + continue; + } else { + ASSERTS(pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_INVALID, "invalid SSDataBlock type"); + } + + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pSup, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + doStreamEventAggImpl(pOperator, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + } + // restore the value + pOperator->status = OP_RES_TO_RETURN; + + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pInfo->pSeUpdated); + copyUpdateResult(&pInfo->pSeUpdated, pInfo->pUpdated, sessionKeyCompareAsc); + removeSessionDeleteResults(pInfo->pSeDeleted, pInfo->pUpdated); + + if (pInfo->isHistoryOp) { + SArray* pHisWins = taosArrayInit(16, sizeof(SEventWindowInfo)); + copyUpdateResult(&pInfo->pAllUpdated, pHisWins, sessionKeyCompareAsc); + getMaxTsWins(pHisWins, pInfo->historyWins); + taosArrayDestroy(pHisWins); + } + + initGroupResInfoFromArrayList(&pInfo->groupResInfo, pInfo->pUpdated); + pInfo->pUpdated = NULL; + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + + SSDataBlock* resBlock = buildEventResult(pOperator); + if (resBlock != NULL) { + return resBlock; + } + setOperatorCompleted(pOperator); + return NULL; +} + +void streamEventReleaseState(SOperatorInfo* pOperator) { + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + int32_t winSize = taosArrayGetSize(pInfo->historyWins) * sizeof(SSessionKey); + int32_t resSize = winSize + sizeof(TSKEY); + char* pBuff = taosMemoryCalloc(1, resSize); + memcpy(pBuff, pInfo->historyWins->pData, winSize); + memcpy(pBuff + winSize, &pInfo->twAggSup.maxTs, sizeof(TSKEY)); + qDebug("===stream=== event window operator relase state. save result count:%d", (int32_t)taosArrayGetSize(pInfo->historyWins)); + pInfo->streamAggSup.stateStore.streamStateSaveInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_STATE_NAME, + strlen(STREAM_EVENT_OP_STATE_NAME), pBuff, resSize); + pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); + taosMemoryFreeClear(pBuff); + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.releaseStreamStateFn) { + downstream->fpSet.releaseStreamStateFn(downstream); + } +} + +void streamEventReloadState(SOperatorInfo* pOperator) { + SStreamEventAggOperatorInfo* pInfo = pOperator->info; + SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; + resetWinRange(&pAggSup->winRange); + + SSessionKey seKey = {.win.skey = INT64_MIN, .win.ekey = INT64_MIN, .groupId = 0}; + int32_t size = 0; + void* pBuf = NULL; + int32_t code = pAggSup->stateStore.streamStateGetInfo(pAggSup->pState, STREAM_EVENT_OP_STATE_NAME, + strlen(STREAM_EVENT_OP_STATE_NAME), &pBuf, &size); + int32_t num = (size - sizeof(TSKEY)) / sizeof(SSessionKey); + qDebug("===stream=== event window operator reload state. get result count:%d", num); + SSessionKey* pSeKeyBuf = (SSessionKey*)pBuf; + ASSERT(size == num * sizeof(SSessionKey) + sizeof(TSKEY)); + + TSKEY ts = *(TSKEY*)((char*)pBuf + size - sizeof(TSKEY)); + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, ts); + pAggSup->stateStore.streamStateReloadInfo(pAggSup->pState, ts); + + if (!pInfo->pSeUpdated && num > 0) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeUpdated = tSimpleHashInit(64, hashFn); + } + if (!pInfo->pSeDeleted && num > 0) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); + } + for (int32_t i = 0; i < num; i++) { + SEventWindowInfo curInfo = {0}; + qDebug("===stream=== reload state. try process result %" PRId64 ", %" PRIu64 ", index:%d", pSeKeyBuf[i].win.skey, + pSeKeyBuf[i].groupId, i); + getSessionWindowInfoByKey(pAggSup, pSeKeyBuf + i, &curInfo.winInfo); + setEventWindowFlag(pAggSup, &curInfo); + if (!curInfo.pWinFlag->startFlag || curInfo.pWinFlag->endFlag) { + continue; + } + + compactEventWindow(pOperator, &curInfo, pInfo->pSeUpdated, pInfo->pSeDeleted, false); + qDebug("===stream=== reload state. save result %" PRId64 ", %" PRIu64, curInfo.winInfo.sessionWin.win.skey, + curInfo.winInfo.sessionWin.groupId); + + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveResult(curInfo.winInfo, pInfo->pSeUpdated); + } else if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + if (!isCloseWindow(&curInfo.winInfo.sessionWin.win, &pInfo->twAggSup)) { + saveDeleteRes(pInfo->pSeDeleted, curInfo.winInfo.sessionWin); + } + SSessionKey key = {0}; + getSessionHashKey(&curInfo.winInfo.sessionWin, &key); + tSimpleHashPut(pAggSup->pResultRows, &key, sizeof(SSessionKey), &curInfo.winInfo, sizeof(SResultWindowInfo)); + } + + if (IS_VALID_SESSION_WIN(curInfo.winInfo)) { + saveSessionOutputBuf(pAggSup, &curInfo.winInfo); + } + } + taosMemoryFree(pBuf); + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + if (downstream->fpSet.reloadStreamStateFn) { + downstream->fpSet.reloadStreamStateFn(downstream); + } +} + +SOperatorInfo* createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, SReadHandle* pHandle) { + SStreamEventWinodwPhysiNode* pEventNode = (SStreamEventWinodwPhysiNode*)pPhyNode; + int32_t tsSlotId = ((SColumnNode*)pEventNode->window.pTspk)->slotId; + int32_t code = TSDB_CODE_SUCCESS; + SStreamEventAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamEventAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + + initResultSizeInfo(&pOperator->resultInfo, 4096); + if (pEventNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pEventNode->window.pExprs, NULL, &numOfScalar); + code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pEventNode->window.watermark, + .calTrigger = pEventNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + }; + + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + SExprSupp* pExpSup = &pOperator->exprSupp; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pEventNode->window.pFuncs, NULL, &numOfCols); + SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); + code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = initStreamAggSupporter(&pInfo->streamAggSup, pExpSup, numOfCols, 0, pTaskInfo->streamInfo.pState, + sizeof(bool) + sizeof(bool), 0, &pTaskInfo->storageAPI.stateStore, pHandle, + &pInfo->twAggSup, GET_TASKID(pTaskInfo), &pTaskInfo->storageAPI); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->primaryTsIndex = tsSlotId; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pSeDeleted = tSimpleHashInit(64, hashFn); + pInfo->pDelIterator = NULL; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + pInfo->pChildren = NULL; + pInfo->ignoreExpiredData = pEventNode->window.igExpired; + pInfo->ignoreExpiredDataSaved = false; + pInfo->pUpdated = NULL; + pInfo->pSeUpdated = NULL; + pInfo->dataVersion = 0; + pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey)); + if (!pInfo->historyWins) { + goto _error; + } + if (pHandle) { + pInfo->isHistoryOp = pHandle->fillHistory; + } + + if (pInfo->isHistoryOp) { + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pAllUpdated = tSimpleHashInit(64, hashFn); + } else { + pInfo->pAllUpdated = NULL; + } + + pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT); + pInfo->reCkBlock = false; + + // for stream + void* buff = NULL; + int32_t len = 0; + int32_t res = + pInfo->streamAggSup.stateStore.streamStateGetInfo(pInfo->streamAggSup.pState, STREAM_EVENT_OP_CHECKPOINT_NAME, + strlen(STREAM_EVENT_OP_CHECKPOINT_NAME), &buff, &len); + if (res == TSDB_CODE_SUCCESS) { + doStreamEventDecodeOpState(buff, len, pOperator); + taosMemoryFree(buff); + } + + setOperatorInfo(pOperator, "StreamEventAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, true, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->fpSet = createOperatorFpSet(optrDummyOpenFn, doStreamEventAgg, NULL, destroyStreamEventOperatorInfo, + optrDefaultBufFn, NULL, optrDefaultGetNextExtFn, NULL); + setOperatorStreamStateFn(pOperator, streamEventReleaseState, streamEventReloadState); + initDownStream(downstream, &pInfo->streamAggSup, pOperator->operatorType, pInfo->primaryTsIndex, &pInfo->twAggSup); + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + code = filterInitFromNode((SNode*)pEventNode->pStartCond, &pInfo->pStartCondInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + code = filterInitFromNode((SNode*)pEventNode->pEndCond, &pInfo->pEndCondInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyStreamEventOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/source/libs/executor/src/streamtimewindowoperator.c b/source/libs/executor/src/streamtimewindowoperator.c index 0d82fcc9c3..86d83e05ad 100644 --- a/source/libs/executor/src/streamtimewindowoperator.c +++ b/source/libs/executor/src/streamtimewindowoperator.c @@ -29,7 +29,7 @@ #define IS_FINAL_INTERVAL_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) #define IS_FINAL_SESSION_OP(op) ((op)->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION) -#define DEAULT_DELETE_MARK (1000LL * 60LL * 60LL * 24LL * 365LL * 10LL); +#define DEAULT_DELETE_MARK INT64_MAX #define STREAM_INTERVAL_OP_STATE_NAME "StreamIntervalHistoryState" #define STREAM_SESSION_OP_STATE_NAME "StreamSessionHistoryState" #define STREAM_STATE_OP_STATE_NAME "StreamStateHistoryState" @@ -146,7 +146,7 @@ static int32_t savePullWindow(SPullWindowInfo* pPullInfo, SArray* pPullWins) { return TSDB_CODE_SUCCESS; } -static int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) { +int32_t saveResult(SResultWindowInfo winInfo, SSHashObj* pStUpdated) { winInfo.sessionWin.win.ekey = winInfo.sessionWin.win.skey; return tSimpleHashPut(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey), &winInfo, sizeof(SResultWindowInfo)); } @@ -279,7 +279,12 @@ static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SSHashObj* resWins) { SWinKey* pKey = tSimpleHashGetKey(pIte, NULL); uint64_t groupId = pKey->groupId; TSKEY ts = pKey->ts; - int32_t code = saveWinResultInfo(ts, groupId, *(SRowBuffPos**)pIte, resWins); + SRowBuffPos* pPos = *(SRowBuffPos**)pIte; + if (!pPos->beUpdated) { + continue; + } + pPos->beUpdated = false; + int32_t code = saveWinResultInfo(ts, groupId, pPos, resWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -848,6 +853,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat pSup->rowEntryInfoOffset, &pInfo->aggSup, &pInfo->stateStore); pResult = (SResultRow*)pResPos->pRowBuff; if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + qError("%s set interval output buff error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } if (IS_FINAL_INTERVAL_OP(pOperator)) { @@ -866,6 +872,7 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDat } if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { + pResPos->beUpdated = true; tSimpleHashPut(pInfo->aggSup.pResultRowHashTable, &key, sizeof(SWinKey), &pResPos, POINTER_BYTES); } @@ -1076,7 +1083,6 @@ void doStreamIntervalDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - ASSERT(0); // debug qError("stream interval state is invalid"); return; } @@ -1159,7 +1165,7 @@ static SSDataBlock* buildIntervalResult(SOperatorInfo* pOperator) { return NULL; } -static int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) { +int32_t copyUpdateResult(SSHashObj** ppWinUpdated, SArray* pUpdated, __compar_fn_t compar) { void* pIte = NULL; int32_t iter = 0; while ((pIte = tSimpleHashIterate(*ppWinUpdated, pIte, &iter)) != NULL) { @@ -1237,7 +1243,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { tSimpleHashCleanup(pInfo->pUpdatedMap); pInfo->pUpdatedMap = NULL; } - + qInfo("%s task is killed, code %s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -1768,8 +1774,9 @@ int32_t reuseOutputBuf(void* pState, SRowBuffPos* pPos, SStateStore* pAPI) { return TSDB_CODE_SUCCESS; } -static void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey key) { - key.win.ekey = key.win.skey; +void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SSHashObj* pResMap, SSessionKey* pKey) { + SSessionKey key = {0}; + getSessionHashKey(pKey, &key); void* pVal = tSimpleHashGet(pHashMap, &key, sizeof(SSessionKey)); if (pVal) { releaseOutputBuf(pAggSup->pState, *(void**)pVal, &pAggSup->pSessionAPI->stateStore); @@ -1778,12 +1785,12 @@ static void removeSessionResult(SStreamAggSupporter* pAggSup, SSHashObj* pHashMa tSimpleHashRemove(pResMap, &key, sizeof(SSessionKey)); } -static void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) { +void getSessionHashKey(const SSessionKey* pKey, SSessionKey* pHashKey) { *pHashKey = *pKey; pHashKey->win.ekey = pKey->win.skey; } -static void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) { +void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) { if (tSimpleHashGetSize(pHashMap) == 0) { return; } @@ -1797,7 +1804,7 @@ static void removeSessionDeleteResults(SSHashObj* pHashMap, SArray* pWins) { } } -static void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins) { +void removeSessionResults(SStreamAggSupporter* pAggSup, SSHashObj* pHashMap, SArray* pWins) { if (tSimpleHashGetSize(pHashMap) == 0) { return; } @@ -1826,7 +1833,7 @@ int32_t updateSessionWindowInfo(SStreamAggSupporter* pAggSup, SResultWindowInfo* if (pStDeleted && pWinInfo->isOutput) { saveDeleteRes(pStDeleted, pWinInfo->sessionWin); } - removeSessionResult(pAggSup, pStUpdated, pResultRows, pWinInfo->sessionWin); + removeSessionResult(pAggSup, pStUpdated, pResultRows, &pWinInfo->sessionWin); pWinInfo->sessionWin.win.skey = pStartTs[i]; } pWinInfo->sessionWin.win.ekey = TMAX(pWinInfo->sessionWin.win.ekey, pStartTs[i]); @@ -1848,7 +1855,7 @@ static int32_t initSessionOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pR return TSDB_CODE_SUCCESS; } -static int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, +int32_t doOneWindowAggImpl(SColumnInfoData* pTimeWindowData, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, int32_t rows, int32_t numOutput, SOperatorInfo* pOperator, int64_t winDelta) { SExprSupp* pSup = &pOperator->exprSupp; @@ -1870,7 +1877,7 @@ static bool doDeleteSessionWindow(SStreamAggSupporter* pAggSup, SSessionKey* pKe return true; } -static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { +int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* pWinInfo) { void* pVal = tSimpleHashGet(pStUpdated, &pWinInfo->sessionWin, sizeof(SSessionKey)); if (pVal) { SResultWindowInfo* pWin = pVal; @@ -1880,7 +1887,7 @@ static int32_t setSessionWinOutputInfo(SSHashObj* pStUpdated, SResultWindowInfo* } void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, SResultWindowInfo* pCurWin, - SResultWindowInfo* pNextWin) { + SResultWindowInfo* pNextWin) { SStreamStateCur* pCur = pAggSup->stateStore.streamStateSessionSeekKeyNext(pAggSup->pState, &pCurWin->sessionWin); pNextWin->isOutput = true; setSessionWinOutputInfo(pStUpdated, pNextWin); @@ -1894,6 +1901,34 @@ void getNextSessionWinInfo(SStreamAggSupporter* pAggSup, SSHashObj* pStUpdated, pAggSup->stateStore.streamStateFreeCur(pCur); } +void compactTimeWindow(SExprSupp* pSup, SStreamAggSupporter* pAggSup, STimeWindowAggSupp* pTwAggSup, + SExecTaskInfo* pTaskInfo, SResultWindowInfo* pCurWin, SResultWindowInfo* pNextWin, + SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { + SResultRow* pCurResult = NULL; + int32_t numOfOutput = pSup->numOfExprs; + initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + SResultRow* pWinResult = NULL; + initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); + pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey); + memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey)); + + int64_t winDelta = 0; + if (addGap) { + winDelta = pAggSup->gap; + } + updateTimeWindowInfo(&pTwAggSup->timeWindowData, &pCurWin->sessionWin.win, winDelta); + compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pTwAggSup->timeWindowData); + tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); + if (pNextWin->isOutput && pStDeleted) { + qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey, + pNextWin->sessionWin.groupId); + saveDeleteRes(pStDeleted, pNextWin->sessionWin); + } + removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, &pNextWin->sessionWin); + doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin); + releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore); +} + static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCurWin, SSHashObj* pStUpdated, SSHashObj* pStDeleted, bool addGap) { SExprSupp* pSup = &pOperator->exprSupp; @@ -1905,7 +1940,7 @@ static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* SResultRow* pCurResult = NULL; int32_t numOfOutput = pOperator->exprSupp.numOfExprs; SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); + // initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); // Just look for the window behind StartIndex while (1) { SResultWindowInfo winInfo = {0}; @@ -1915,23 +1950,7 @@ static int32_t compactSessionWindow(SOperatorInfo* pOperator, SResultWindowInfo* releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); break; } - SResultRow* pWinResult = NULL; - initSessionOutputBuf(&winInfo, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); - pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, winInfo.sessionWin.win.ekey); - memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey)); - int64_t winDelta = 0; - if (addGap) { - winDelta = pAggSup->gap; - } - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, winDelta); - compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - tSimpleHashRemove(pStUpdated, &winInfo.sessionWin, sizeof(SSessionKey)); - if (winInfo.isOutput && pStDeleted) { - saveDeleteRes(pStDeleted, winInfo.sessionWin); - } - removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, winInfo.sessionWin); - doDeleteSessionWindow(pAggSup, &winInfo.sessionWin); - releaseOutputBuf(pAggSup->pState, winInfo.pStatePos, &pAggSup->pSessionAPI->stateStore); + compactTimeWindow(pSup, pAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, &winInfo, pStUpdated, pStDeleted, true); winNum++; } return winNum; @@ -2017,6 +2036,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, winDelta); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + qError("%s do stream session aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } compactSessionWindow(pOperator, &winInfo, pStUpdated, pStDeleted, addGap); @@ -2025,6 +2045,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pStUpdated) { code = saveResult(winInfo, pStUpdated); if (code != TSDB_CODE_SUCCESS) { + qError("%s do stream session aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } @@ -2038,7 +2059,7 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } -static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { +void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* result) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* startDatas = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); @@ -2060,7 +2081,7 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc } } -static inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { +inline int32_t sessionKeyCompareAsc(const void* pKey1, const void* pKey2) { SResultWindowInfo* pWinInfo1 = (SResultWindowInfo*)pKey1; SResultWindowInfo* pWinInfo2 = (SResultWindowInfo*)pKey2; SSessionKey* pWin1 = &pWinInfo1->sessionWin; @@ -2490,8 +2511,7 @@ void* doStreamSessionDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOpera int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - ASSERT(0); // debug - qError("stream interval state is invalid"); + qError("stream session state is invalid"); return buf; } } @@ -2559,6 +2579,12 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { return opRes; } + if (pInfo->reCkBlock) { + pInfo->reCkBlock = false; + printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pCheckpointRes; + } + if (pInfo->recvGetAll) { pInfo->recvGetAll = false; resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); @@ -2631,6 +2657,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0, NULL); if (!pChildOp) { + qError("%s create stream child of final session error", GET_TASKID(pTaskInfo)); T_LONG_JMP(pOperator->pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } taosArrayPush(pInfo->pChildren, &pChildOp); @@ -2897,6 +2924,14 @@ static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { pInfo->streamAggSup.stateStore.streamStateSessionClear(pInfo->streamAggSup.pState); } +void deleteSessionWinState(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SSHashObj* pMapUpdate, SSHashObj* pMapDelete) { + SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); + doDeleteTimeWindows(pAggSup, pBlock, pWins); + removeSessionResults(pAggSup, pMapUpdate, pWins); + copyDeleteWindowInfo(pWins, pMapDelete); + taosArrayDestroy(pWins); +} + static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -2923,6 +2958,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } if (pOperator->status == OP_RES_TO_RETURN) { + if (pInfo->reCkBlock) { + pInfo->reCkBlock = false; + printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pCheckpointRes; + } clearFunctionContext(&pOperator->exprSupp); // semi session operator clear disk buffer clearStreamSessionOperator(pInfo); @@ -2951,11 +2991,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { // gap must be 0 - SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); - doDeleteTimeWindows(pAggSup, pBlock, pWins); - removeSessionResults(pAggSup, pInfo->pStUpdated, pWins); - copyDeleteWindowInfo(pWins, pInfo->pStDeleted); - taosArrayDestroy(pWins); + deleteSessionWinState(pAggSup, pBlock, pInfo->pStUpdated, pInfo->pStDeleted); pInfo->clearState = true; break; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -3248,7 +3284,7 @@ int32_t updateStateWindowInfo(SStreamAggSupporter* pAggSup, SStateWindowInfo* pW if (pSeDeleted && pWinInfo->winInfo.isOutput) { saveDeleteRes(pSeDeleted, pWinInfo->winInfo.sessionWin); } - removeSessionResult(pAggSup, pSeUpdated, pResultRows, pWinInfo->winInfo.sessionWin); + removeSessionResult(pAggSup, pSeUpdated, pResultRows, &pWinInfo->winInfo.sessionWin); pWinInfo->winInfo.sessionWin.win.skey = pTs[i]; } pWinInfo->winInfo.sessionWin.win.ekey = TMAX(pWinInfo->winInfo.sessionWin.win.ekey, pTs[i]); @@ -3318,6 +3354,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl code = doOneWindowAggImpl(&pInfo->twAggSup.timeWindowData, &curWin.winInfo, &pResult, i, winRows, rows, numOfOutput, pOperator, 0); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + qError("%s do one window aggregate impl error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } saveSessionOutputBuf(pAggSup, &curWin.winInfo); @@ -3325,6 +3362,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { code = saveResult(curWin.winInfo, pSeUpdated); if (code != TSDB_CODE_SUCCESS) { + qError("%s do stream state aggregate impl, set result error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); T_LONG_JMP(pTaskInfo->env, TSDB_CODE_OUT_OF_MEMORY); } } @@ -3396,8 +3434,7 @@ void* doStreamStateDecodeOpState(void* buf, int32_t len, SOperatorInfo* pOperato int32_t dataLen = len - sizeof(uint32_t); void* pCksum = POINTER_SHIFT(buf, dataLen); if (taosCheckChecksum(buf, dataLen, *(uint32_t*)pCksum) != TSDB_CODE_SUCCESS) { - ASSERT(0); // debug - qError("stream interval state is invalid"); + qError("stream state_window state is invalid"); return buf; } } @@ -3475,6 +3512,12 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return resBlock; } + if (pInfo->reCkBlock) { + pInfo->reCkBlock = false; + printDataBlock(pInfo->pCheckpointRes, getStreamOpName(pOperator->operatorType), GET_TASKID(pTaskInfo)); + return pInfo->pCheckpointRes; + } + if (pInfo->recvGetAll) { pInfo->recvGetAll = false; resetUnCloseSessionWinInfo(pInfo->streamAggSup.pResultRows); @@ -3501,11 +3544,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT || pBlock->info.type == STREAM_CLEAR) { - SArray* pWins = taosArrayInit(16, sizeof(SSessionKey)); - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, pWins); - removeSessionResults(&pInfo->streamAggSup, pInfo->pSeUpdated, pWins); - copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); - taosArrayDestroy(pWins); + deleteSessionWinState(&pInfo->streamAggSup, pBlock, pInfo->pSeUpdated, pInfo->pSeDeleted); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { pInfo->recvGetAll = true; @@ -3515,7 +3554,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return pBlock; } else if (pBlock->info.type == STREAM_CHECKPOINT) { pInfo->streamAggSup.stateStore.streamStateCommit(pInfo->streamAggSup.pState); - doStreamSessionSaveCheckpoint(pOperator); + doStreamStateSaveCheckpoint(pOperator); copyDataBlock(pInfo->pCheckpointRes, pBlock); continue; } else { @@ -3577,29 +3616,8 @@ static void compactStateWindow(SOperatorInfo* pOperator, SResultWindowInfo* pCur SSHashObj* pStUpdated, SSHashObj* pStDeleted) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SStorageAPI* pAPI = &pOperator->pTaskInfo->storageAPI; - SStreamStateAggOperatorInfo* pInfo = pOperator->info; - SResultRow* pCurResult = NULL; - int32_t numOfOutput = pOperator->exprSupp.numOfExprs; - SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; - initSessionOutputBuf(pCurWin, &pCurResult, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset); - SResultRow* pWinResult = NULL; - initSessionOutputBuf(pNextWin, &pWinResult, pAggSup->pDummyCtx, numOfOutput, pSup->rowEntryInfoOffset); - pCurWin->sessionWin.win.ekey = TMAX(pCurWin->sessionWin.win.ekey, pNextWin->sessionWin.win.ekey); - memcpy(pCurWin->pStatePos->pKey, &pCurWin->sessionWin, sizeof(SSessionKey)); - - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->sessionWin.win, 1); - compactFunctions(pSup->pCtx, pAggSup->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); - tSimpleHashRemove(pStUpdated, &pNextWin->sessionWin, sizeof(SSessionKey)); - if (pNextWin->isOutput && pStDeleted) { - qDebug("===stream=== save delete window info %" PRId64 ", %" PRIu64, pNextWin->sessionWin.win.skey, - pNextWin->sessionWin.groupId); - saveDeleteRes(pStDeleted, pNextWin->sessionWin); - } - removeSessionResult(pAggSup, pStUpdated, pAggSup->pResultRows, pNextWin->sessionWin); - doDeleteSessionWindow(pAggSup, &pNextWin->sessionWin); - releaseOutputBuf(pAggSup->pState, pNextWin->pStatePos, &pAggSup->pSessionAPI->stateStore); + compactTimeWindow(pSup, &pInfo->streamAggSup, &pInfo->twAggSup, pTaskInfo, pCurWin, pNextWin, pStUpdated, pStDeleted, false); } void streamStateReloadState(SOperatorInfo* pOperator) { @@ -3771,12 +3789,6 @@ _error: return NULL; } -static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) { - pRes->info.id.groupId = pMiaInfo->groupId; - pMiaInfo->curTs = INT64_MIN; - pMiaInfo->groupId = 0; -} - static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type) { for (int i = 0; i < num; i++) { if (type == STREAM_INVERT) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a55bd5663f..1a65a29259 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -7363,7 +7363,7 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; if (TSDB_DATA_TYPE_TIMESTAMP != ((SExprNode*)nodesListGetNode(pSelect->pProjectionList, 0))->resType.type || !isTimeLineQuery(pStmt->pQuery) || crossTableWithoutAggOper(pSelect) || NULL != pSelect->pOrderByList || - crossTableWithUdaf(pSelect) || isEventWindowQuery(pSelect) || hasJsonTypeProjection(pSelect)) { + crossTableWithUdaf(pSelect) || hasJsonTypeProjection(pSelect)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, "Unsupported stream query"); } if (NULL != pSelect->pSubtable && TSDB_DATA_TYPE_VARCHAR != ((SExprNode*)pSelect->pSubtable)->resType.type) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 4eda11a6a4..e71e18d37d 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -2784,11 +2784,44 @@ static int32_t splitCacheLastFuncOptCreateAggLogicNode(SAggLogicNode** pNewAgg, pNew->hasGroup = pAgg->hasGroup; pNew->node.pChildren = nodesCloneList(pAgg->node.pChildren); - SNode* pNode = NULL; - FOREACH(pNode, pNew->node.pChildren) { - if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { - OPTIMIZE_FLAG_CLEAR_MASK(((SScanLogicNode*)pNode)->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH); + int32_t code = 0; + SNode* pNode = nodesListGetNode(pNew->node.pChildren, 0); + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + SNodeList* pOldScanCols = NULL; + TSWAP(pScan->pScanCols, pOldScanCols); + nodesDestroyList(pScan->pScanPseudoCols); + pScan->pScanPseudoCols = NULL; + nodesDestroyList(pScan->node.pTargets); + pScan->node.pTargets = NULL; + SNodeListNode* list = (SNodeListNode*)nodesMakeNode(QUERY_NODE_NODE_LIST); + list->pNodeList = pFunc; + code = nodesCollectColumnsFromNode((SNode*)list, NULL, COLLECT_COL_TYPE_COL, &pScan->pScanCols); + if (TSDB_CODE_SUCCESS != code) { + return code; } + nodesFree(list); + bool found = false; + FOREACH(pNode, pScan->pScanCols) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { + found = true; + break; + } + } + if (!found) { + FOREACH(pNode, pOldScanCols) { + if (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) { + nodesListMakeStrictAppend(&pScan->pScanCols, nodesCloneNode(pNode)); + break; + } + } + } + nodesDestroyList(pOldScanCols); + code = createColumnByRewriteExprs(pScan->pScanCols, &pScan->node.pTargets); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + OPTIMIZE_FLAG_CLEAR_MASK(pScan->node.optimizedFlag, OPTIMIZE_FLAG_SCAN_PATH); } *pNewAgg = pNew; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e7c6297f44..6144ebd340 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1083,6 +1083,15 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * memmove(fraction, fraction + TSDB_TIME_PRECISION_SEC_DIGITS, TSDB_TIME_PRECISION_SEC_DIGITS); } + // trans current timezone's unix ts to dest timezone + // offset = delta from dest timezone to zero + // delta from zero to current timezone = 3600 * (cur)tsTimezone + int64_t offset = 0; + if (0 != offsetOfTimezone(tz, &offset)) { + goto _end; + } + timeVal -= offset + 3600 * ((int64_t)tsTimezone); + struct tm tmInfo; int32_t len = 0; diff --git a/source/libs/scheduler/src/schStatus.c b/source/libs/scheduler/src/schStatus.c index 9d0ad30e2a..d37393137f 100644 --- a/source/libs/scheduler/src/schStatus.c +++ b/source/libs/scheduler/src/schStatus.c @@ -68,7 +68,7 @@ int32_t schHandleOpBeginEvent(int64_t jobId, SSchJob** job, SCH_OP_TYPE type, SS SSchJob* pJob = schAcquireJob(jobId); if (NULL == pJob) { qDebug("Acquire sch job failed, may be dropped, jobId:0x%" PRIx64, jobId); - SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + SCH_ERR_RET(TSDB_CODE_SCH_JOB_NOT_EXISTS); } *job = pJob; diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 441c71662e..3eadea3cdd 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -122,7 +122,7 @@ int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** p int32_t streamDefaultDel_rocksdb(SStreamState* pState, const void* key); int32_t streamDefaultIterGet_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); void* streamDefaultIterCreate_rocksdb(SStreamState* pState); -int32_t streamDefaultIterValid_rocksdb(void* iter); +bool streamDefaultIterValid_rocksdb(void* iter); void streamDefaultIterSeek_rocksdb(void* iter, const char* key); void streamDefaultIterNext_rocksdb(void* iter); char* streamDefaultIterKey_rocksdb(void* iter, int32_t* len); diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index 63dc497c6f..c23483fffb 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -2859,9 +2859,12 @@ void* streamDefaultIterCreate_rocksdb(SStreamState* pState) { pCur->number = pState->number; return pCur; } -int32_t streamDefaultIterValid_rocksdb(void* iter) { +bool streamDefaultIterValid_rocksdb(void* iter) { + if (iter) { + return false; + } SStreamStateCur* pCur = iter; - return rocksdb_iter_valid(pCur->iter) ? 1 : 0; + return (rocksdb_iter_valid(pCur->iter) && !iterValueIsStale(pCur->iter)) ? true : false; } void streamDefaultIterSeek_rocksdb(void* iter, const char* key) { SStreamStateCur* pCur = iter; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 15b4bd634a..edb61be2d5 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -1106,8 +1106,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i taosArrayPush(pTask->msgInfo.pRetryList, &pRsp->downstreamNodeId); taosThreadMutexUnlock(&pTask->lock); - stError("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for %dms and retry dispatch data", id, - pRsp->downstreamTaskId, pRsp->downstreamNodeId, DISPATCH_RETRY_INTERVAL_MS); + stWarn("s-task:%s inputQ of downstream task:0x%x(vgId:%d) is full, wait for %dms and retry dispatch", id, + pRsp->downstreamTaskId, pRsp->downstreamNodeId, DISPATCH_RETRY_INTERVAL_MS); } else if (pRsp->inputStatus == TASK_INPUT_STATUS__REFUSED) { stError("s-task:%s downstream task:0x%x(vgId:%d) refused the dispatch msg, treat it as success", id, pRsp->downstreamTaskId, pRsp->downstreamNodeId); @@ -1147,8 +1147,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i } int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s failed to dispatch msg to downstream code:%s, add timer to retry in %dms, ref:%d", - pTask->id.idStr, tstrerror(terrno), DISPATCH_RETRY_INTERVAL_MS, ref); + stDebug("s-task:%s failed to dispatch msg to downstream, add into timer to retry in %dms, ref:%d", + pTask->id.idStr, DISPATCH_RETRY_INTERVAL_MS, ref); streamRetryDispatchData(pTask, DISPATCH_RETRY_INTERVAL_MS); } else { // this message has been sent successfully, let's try next one. diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index ac04c6c214..039a1d0737 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -262,18 +262,33 @@ int32_t streamMetaReopen(SStreamMeta* pMeta) { } } - // todo: not wait in a critical region - while ((pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId)) == NULL) { - stInfo("vgId:%d failed to init stream backend, retry in 100ms", pMeta->vgId); - taosMsleep(100); + taosMemoryFree(defaultPath); + taosMemoryFree(newPath); + + return 0; +} + +// todo refactor: the lock shoud be restricted in one function +void streamMetaInitBackend(SStreamMeta* pMeta) { + pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); + if (pMeta->streamBackend == NULL) { + streamMetaWUnLock(pMeta); + + while (1) { + streamMetaWLock(pMeta); + pMeta->streamBackend = streamBackendInit(pMeta->path, pMeta->chkpId, pMeta->vgId); + if (pMeta->streamBackend != NULL) { + break; + } + + streamMetaWUnLock(pMeta); + stInfo("vgId:%d failed to init stream backend, retry in 100ms", pMeta->vgId); + taosMsleep(100); + } } pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); streamBackendLoadCheckpointInfo(pMeta); - - taosMemoryFree(defaultPath); - taosMemoryFree(newPath); - return 0; } void streamMetaClear(SStreamMeta* pMeta) { diff --git a/source/libs/stream/src/streamSessionState.c b/source/libs/stream/src/streamSessionState.c index cc96778762..45d656c456 100644 --- a/source/libs/stream/src/streamSessionState.c +++ b/source/libs/stream/src/streamSessionState.c @@ -72,7 +72,7 @@ bool inSessionWindow(SSessionKey* pKey, TSKEY ts, int64_t gap) { return false; } -static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, SSessionKey* pKey) { +static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey) { SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); @@ -80,7 +80,7 @@ static SRowBuffPos* addNewSessionWindow(SStreamFileState* pFileState, SArray* pW return pNewPos; } -static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, SSessionKey* pKey, int32_t index) { +static SRowBuffPos* insertNewSessionWindow(SStreamFileState* pFileState, SArray* pWinInfos, const SSessionKey* pKey, int32_t index) { SRowBuffPos* pNewPos = getNewRowPosForWrite(pFileState); ASSERT(pNewPos->pRowBuff); memcpy(pNewPos->pKey, pKey, sizeof(SSessionKey)); @@ -270,6 +270,51 @@ int32_t deleteSessionWinStateBuffByPosFn(SStreamFileState* pFileState, SRowBuffP return TSDB_CODE_SUCCESS; } +int32_t allocSessioncWinBuffByNextPosition(SStreamFileState* pFileState, SStreamStateCur* pCur, const SSessionKey* pWinKey, void** ppVal, int32_t* pVLen) { + SRowBuffPos* pNewPos = NULL; + SSHashObj* pSessionBuff = getRowStateBuff(pFileState); + void** ppBuff = tSimpleHashGet(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t)); + SArray* pWinStates = NULL; + if (!ppBuff) { + pWinStates = taosArrayInit(16, POINTER_BYTES); + tSimpleHashPut(pSessionBuff, &pWinKey->groupId, sizeof(uint64_t), &pWinStates, POINTER_BYTES); + } else { + pWinStates = (SArray*)(*ppBuff); + } + if (!pCur) { + pNewPos = addNewSessionWindow(pFileState, pWinStates, pWinKey); + goto _end; + } + + int32_t size = taosArrayGetSize(pWinStates); + if (pCur->buffIndex >= 0) { + if (pCur->buffIndex >= size) { + pNewPos = insertNewSessionWindow(pFileState, pWinStates, pWinKey, size); + goto _end; + } + pNewPos = insertNewSessionWindow(pFileState, pWinStates, pWinKey, pCur->buffIndex); + goto _end; + } else { + if (size > 0) { + SRowBuffPos* pPos = taosArrayGetP(pWinStates, 0); + if (sessionWinKeyCmpr(pWinKey, pPos->pKey) >=0) { + // pCur is invalid + SSessionKey pTmpKey = *pWinKey; + int32_t code = getSessionWinResultBuff(pFileState, &pTmpKey, 0, (void**)&pNewPos, pVLen); + ASSERT(code == TSDB_CODE_FAILED); + goto _end; + } + } + pNewPos = getNewRowPosForWrite(pFileState); + pNewPos->needFree = true; + } + +_end: + memcpy(pNewPos->pKey, pWinKey, sizeof(SSessionKey)); + (*ppVal) = pNewPos; + return TSDB_CODE_SUCCESS; +} + void sessionWinStateClear(SStreamFileState* pFileState) { int32_t buffSize = getRowStateRowSize(pFileState); void* pIte = NULL; diff --git a/source/libs/stream/src/streamSnapshot.c b/source/libs/stream/src/streamSnapshot.c index 103fd9e087..5893bc14f1 100644 --- a/source/libs/stream/src/streamSnapshot.c +++ b/source/libs/stream/src/streamSnapshot.c @@ -195,7 +195,7 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* pHandle, char* path, int64_t chk } } if (qDebugFlag & DEBUG_TRACE) { - char* buf = taosMemoryCalloc(1, 128 + taosArrayGetSize(pFile->pSst) * 16); + char* buf = taosMemoryCalloc(1, 128 + taosArrayGetSize(pFile->pSst) * 64); sprintf(buf, "[current: %s,", pFile->pCurrent); sprintf(buf + strlen(buf), "MANIFEST: %s,", pFile->pMainfest); sprintf(buf + strlen(buf), "options: %s,", pFile->pOptions); @@ -483,7 +483,7 @@ int32_t streamSnapWrite(SStreamSnapWriter* pWriter, uint8_t* pData, uint32_t nDa int32_t streamSnapWriterClose(SStreamSnapWriter* pWriter, int8_t rollback) { SStreamSnapHandle* handle = &pWriter->handle; if (qDebugFlag & DEBUG_TRACE) { - char* buf = (char*)taosMemoryMalloc(128 + taosArrayGetSize(handle->pFileList) * 16); + char* buf = (char*)taosMemoryMalloc(128 + taosArrayGetSize(handle->pFileList) * 64); int n = sprintf(buf, "["); for (int i = 0; i < taosArrayGetSize(handle->pFileList); i++) { SBackendFileItem* item = taosArrayGet(handle->pFileList, i); diff --git a/source/libs/stream/src/streamStart.c b/source/libs/stream/src/streamStart.c index 0b2bf6b4ba..97eb7b79a2 100644 --- a/source/libs/stream/src/streamStart.c +++ b/source/libs/stream/src/streamStart.c @@ -134,7 +134,7 @@ int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration) pTask->schedHistoryInfo.numOfTicks = numOfTicks; int32_t ref = atomic_add_fetch_32(&pTask->status.timerActive, 1); - stDebug("s-task:%s scan-history start in %.2fs, ref:%d", pTask->id.idStr, numOfTicks*0.1, ref); + stDebug("s-task:%s scan-history resumed in %.2fs, ref:%d", pTask->id.idStr, numOfTicks*0.1, ref); if (pTask->schedHistoryInfo.pTimer == NULL) { pTask->schedHistoryInfo.pTimer = taosTmrStart(doReExecScanhistory, SCANHISTORY_IDLE_TIME_SLICE, pTask, streamEnv.timer); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 6ca7bc5e7b..2e51200fe4 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -735,6 +735,14 @@ int32_t streamStateSessionPut(SStreamState* pState, const SSessionKey* key, void #endif } +int32_t streamStateSessionAllocWinBuffByNextPosition(SStreamState* pState, SStreamStateCur* pCur, const SSessionKey* pKey, void** pVal, int32_t* pVLen) { +#ifdef USE_ROCKSDB + return allocSessioncWinBuffByNextPosition(pState->pFileState, pCur, pKey, pVal, pVLen); +#else + return TSDB_CODE_FAILED; +#endif +} + int32_t streamStateSessionGet(SStreamState* pState, SSessionKey* key, void** pVal, int32_t* pVLen) { #ifdef USE_ROCKSDB return getSessionFlushedBuff(pState->pFileState, key, pVal, pVLen); diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index fc47498a3c..d31e4cbfcd 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -321,6 +321,10 @@ void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uin while ((pNode = tdListNext(&iter)) != NULL && i < max) { SRowBuffPos* pPos = *(SRowBuffPos**)pNode->data; if (pPos->beUsed == used) { + if (used && !pPos->pRowBuff) { + ASSERT(pPos->needFree == true); + continue; + } tdListAppend(pFlushList, &pPos); pFileState->flushMark = TMAX(pFileState->flushMark, pFileState->getTs(pPos->pKey)); pFileState->stateBuffRemoveByPosFn(pFileState, pPos); @@ -428,6 +432,7 @@ SRowBuffPos* getNewRowPosForWrite(SStreamFileState* pFileState) { newPos->beUsed = true; newPos->beFlushed = false; newPos->needFree = false; + newPos->beUpdated = true; return newPos; } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index c483d82027..65b0058cfe 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -293,6 +293,15 @@ int32_t httpSendQuit() { static int32_t taosSendHttpReportImpl(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, EHttpCompFlag flag) { + if (server == NULL || uri == NULL) { + tError("http-report failed to report to invalid addr"); + return -1; + } + + if (pCont == NULL || contLen == 0) { + tError("http-report failed to report empty packet"); + return -1; + } SHttpModule* load = taosAcquireRef(httpRefMgt, httpRef); if (load == NULL) { tError("http-report already released"); diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index dc89a24180..06b82f3ba1 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -52,6 +52,7 @@ #include "lz4.h" #include "tRealloc.h" #include "tlog.h" +#include "ttypes.h" #ifdef TD_TSZ #include "td_sz.h" @@ -62,8 +63,6 @@ static const int32_t TEST_NUMBER = 1; #define SIMPLE8B_MAX_INT64 ((uint64_t)1152921504606846974LL) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) -#define ZIGZAG_ENCODE(T, v) (((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1)) // zigzag encode -#define ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) // zigzag decode #ifdef TD_TSZ bool lossyFloat = false; @@ -99,24 +98,7 @@ int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; // get the byte limit. - int32_t word_length = 0; - switch (type) { - case TSDB_DATA_TYPE_BIGINT: - word_length = LONG_BYTES; - break; - case TSDB_DATA_TYPE_INT: - word_length = INT_BYTES; - break; - case TSDB_DATA_TYPE_SMALLINT: - word_length = SHORT_BYTES; - break; - case TSDB_DATA_TYPE_TINYINT: - word_length = CHAR_BYTES; - break; - default: - uError("Invalid compress integer type:%d", type); - return -1; - } + int32_t word_length = getWordLength(type); int32_t byte_limit = nelements * word_length + 1; int32_t opos = 1; @@ -221,24 +203,9 @@ int32_t tsCompressINTImp(const char *const input, const int32_t nelements, char } int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, char *const output, const char type) { - - int32_t word_length = 0; - switch (type) { - case TSDB_DATA_TYPE_BIGINT: - word_length = LONG_BYTES; - break; - case TSDB_DATA_TYPE_INT: - word_length = INT_BYTES; - break; - case TSDB_DATA_TYPE_SMALLINT: - word_length = SHORT_BYTES; - break; - case TSDB_DATA_TYPE_TINYINT: - word_length = CHAR_BYTES; - break; - default: - uError("Invalid decompress integer type:%d", type); - return -1; + int32_t word_length = getWordLength(type); + if (word_length == -1) { + return word_length; } // If not compressed. @@ -247,8 +214,11 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha return nelements * word_length; } - // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 - // 12 13 14 15 +#if __AVX2__ + tsDecompressIntImpl_Hw(input, nelements, output, type); + return nelements * word_length; +#else + // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; @@ -257,185 +227,6 @@ int32_t tsDecompressINTImp(const char *const input, const int32_t nelements, cha int32_t _pos = 0; int64_t prev_value = 0; -#if __AVX2__ - while (1) { - if (_pos == nelements) break; - - uint64_t w = 0; - memcpy(&w, ip, LONG_BYTES); - - char selector = (char)(w & INT64MASK(4)); // selector = 4 - char bit = bit_per_integer[(int32_t)selector]; // bit = 3 - int32_t elems = selector_to_elems[(int32_t)selector]; - - // Optimize the performance, by remove the constantly switch operation. - int32_t v = 4; - uint64_t zigzag_value = 0; - uint64_t mask = INT64MASK(bit); - - switch (type) { - case TSDB_DATA_TYPE_BIGINT: { - int64_t* p = (int64_t*) output; - - int32_t gRemainder = (nelements - _pos); - int32_t num = (gRemainder > elems)? elems:gRemainder; - - int32_t batch = num >> 2; - int32_t remain = num & 0x03; - if (selector == 0 || selector == 1) { - if (tsAVX2Enable && tsSIMDEnable) { - for (int32_t i = 0; i < batch; ++i) { - __m256i prev = _mm256_set1_epi64x(prev_value); - _mm256_storeu_si256((__m256i *)&p[_pos], prev); - _pos += 4; - } - - for (int32_t i = 0; i < remain; ++i) { - p[_pos++] = prev_value; - } - } else { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - p[_pos++] = prev_value; - v += bit; - } - } - } else { - if (tsAVX2Enable && tsSIMDEnable) { - __m256i base = _mm256_set1_epi64x(w); - __m256i maskVal = _mm256_set1_epi64x(mask); - - __m256i shiftBits = _mm256_set_epi64x(bit * 3 + 4, bit * 2 + 4, bit + 4, 4); - __m256i inc = _mm256_set1_epi64x(bit << 2); - - for (int32_t i = 0; i < batch; ++i) { - __m256i after = _mm256_srlv_epi64(base, shiftBits); - __m256i zigzagVal = _mm256_and_si256(after, maskVal); - - // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) - __m256i signmask = _mm256_and_si256(_mm256_set1_epi64x(1), zigzagVal); - signmask = _mm256_sub_epi64(_mm256_setzero_si256(), signmask); - // get the four zigzag values here - __m256i delta = _mm256_xor_si256(_mm256_srli_epi64(zigzagVal, 1), signmask); - - // calculate the cumulative sum (prefix sum) for each number - // decode[0] = prev_value + final[0] - // decode[1] = decode[0] + final[1] -----> prev_value + final[0] + final[1] - // decode[2] = decode[1] + final[2] -----> prev_value + final[0] + final[1] + final[2] - // decode[3] = decode[2] + final[3] -----> prev_value + final[0] + final[1] + final[2] + final[3] - - // 1, 2, 3, 4 - //+ 0, 1, 0, 3 - // 1, 3, 3, 7 - // shift and add for the first round - __m128i prev = _mm_set1_epi64x(prev_value); - __m256i x = _mm256_slli_si256(delta, 8); - - delta = _mm256_add_epi64(delta, x); - _mm256_storeu_si256((__m256i *)&p[_pos], delta); - - // 1, 3, 3, 7 - //+ 0, 0, 3, 3 - // 1, 3, 6, 10 - // shift and add operation for the second round - __m128i firstPart = _mm_loadu_si128((__m128i *)&p[_pos]); - __m128i secondItem = _mm_set1_epi64x(p[_pos + 1]); - __m128i secPart = _mm_add_epi64(_mm_loadu_si128((__m128i *)&p[_pos + 2]), secondItem); - firstPart = _mm_add_epi64(firstPart, prev); - secPart = _mm_add_epi64(secPart, prev); - - // save it in the memory - _mm_storeu_si128((__m128i *)&p[_pos], firstPart); - _mm_storeu_si128((__m128i *)&p[_pos + 2], secPart); - - shiftBits = _mm256_add_epi64(shiftBits, inc); - prev_value = p[_pos + 3]; -// uDebug("_pos:%d %"PRId64", %"PRId64", %"PRId64", %"PRId64, _pos, p[_pos], p[_pos+1], p[_pos+2], p[_pos+3]); - _pos += 4; - } - - // handle the remain value - for (int32_t i = 0; i < remain; i++) { - zigzag_value = ((w >> (v + (batch * bit * 4))) & mask); - prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); - - p[_pos++] = prev_value; -// uDebug("_pos:%d %"PRId64, _pos-1, p[_pos-1]); - - v += bit; - } - } else { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - zigzag_value = ((w >> v) & mask); - prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); - - p[_pos++] = prev_value; -// uDebug("_pos:%d %"PRId64, _pos-1, p[_pos-1]); - - v += bit; - } - } - } - } break; - case TSDB_DATA_TYPE_INT: { - int32_t* p = (int32_t*) output; - - if (selector == 0 || selector == 1) { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - p[_pos++] = (int32_t)prev_value; - } - } else { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - zigzag_value = ((w >> v) & mask); - prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); - - p[_pos++] = (int32_t)prev_value; - v += bit; - } - } - } break; - case TSDB_DATA_TYPE_SMALLINT: { - int16_t* p = (int16_t*) output; - - if (selector == 0 || selector == 1) { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - p[_pos++] = (int16_t)prev_value; - } - } else { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - zigzag_value = ((w >> v) & mask); - prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); - - p[_pos++] = (int16_t)prev_value; - v += bit; - } - } - } break; - - case TSDB_DATA_TYPE_TINYINT: { - int8_t *p = (int8_t *)output; - - if (selector == 0 || selector == 1) { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - p[_pos++] = (int8_t)prev_value; - } - } else { - for (int32_t i = 0; i < elems && count < nelements; i++, count++) { - zigzag_value = ((w >> v) & mask); - prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); - - p[_pos++] = (int8_t)prev_value; - v += bit; - } - } - } break; - } - - ip += LONG_BYTES; - } - - return nelements * word_length; -#else - while (1) { if (count == nelements) break; @@ -644,6 +435,8 @@ int32_t tsDecompressStringImp(const char *const input, int32_t compressedSize, c // TODO: Take care here, we assumes little endian encoding. int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { int32_t _pos = 1; + int32_t longBytes = LONG_BYTES; + ASSERTS(nelements >= 0, "nelements is negative"); if (nelements == 0) return 0; @@ -684,25 +477,25 @@ int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, } flags = flag1 | (flag2 << 4); // Encode the flag. - if ((_pos + CHAR_BYTES - 1) >= nelements * LONG_BYTES) goto _exit_over; + if ((_pos + CHAR_BYTES - 1) >= nelements * longBytes) goto _exit_over; memcpy(output + _pos, &flags, CHAR_BYTES); _pos += CHAR_BYTES; /* Here, we assume it is little endian encoding method. */ // Encode dd1 if (is_bigendian()) { - if ((_pos + flag1 - 1) >= nelements * LONG_BYTES) goto _exit_over; - memcpy(output + _pos, (char *)(&dd1) + LONG_BYTES - flag1, flag1); + if ((_pos + flag1 - 1) >= nelements * longBytes) goto _exit_over; + memcpy(output + _pos, (char *)(&dd1) + longBytes - flag1, flag1); } else { - if ((_pos + flag1 - 1) >= nelements * LONG_BYTES) goto _exit_over; + if ((_pos + flag1 - 1) >= nelements * longBytes) goto _exit_over; memcpy(output + _pos, (char *)(&dd1), flag1); } _pos += flag1; // Encode dd2; if (is_bigendian()) { - if ((_pos + flag2 - 1) >= nelements * LONG_BYTES) goto _exit_over; - memcpy(output + _pos, (char *)(&dd2) + LONG_BYTES - flag2, flag2); + if ((_pos + flag2 - 1) >= nelements * longBytes) goto _exit_over; + memcpy(output + _pos, (char *)(&dd2) + longBytes - flag2, flag2); } else { - if ((_pos + flag2 - 1) >= nelements * LONG_BYTES) goto _exit_over; + if ((_pos + flag2 - 1) >= nelements * longBytes) goto _exit_over; memcpy(output + _pos, (char *)(&dd2), flag2); } _pos += flag2; @@ -715,15 +508,15 @@ int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, flag2 = 0; flags = flag1 | (flag2 << 4); // Encode the flag. - if ((_pos + CHAR_BYTES - 1) >= nelements * LONG_BYTES) goto _exit_over; + if ((_pos + CHAR_BYTES - 1) >= nelements * longBytes) goto _exit_over; memcpy(output + _pos, &flags, CHAR_BYTES); _pos += CHAR_BYTES; // Encode dd1; if (is_bigendian()) { - if ((_pos + flag1 - 1) >= nelements * LONG_BYTES) goto _exit_over; - memcpy(output + _pos, (char *)(&dd1) + LONG_BYTES - flag1, flag1); + if ((_pos + flag1 - 1) >= nelements * longBytes) goto _exit_over; + memcpy(output + _pos, (char *)(&dd1) + longBytes - flag1, flag1); } else { - if ((_pos + flag1 - 1) >= nelements * LONG_BYTES) goto _exit_over; + if ((_pos + flag1 - 1) >= nelements * longBytes) goto _exit_over; memcpy(output + _pos, (char *)(&dd1), flag1); } _pos += flag1; @@ -734,17 +527,19 @@ int32_t tsCompressTimestampImp(const char *const input, const int32_t nelements, _exit_over: output[0] = 0; // Means the string is not compressed - memcpy(output + 1, input, nelements * LONG_BYTES); - return nelements * LONG_BYTES + 1; + memcpy(output + 1, input, nelements * longBytes); + return nelements * longBytes + 1; } int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelements, char *const output) { + int64_t longBytes = LONG_BYTES; + ASSERTS(nelements >= 0, "nelements is negative"); if (nelements == 0) return 0; if (input[0] == 0) { - memcpy(output, input + 1, nelements * LONG_BYTES); - return nelements * LONG_BYTES; + memcpy(output, input + 1, nelements * longBytes); + return nelements * longBytes; } else if (input[0] == 1) { // Decompress int64_t *ostream = (int64_t *)output; @@ -763,7 +558,7 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement delta_of_delta = 0; } else { if (is_bigendian()) { - memcpy(((char *)(&dd1)) + LONG_BYTES - nbytes, input + ipos, nbytes); + memcpy(((char *)(&dd1)) + longBytes - nbytes, input + ipos, nbytes); } else { memcpy(&dd1, input + ipos, nbytes); } @@ -779,7 +574,7 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement prev_value = prev_value + prev_delta; ostream[opos++] = prev_value; } - if (opos == nelements) return nelements * LONG_BYTES; + if (opos == nelements) return nelements * longBytes; // Decode dd2 uint64_t dd2 = 0; @@ -788,7 +583,7 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement delta_of_delta = 0; } else { if (is_bigendian()) { - memcpy(((char *)(&dd2)) + LONG_BYTES - nbytes, input + ipos, nbytes); + memcpy(((char *)(&dd2)) + longBytes - nbytes, input + ipos, nbytes); } else { memcpy(&dd2, input + ipos, nbytes); } @@ -799,7 +594,7 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement prev_delta = delta_of_delta + prev_delta; prev_value = prev_value + prev_delta; ostream[opos++] = prev_value; - if (opos == nelements) return nelements * LONG_BYTES; + if (opos == nelements) return nelements * longBytes; } } else { @@ -807,11 +602,13 @@ int32_t tsDecompressTimestampImp(const char *const input, const int32_t nelement return -1; } } -/* --------------------------------------------Double Compression - * ---------------------------------------------- */ + +/* --------------------------------------------Double Compression ---------------------------------------------- */ void encodeDoubleValue(uint64_t diff, uint8_t flag, char *const output, int32_t *const pos) { + int32_t longBytes = LONG_BYTES; + uint8_t nbytes = (flag & INT8MASK(3)) + 1; - int32_t nshift = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t nshift = (longBytes * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff >>= nshift; while (nbytes) { @@ -906,12 +703,14 @@ int32_t tsCompressDoubleImp(const char *const input, const int32_t nelements, ch } FORCE_INLINE uint64_t decodeDoubleValue(const char *const input, int32_t *const ipos, uint8_t flag) { + int32_t longBytes = LONG_BYTES; + uint64_t diff = 0ul; int32_t nbytes = (flag & 0x7) + 1; for (int32_t i = 0; i < nbytes; i++) { diff |= (((uint64_t)0xff & input[(*ipos)++]) << BITS_PER_BYTE * i); } - int32_t shift_width = (LONG_BYTES * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); + int32_t shift_width = (longBytes * BITS_PER_BYTE - nbytes * BITS_PER_BYTE) * (flag >> 3); diff <<= shift_width; return diff; @@ -1061,14 +860,7 @@ uint32_t decodeFloatValue(const char *const input, int32_t *const ipos, uint8_t return diff; } -int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output) { - float *ostream = (float *)output; - - if (input[0] == 1) { - memcpy(output, input + 1, nelements * FLOAT_BYTES); - return nelements * FLOAT_BYTES; - } - +static void tsDecompressFloatHelper(const char *const input, const int32_t nelements, float* ostream) { uint8_t flags = 0; int32_t ipos = 1; int32_t opos = 0; @@ -1094,6 +886,21 @@ int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, c ostream[opos++] = curr.real; } +} + +int32_t tsDecompressFloatImp(const char *const input, const int32_t nelements, char *const output) { + if (input[0] == 1) { + memcpy(output, input + 1, nelements * FLOAT_BYTES); + return nelements * FLOAT_BYTES; + } + + if (tsSIMDEnable && tsAVX2Enable) { + tsDecompressFloatImplAvx2(input, nelements, output); + } else if (tsSIMDEnable && tsAVX512Enable) { + tsDecompressFloatImplAvx512(input, nelements, output); + } else { // alternative implementation without SIMD instructions. + tsDecompressFloatHelper(input, nelements, (float*)output); + } return nelements * FLOAT_BYTES; } diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index bb282b4ee3..a78d930326 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -22,6 +22,7 @@ #include "tjson.h" #include "tlog.h" #include "tutil.h" +#include "tunit.h" #define CFG_NAME_PRINT_LEN 24 #define CFG_SRC_PRINT_LEN 12 @@ -173,7 +174,7 @@ static int32_t cfgSetBool(SConfigItem *pItem, const char *value, ECfgSrcType sty } static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - int32_t ival = (int32_t)atoi(value); + int32_t ival = taosStrHumanToInt32(value); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax); @@ -187,7 +188,7 @@ static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType st } static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - int64_t ival = (int64_t)atoll(value); + int64_t ival = taosStrHumanToInt64(value); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax); @@ -379,7 +380,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } } break; case CFG_DTYPE_INT32: { - int32_t ival = (int32_t)atoi(pVal); + int32_t ival = (int32_t)taosStrHumanToInt32(pVal); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s value:%d out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); @@ -388,7 +389,7 @@ int32_t cfgCheckRangeForDynUpdate(SConfig *pCfg, const char *name, const char *p } } break; case CFG_DTYPE_INT64: { - int64_t ival = (int64_t)atoll(pVal); + int64_t ival = (int64_t)taosStrHumanToInt64(pVal); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), ival, pItem->imin, pItem->imax); diff --git a/source/util/src/tdecompress.c b/source/util/src/tdecompress.c new file mode 100644 index 0000000000..f32a4014d6 --- /dev/null +++ b/source/util/src/tdecompress.c @@ -0,0 +1,251 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "ttypes.h" +#include "tcompression.h" + +int32_t getWordLength(char type) { + int32_t wordLength = 0; + switch (type) { + case TSDB_DATA_TYPE_BIGINT: + wordLength = LONG_BYTES; + break; + case TSDB_DATA_TYPE_INT: + wordLength = INT_BYTES; + break; + case TSDB_DATA_TYPE_SMALLINT: + wordLength = SHORT_BYTES; + break; + case TSDB_DATA_TYPE_TINYINT: + wordLength = CHAR_BYTES; + break; + default: + uError("Invalid decompress integer type:%d", type); + return -1; + } + + return wordLength; +} + +int32_t tsDecompressIntImpl_Hw(const char *const input, const int32_t nelements, char *const output, const char type) { + int32_t word_length = getWordLength(type); + + // Selector value: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 + char bit_per_integer[] = {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 20, 30, 60}; + int32_t selector_to_elems[] = {240, 120, 60, 30, 20, 15, 12, 10, 8, 7, 6, 5, 4, 3, 2, 1}; + + const char *ip = input + 1; + int32_t count = 0; + int32_t _pos = 0; + int64_t prev_value = 0; + +#if __AVX2__ + while (1) { + if (_pos == nelements) break; + + uint64_t w = 0; + memcpy(&w, ip, LONG_BYTES); + + char selector = (char)(w & INT64MASK(4)); // selector = 4 + char bit = bit_per_integer[(int32_t)selector]; // bit = 3 + int32_t elems = selector_to_elems[(int32_t)selector]; + + // Optimize the performance, by remove the constantly switch operation. + int32_t v = 4; + uint64_t zigzag_value = 0; + uint64_t mask = INT64MASK(bit); + + switch (type) { + case TSDB_DATA_TYPE_BIGINT: { + int64_t* p = (int64_t*) output; + + int32_t gRemainder = (nelements - _pos); + int32_t num = (gRemainder > elems)? elems:gRemainder; + + int32_t batch = num >> 2; + int32_t remain = num & 0x03; + if (selector == 0 || selector == 1) { + if (tsSIMDEnable && tsAVX2Enable) { + for (int32_t i = 0; i < batch; ++i) { + __m256i prev = _mm256_set1_epi64x(prev_value); + _mm256_storeu_si256((__m256i *)&p[_pos], prev); + _pos += 4; + } + + for (int32_t i = 0; i < remain; ++i) { + p[_pos++] = prev_value; + } + } else if (tsSIMDEnable && tsAVX512Enable) { +#if __AVX512F__ + // todo add avx512 impl +#endif + } else { // alternative implementation without SIMD instructions. + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + p[_pos++] = prev_value; + v += bit; + } + } + } else { + if (tsSIMDEnable && tsAVX2Enable) { + __m256i base = _mm256_set1_epi64x(w); + __m256i maskVal = _mm256_set1_epi64x(mask); + + __m256i shiftBits = _mm256_set_epi64x(bit * 3 + 4, bit * 2 + 4, bit + 4, 4); + __m256i inc = _mm256_set1_epi64x(bit << 2); + + for (int32_t i = 0; i < batch; ++i) { + __m256i after = _mm256_srlv_epi64(base, shiftBits); + __m256i zigzagVal = _mm256_and_si256(after, maskVal); + + // ZIGZAG_DECODE(T, v) (((v) >> 1) ^ -((T)((v)&1))) + __m256i signmask = _mm256_and_si256(_mm256_set1_epi64x(1), zigzagVal); + signmask = _mm256_sub_epi64(_mm256_setzero_si256(), signmask); + + // get the four zigzag values here + __m256i delta = _mm256_xor_si256(_mm256_srli_epi64(zigzagVal, 1), signmask); + + // calculate the cumulative sum (prefix sum) for each number + // decode[0] = prev_value + final[0] + // decode[1] = decode[0] + final[1] -----> prev_value + final[0] + final[1] + // decode[2] = decode[1] + final[2] -----> prev_value + final[0] + final[1] + final[2] + // decode[3] = decode[2] + final[3] -----> prev_value + final[0] + final[1] + final[2] + final[3] + + // 1, 2, 3, 4 + //+ 0, 1, 0, 3 + // 1, 3, 3, 7 + // shift and add for the first round + __m128i prev = _mm_set1_epi64x(prev_value); + __m256i x = _mm256_slli_si256(delta, 8); + + delta = _mm256_add_epi64(delta, x); + _mm256_storeu_si256((__m256i *)&p[_pos], delta); + + // 1, 3, 3, 7 + //+ 0, 0, 3, 3 + // 1, 3, 6, 10 + // shift and add operation for the second round + __m128i firstPart = _mm_loadu_si128((__m128i *)&p[_pos]); + __m128i secondItem = _mm_set1_epi64x(p[_pos + 1]); + __m128i secPart = _mm_add_epi64(_mm_loadu_si128((__m128i *)&p[_pos + 2]), secondItem); + firstPart = _mm_add_epi64(firstPart, prev); + secPart = _mm_add_epi64(secPart, prev); + + // save it in the memory + _mm_storeu_si128((__m128i *)&p[_pos], firstPart); + _mm_storeu_si128((__m128i *)&p[_pos + 2], secPart); + + shiftBits = _mm256_add_epi64(shiftBits, inc); + prev_value = p[_pos + 3]; + _pos += 4; + } + + // handle the remain value + for (int32_t i = 0; i < remain; i++) { + zigzag_value = ((w >> (v + (batch * bit * 4))) & mask); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + + p[_pos++] = prev_value; + v += bit; + } + } else if (tsSIMDEnable && tsAVX512Enable) { +#if __AVX512F__ + // todo add avx512 impl +#endif + } else { // alternative implementation without SIMD instructions. + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + zigzag_value = ((w >> v) & mask); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + + p[_pos++] = prev_value; + v += bit; + } + } + } + } break; + case TSDB_DATA_TYPE_INT: { + int32_t* p = (int32_t*) output; + + if (selector == 0 || selector == 1) { + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + p[_pos++] = (int32_t)prev_value; + } + } else { + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + zigzag_value = ((w >> v) & mask); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + + p[_pos++] = (int32_t)prev_value; + v += bit; + } + } + } break; + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* p = (int16_t*) output; + + if (selector == 0 || selector == 1) { + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + p[_pos++] = (int16_t)prev_value; + } + } else { + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + zigzag_value = ((w >> v) & mask); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + + p[_pos++] = (int16_t)prev_value; + v += bit; + } + } + } break; + + case TSDB_DATA_TYPE_TINYINT: { + int8_t *p = (int8_t *)output; + + if (selector == 0 || selector == 1) { + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + p[_pos++] = (int8_t)prev_value; + } + } else { + for (int32_t i = 0; i < elems && count < nelements; i++, count++) { + zigzag_value = ((w >> v) & mask); + prev_value += ZIGZAG_DECODE(int64_t, zigzag_value); + + p[_pos++] = (int8_t)prev_value; + v += bit; + } + } + } break; + } + + ip += LONG_BYTES; + } + +#endif + return nelements * word_length; +} + +int32_t tsDecompressFloatImplAvx512(const char *const input, const int32_t nelements, char *const output) { +#if __AVX512F__ + // todo add it +#endif + return 0; +} + +// todo add later +int32_t tsDecompressFloatImplAvx2(const char *const input, const int32_t nelements, char *const output) { +#if __AVX2__ +#endif + return 0; +} \ No newline at end of file diff --git a/source/util/src/terror.c b/source/util/src/terror.c index bcdbb3e3ac..21022f2016 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -147,6 +147,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CACHE_ERROR, "Stmt cache error") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INTERNAL_ERROR, "Internal error") // mnode-common +TAOS_DEFINE_ERROR(TSDB_CODE_MND_REQ_REJECTED, "Request rejected") TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_RIGHTS, "Insufficient privilege for operation") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, "Data expired") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, "Invalid query id") @@ -503,6 +504,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_STATUS_ERROR, "scheduler status erro TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal error") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_TIMEOUT_ERROR, "Task timeout") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_JOB_IS_DROPPING, "Job is dropping") +TAOS_DEFINE_ERROR(TSDB_CODE_SCH_JOB_NOT_EXISTS, "Job no longer exist") // parser TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYNTAX_ERROR, "syntax error near") diff --git a/source/util/src/tunit.c b/source/util/src/tunit.c new file mode 100644 index 0000000000..d3447294ea --- /dev/null +++ b/source/util/src/tunit.c @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tunit.h" + +#define UNIT_SIZE_CONVERT_FACTOR 1024LL +#define UNIT_ONE_KIBIBYTE UNIT_SIZE_CONVERT_FACTOR +#define UNIT_ONE_MEBIBYTE (UNIT_ONE_KIBIBYTE * UNIT_SIZE_CONVERT_FACTOR) +#define UNIT_ONE_GIBIBYTE (UNIT_ONE_MEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) +#define UNIT_ONE_TEBIBYTE (UNIT_ONE_GIBIBYTE * UNIT_SIZE_CONVERT_FACTOR) +#define UNIT_ONE_PEBIBYTE (UNIT_ONE_TEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) +#define UNIT_ONE_EXBIBYTE (UNIT_ONE_PEBIBYTE * UNIT_SIZE_CONVERT_FACTOR) + +int64_t taosStrHumanToInt64(const char* str) { + size_t sLen = strlen(str); + if (sLen < 2) return atoll(str); + + int64_t val = 0; + + char* strNoUnit = NULL; + char unit = str[sLen - 1]; + if ((unit == 'P') || (unit == 'p')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_PEBIBYTE; + } else if ((unit == 'T') || (unit == 't')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_TEBIBYTE; + } else if ((unit == 'G') || (unit == 'g')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_GIBIBYTE; + } else if ((unit == 'M') || (unit == 'm')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_MEBIBYTE; + } else if ((unit == 'K') || (unit == 'k')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_KIBIBYTE; + } else { + val = atoll(str); + } + + taosMemoryFree(strNoUnit); + return val; +} + +void taosInt64ToHumanStr(int64_t val, char* outStr) { + if (((val >= UNIT_ONE_EXBIBYTE) || (-val >= UNIT_ONE_EXBIBYTE)) && ((val % UNIT_ONE_EXBIBYTE) == 0)) { + sprintf(outStr, "%qdE", (long long)val / UNIT_ONE_EXBIBYTE); + } else if (((val >= UNIT_ONE_PEBIBYTE) || (-val >= UNIT_ONE_PEBIBYTE)) && ((val % UNIT_ONE_PEBIBYTE) == 0)) { + sprintf(outStr, "%qdP", (long long)val / UNIT_ONE_PEBIBYTE); + } else if (((val >= UNIT_ONE_TEBIBYTE) || (-val >= UNIT_ONE_TEBIBYTE)) && ((val % UNIT_ONE_TEBIBYTE) == 0)) { + sprintf(outStr, "%qdT", (long long)val / UNIT_ONE_TEBIBYTE); + } else if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { + sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); + } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { + sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); + } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { + sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); + } else + sprintf(outStr, "%qd", (long long)val); +} + +int32_t taosStrHumanToInt32(const char* str) { + size_t sLen = strlen(str); + if (sLen < 2) return atoll(str); + + int32_t val = 0; + + char* strNoUnit = NULL; + char unit = str[sLen - 1]; + if ((unit == 'G') || (unit == 'g')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_GIBIBYTE; + } else if ((unit == 'M') || (unit == 'm')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_MEBIBYTE; + } else if ((unit == 'K') || (unit == 'k')) { + strNoUnit = taosMemoryCalloc(sLen, 1); + memcpy(strNoUnit, str, sLen - 1); + + val = atof(strNoUnit) * UNIT_ONE_KIBIBYTE; + } else { + val = atoll(str); + } + + taosMemoryFree(strNoUnit); + return val; +} + +void taosInt32ToHumanStr(int32_t val, char* outStr) { + if (((val >= UNIT_ONE_GIBIBYTE) || (-val >= UNIT_ONE_GIBIBYTE)) && ((val % UNIT_ONE_GIBIBYTE) == 0)) { + sprintf(outStr, "%qdG", (long long)val / UNIT_ONE_GIBIBYTE); + } else if (((val >= UNIT_ONE_MEBIBYTE) || (-val >= UNIT_ONE_MEBIBYTE)) && ((val % UNIT_ONE_MEBIBYTE) == 0)) { + sprintf(outStr, "%qdM", (long long)val / UNIT_ONE_MEBIBYTE); + } else if (((val >= UNIT_ONE_KIBIBYTE) || (-val >= UNIT_ONE_KIBIBYTE)) && ((val % UNIT_ONE_KIBIBYTE) == 0)) { + sprintf(outStr, "%qdK", (long long)val / UNIT_ONE_KIBIBYTE); + } else + sprintf(outStr, "%qd", (long long)val); +} diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 6a7c0b47ec..92eaec52b5 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -120,6 +120,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/slimit.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ts-4233.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreDnode.py -N 5 -M 3 -i False ,,y,system-test,./pytest.sh python3 ./test.py -f 3-enterprise/restore/restoreVnode.py -N 5 -M 3 -i False @@ -239,6 +243,7 @@ e ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/splitVGroupRep3.py -N 3 ,,n,system-test,python3 ./test.py -f 0-others/timeRangeWise.py -N 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/delete_check.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/test_hot_refresh_configurations.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_database.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_replica.py -N 3 diff --git a/tests/script/tsim/stream/checkpointInterval1.sim b/tests/script/tsim/stream/checkpointInterval1.sim index 21825e7f48..36f361ad64 100644 --- a/tests/script/tsim/stream/checkpointInterval1.sim +++ b/tests/script/tsim/stream/checkpointInterval1.sim @@ -15,6 +15,8 @@ sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart, count(*) c1, sum(a) from st interval(10s); +sleep 1000 + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t2 values(1648791213001,2,2,3,1.1); diff --git a/tests/script/tsim/stream/event0.sim b/tests/script/tsim/stream/event0.sim new file mode 100644 index 0000000000..57116bbc79 --- /dev/null +++ b/tests/script/tsim/stream/event0.sim @@ -0,0 +1,322 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print step1 +print =============== create database +sql create database test vgroups 1; +sql use test; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with a = 9; +sleep 1000 +sql insert into t1 values(1648791213000,0,1,1,1.0); +sql insert into t1 values(1648791223001,9,2,2,1.1); +sql insert into t1 values(1648791213009,0,3,3,1.0); + +$loop_count = 0 +loop0: + +sleep 300 +print 1 sql select * from streamt; +sql select * from streamt; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data01 != 3 then + print ======data01=$data01 + goto loop0 +endi + +if $data02 != 6 then + print ======data02=$data02 + goto loop0 +endi + +if $data03 != 3 then + print ======data03=$data03 + goto loop0 +endi + +sql insert into t1 values(1648791243006,1,1,1,1.1); +sql insert into t1 values(1648791253000,2,2,2,1.1); + + +$loop_count = 0 +loop1: + +sleep 300 +print 2 sql select * from streamt; +sql select * from streamt; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 1 then + print ======rows=$rows + goto loop1 +endi + +sql insert into t1 values(1648791243000,0,3,3,1.1); + +$loop_count = 0 +loop2: + +sleep 300 +print 3 sql select * from streamt; +sql select * from streamt; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 1 then + print ======rows=$rows + goto loop2 +endi + +sql insert into t1 values(1648791253009,9,4,4,1.1); + +$loop_count = 0 +loop3: + +sleep 300 +print 4 sql select * from streamt; +sql select * from streamt; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 2 then + print ======rows=$rows + goto loop3 +endi + +# row 0 +if $data01 != 3 then + print ======data01=$data01 + goto loop3 +endi + +if $data02 != 6 then + print ======data02=$data02 + goto loop3 +endi + +if $data03 != 3 then + print ======data03=$data03 + goto loop3 +endi + +# row 1 +if $data11 != 4 then + print ======data11=$data11 + goto loop3 +endi + +if $data12 != 10 then + print ======data12=$data12 + goto loop3 +endi + +if $data13 != 4 then + print ======data13=$data13 + goto loop3 +endi + +print step2 +print =============== create database test2 +sql create database test2 vgroups 1; +sql use test2; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; +sleep 1000 +sql insert into t1 values(1648791213000,0,1,1,1.0); +sql insert into t1 values(1648791213009,1,2,2,2.1); +sql insert into t1 values(1648791223000,0,9,9,9.0); + + +sql insert into t1 values(1648791233000,0,9,9,9.0); + + +$loop_count = 0 +loop4: + +sleep 300 +print sql select * from streamt2; +sql select * from streamt2; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 2 then + print ======rows=$rows + goto loop4 +endi + +# row 0 +if $data01 != 3 then + print ======data01=$data01 + goto loop4 +endi + +if $data11 != 1 then + print ======data11=$data11 + goto loop4 +endi + +print step3 +print =============== create database test3 +sql create database test3 vgroups 1; +sql use test3; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; +sleep 1000 + +sql insert into t1 values(1648791233009,1,2,2,2.1); + +sql insert into t1 values(1648791233000,0,1,1,1.0); +sql insert into t1 values(1648791243000,0,9,9,9.0); + +$loop_count = 0 +loop5: + +sleep 300 +print 1 sql select * from streamt3; +sql select * from streamt3; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $rows != 1 then + print ======rows=$rows + goto loop5 +endi + +# row 0 +if $data01 != 3 then + print ======data01=$data01 + goto loop5 +endi + +sql insert into t1 values(1648791223000,0,9,9,9.0); + +$loop_count = 0 +loop6: + +sleep 300 +print 2 sql select * from streamt3; +sql select * from streamt3; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print ======rows=$rows + goto loop6 +endi + +# row 0 +if $data01 != 1 then + print ======data01=$data01 + goto loop6 +endi + +# row 1 +if $data11 != 3 then + print ======data11=$data11 + goto loop6 +endi + +sql insert into t1 values(1648791213000,0,1,1,1.0); + +sleep 300 + +sql insert into t1 values(1648791213001,1,9,9,9.0); + +$loop_count = 0 +loop7: + +sleep 300 +print 3 sql select * from streamt3; +sql select * from streamt3; + +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print ======rows=$rows + goto loop7 +endi + +# row 0 +if $data01 != 2 then + print ======data01=$data01 + goto loop7 +endi + +# row 1 +if $data11 != 1 then + print ======data11=$data11 + goto loop7 +endi + +# row 2 +if $data21 != 3 then + print ======data21=$data21 + goto loop7 +endi + +print event0 end +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/event1.sim b/tests/script/tsim/stream/event1.sim new file mode 100644 index 0000000000..f3b0e4eb01 --- /dev/null +++ b/tests/script/tsim/stream/event1.sim @@ -0,0 +1,332 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print step1 +print =============== create database test1 +sql create database test1 vgroups 1; +sql use test1; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt1 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; +sleep 1000 + +sql insert into t1 values(1648791233000,0,1,1,1.0); +sql insert into t1 values(1648791243000,1,9,9,9.0); + + +sql insert into t1 values(1648791223000,3,3,3,3.0); + +$loop_count = 0 +loop0: + +sleep 300 +print 1 sql select * from streamt1; +sql select * from streamt1; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print ======rows=$rows + goto loop0 +endi + +# row 0 +if $data01 != 2 then + print ======data01=$data01 + goto loop0 +endi + +print step2 +print =============== create database test2 +sql create database test2 vgroups 1; +sql use test2; + +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams2 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart as s, count(*) c1, sum(b), max(c) from t1 event_window start with a = 0 end with b = 9; +sleep 1000 + +sql insert into t1 values(1648791223000,0,3,3,3.0); + +sql insert into t1 values(1648791233000,0,1,1,1.0); +sql insert into t1 values(1648791243000,1,9,2,2.0); + + +$loop_count = 0 +loop1: + +sleep 300 +print 1 sql select * from streamt2; +sql select * from streamt2; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print ======rows=$rows + goto loop1 +endi + +# row 0 +if $data01 != 3 then + print ======data01=$data01 + goto loop1 +endi + + +sql insert into t1 values(1648791223000,1,1,4,4.0); + +$loop_count = 0 +loop2: + +sleep 300 +print 1 sql select * from streamt2; +sql select * from streamt2; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print ======rows=$rows + goto loop2 +endi + +# row 0 +if $data01 != 2 then + print ======data01=$data01 + goto loop2 +endi + +sql insert into t1 values(1648791243000,1,1,5,5.0); + +$loop_count = 0 +loop3: + +sleep 300 +print 1 sql select * from streamt2; +sql select * from streamt2; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print ======rows=$rows + goto loop3 +endi + +sql insert into t1 values(1648791253000,1,9,6,6.0); + +$loop_count = 0 +loop4: + +sleep 300 +print 1 sql select * from streamt2; +sql select * from streamt2; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print ======rows=$rows + goto loop4 +endi + +# row 0 +if $data01 != 3 then + print ======data01=$data01 + goto loop4 +endi + +sql delete from t1 where ts = 1648791253000; + +$loop_count = 0 +loop5: + +sleep 300 +print 1 sql select * from streamt2; +sql select * from streamt2; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print ======rows=$rows + goto loop5 +endi + +sql insert into t1 values(1648791263000,1,9,7,7.0); +sql delete from t1 where ts = 1648791243000; + +$loop_count = 0 +loop6: + +sleep 300 +print 1 sql select * from streamt2; +sql select * from streamt2; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print ======rows=$rows + goto loop6 +endi + +# row 0 +if $data01 != 2 then + print ======data01=$data01 + goto loop6 +endi + +print step3 +print =============== create database test3 +sql create database test3 vgroups 1; +sql use test3; + +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream streams3 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt3 as select _wstart as s, count(*) c1, sum(b), max(c) from st partition by tbname event_window start with a = 0 end with b = 9; +sleep 1000 + +sql insert into t1 values(1648791223000,0,3,3,3.0); + +sql insert into t1 values(1648791233000,0,1,1,1.0); +sql insert into t1 values(1648791243000,1,9,2,2.0); + +sql insert into t2 values(1648791223000,0,3,3,3.0); + +sql insert into t2 values(1648791233000,0,1,1,1.0); +sql insert into t2 values(1648791243000,1,9,2,2.0); + + +$loop_count = 0 +loop7: + +sleep 300 +print 1 sql select * from streamt3; +sql select * from streamt3; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print ======rows=$rows + goto loop7 +endi + +if $data01 != 3 then + print ======data01=$data01 + goto loop7 +endi + +if $data11 != 3 then + print ======data11=$data11 + goto loop7 +endi + +print update data +sql insert into t1 values(1648791243000,1,3,3,3.0); +sql insert into t2 values(1648791243000,1,3,3,3.0); + +sleep 1000 + +sql insert into t1 values(1648791253000,1,9,3,3.0); +sql insert into t2 values(1648791253000,1,9,3,3.0); + +$loop_count = 0 +loop8: + +sleep 300 +print 1 sql select * from streamt3; +sql select * from streamt3; + +print +print $data00 $data01 $data02 $data03 +print $data10 $data11 $data12 $data13 +print $data20 $data21 $data22 $data23 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print ======rows=$rows + goto loop8 +endi + +if $data01 != 4 then + print ======data01=$data01 + goto loop8 +endi + +if $data11 != 4 then + print ======data11=$data11 + goto loop8 +endi + +print event1 end +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/event2.sim b/tests/script/tsim/stream/event2.sim new file mode 100644 index 0000000000..eb9fca46e6 --- /dev/null +++ b/tests/script/tsim/stream/event2.sim @@ -0,0 +1,84 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print step1 +print =============== create database test +sql create database test vgroups 1; +sql use test; + +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); +sql create table t4 using st tags(3,3,3); + +sql insert into t1 values(1648791223000,0,1,1,1.0); +sql insert into t1 values(1648791233000,0,2,2,2.0); +sql insert into t1 values(1648791243000,1,3,3,3.0); + +sql insert into t2 values(1648791223000,0,1,4,3.0); +sql insert into t2 values(1648791233000,0,2,5,1.0); +sql insert into t2 values(1648791243000,1,3,6,2.0); + +sql insert into t3 values(1648791223000,1,1,7,3.0); +sql insert into t3 values(1648791233000,1,2,8,1.0); +sql insert into t3 values(1648791243000,1,3,9,2.0); + +sql insert into t4 values(1648791223000,1,1,10,3.0); +sql insert into t4 values(1648791233000,0,2,11,1.0); +sql insert into t4 values(1648791243000,1,9,12,2.0); + +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 1 into streamt0 as select _wstart as s, count(*) c1, sum(b), max(c), _wend as e from st partition by tbname event_window start with a = 0 end with b = 9; + +sleep 1000 + +sql insert into t1 values(1648791253000,1,9,13,2.0); +sql insert into t2 values(1648791253000,1,9,14,2.0); +sql insert into t3 values(1648791253000,1,9,15,2.0); +sql insert into t4 values(1648791253000,1,9,16,2.0); + +$loop_count = 0 +loop0: + +sleep 300 +print 1 sql select * from streamt0 order by 1, 2, 3, 4; +sql select * from streamt0 order by 1, 2, 3, 4; + +print +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print ======rows=$rows + goto loop0 +endi + +if $data01 != 4 then + print ======data01=$data01 + goto loop0 +endi + +if $data11 != 4 then + print ======data11=$data11 + goto loop0 +endi + +if $data21 != 2 then + print ======data21=$data21 + goto loop0 +endi + + +print event1 end +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/eventtmp.sim b/tests/script/tsim/stream/eventtmp.sim new file mode 100644 index 0000000000..52ca4fd9ef --- /dev/null +++ b/tests/script/tsim/stream/eventtmp.sim @@ -0,0 +1,61 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +print step1 +print =============== create database test +sql create database test vgroups 1; +sql use test; + +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(3,3,3); +sql create table t4 using st tags(3,3,3); + + +sql insert into t4 values(1648791223000,1,1,10,3.0); +sql insert into t4 values(1648791233000,0,2,11,1.0); +sql insert into t4 values(1648791243000,1,9,12,2.0); + +sql create stream streams0 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 fill_history 1 into streamt0 as select _wstart as s, count(*) c1, sum(b), max(c), _wend as e from st partition by tbname event_window start with a = 0 end with b = 9; + +sleep 1000 + +sql insert into t4 values(1648791253000,1,9,16,2.0); + +$loop_count = 0 +loop0: + +sleep 300 +print 1 sql select * from streamt0 order by 1, 2, 3, 4; +sql select * from streamt0 order by 1, 2, 3, 4; + +print +print $data00 $data01 $data02 $data03 $data04 +print $data10 $data11 $data12 $data13 $data14 +print $data20 $data21 $data22 $data23 $data24 +print $data30 $data31 $data32 $data33 $data34 +print + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print ======rows=$rows + goto loop0 +endi + +if $data01 != 2 then + print ======data01=$data01 + goto loop0 +endi + + + +print event1 end +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/fillIntervalRange.sim b/tests/script/tsim/stream/fillIntervalRange.sim index 99c1fe8ad4..e5316e6a1e 100644 --- a/tests/script/tsim/stream/fillIntervalRange.sim +++ b/tests/script/tsim/stream/fillIntervalRange.sim @@ -64,7 +64,7 @@ endi sql select count(*) from streamt; if $data00 != 9098 then - print =====rows=$rows + print =====data00=$data00 goto loop1 endi diff --git a/tests/script/tsim/stream/sliding.sim b/tests/script/tsim/stream/sliding.sim index a92da7f472..cc4d14b6fb 100644 --- a/tests/script/tsim/stream/sliding.sim +++ b/tests/script/tsim/stream/sliding.sim @@ -23,6 +23,8 @@ sql create stream stream_t1 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 i sql create stream stream_t2 trigger at_once watermark 1d IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamtST2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s) sliding (5s); sleep 1000 +sleep 1000 + sql insert into t1 values(1648791210000,1,2,3,1.0); sql insert into t1 values(1648791216000,2,2,3,1.1); sql insert into t1 values(1648791220000,3,2,3,2.1); @@ -314,6 +316,8 @@ sql create stream streams11 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 i sql create stream streams12 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt2 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); sleep 1000 +sleep 1000 + sql insert into t1 values(1648791213000,1,2,3,1.0); sql insert into t1 values(1648791223001,2,2,3,1.1); sql insert into t1 values(1648791233002,3,2,3,2.1); @@ -448,6 +452,8 @@ sql create stream streams21 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 i sql create stream streams22 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt22 as select _wstart, count(*) c1, sum(a) c3 , max(b) c4, min(c) c5 from st interval(10s, 5s); sleep 1000 +sleep 1000 + sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791223001,2,2,2,1.1); sql insert into t1 values(1648791233002,3,3,3,2.1); @@ -712,6 +718,8 @@ sql create table t2 using st tags(2,2,2); sql create stream streams4 trigger at_once IGNORE EXPIRED 0 IGNORE UPDATE 0 into streamt4 as select _wstart as ts, count(*),min(a) c1 from st interval(10s) sliding(5s); sleep 1000 +sleep 1000 + sql insert into t1 values(1648791213000,1,1,1,1.0); sql insert into t1 values(1648791243000,2,1,1,1.0); diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 5bd17e076e..67678963ea 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -134,6 +134,162 @@ if $rows != 2 then goto loop1 endi +print step 1 max delay 2s +sql create database test3 vgroups 4; +sql use test3; +sql create table t1(ts timestamp, a int, b int , c int, d double); + +sql create stream stream13 trigger max_delay 2s into streamt13 as select _wstart, sum(a), now from t1 interval(10s); + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.1); + +$loop_count = 0 + +loop2: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +sql select * from streamt13; + +if $rows != 2 then + print ======rows=$rows + goto loop2 +endi + +$now02 = $data02 +$now12 = $data12 + + +print step1 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt13; + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +print step 2 max delay 2s + +sql create database test4 vgroups 4; +sql use test4; + +sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); + +sql create stream stream14 trigger max_delay 2s into streamt14 as select _wstart, sum(a), now from st partition by tbname interval(10s); + +sleep 1000 + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t1 values(1648791223000,2,2,3,1.1); + +sql insert into t2 values(1648791213000,3,2,3,1.0); +sql insert into t2 values(1648791223000,4,2,3,1.1); + +$loop_count = 0 + +loop3: + +sleep 1000 + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +sql select * from streamt14 order by 2; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 + +if $rows != 4 then + print ======rows=$rows + goto loop3 +endi + +$now02 = $data02 +$now12 = $data12 +$now22 = $data22 +$now32 = $data32 + +print step2 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt14 order by 2; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +if $data22 != $now22 then + print ======data22=$data22 + return -1 +endi + +if $data32 != $now32 then + print ======data32=$data32 + return -1 +endi + +print step2 max delay 2s......... sleep 3s +sleep 3000 + +sql select * from streamt14 order by 2; +print $data00 $data01 $data02 +print $data10 $data11 $data12 +print $data20 $data21 $data22 +print $data30 $data31 $data32 + + +if $data02 != $now02 then + print ======data02=$data02 + return -1 +endi + +if $data12 != $now12 then + print ======data12=$data12 + return -1 +endi + +if $data22 != $now22 then + print ======data22=$data22 + return -1 +endi + +if $data32 != $now32 then + print ======data32=$data32 + return -1 +endi + print ======over system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/system-test/0-others/test_hot_refresh_configurations.py b/tests/system-test/0-others/test_hot_refresh_configurations.py new file mode 100644 index 0000000000..7aed7274a4 --- /dev/null +++ b/tests/system-test/0-others/test_hot_refresh_configurations.py @@ -0,0 +1,222 @@ +import subprocess +import random +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + + +class TDTestCase: + """This test case is used to veirfy hot refresh configurations + """ + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + self.configration_dic = { + "cli": [ + { + "name": "keepAliveIdle", + "values": [1, 100, 7200000], + "except_values": [0, 7200001] + }, + { + "name": "queryPolicy", + "values": [1, 2, 4], + "except_values": [0, 5] + }, + { + "name": "numOfLogLines", + "values": [1000, 2000, 2000000000], + "except_values": [999, 2000000001] + }, + { + "name": "logKeepDays", + "values": [-365000, 2000, 365000], + "except_values": [-365001, 365001] + } + ], + "svr": [ + { + "name": "keepAliveIdle", + "alias": "tsKeepAliveIdle", + "values": [1, 100, 7200000], + "except_values": [0, 7200001] + }, + { + "name": "mndSdbWriteDelta", + "alias": "tsMndSdbWriteDelta", + "values": [20, 1000, 10000], + "except_values": [19, 10001] + }, + { + "name": "enableWhiteList", + "alias": "tsEnableWhiteList", + "values": [0, 1], + "except_values": [-1, 2] + }, + { + "name": "audit", + "alias": "tsEnableAudit", + "values": [0, 1], + "except_values": [-1, 2] + }, + { + "name": "telemetryReporting", + "alias": "tsEnableTelem", + "values": [0, 1], + "except_values": [-1, 2] + }, + { + "name": "cacheLazyLoadThreshold", + "alias": "tsCacheLazyLoadThreshold", + "values": [0, 200, 100000], + "except_values": [-1, 100001] + }, + { + "name": "queryRspPolicy", + "alias": "tsQueryRspPolicy", + "values": [0, 1], + "except_values": [-1, 2] + }, + { + "name": "ttlFlushThreshold", + "alias": "tsTtlFlushThreshold", + "values": [-1, 200, 1000000], + "except_values": [-2, 1000001] + }, + { + "name": "timeseriesThreshold", + "alias": "tsTimeSeriesThreshold", + "values": [0, 200, 2000], + "except_values": [-2, 2001] + }, + { + "name": "minDiskFreeSize", + "alias": "tsMinDiskFreeSize", + "values": ["51200K", "100M", "1G"], + "except_values": ["1024K", "1.1G", "1T"] + }, + { + "name": "tmqMaxTopicNum", + "alias": "tmqMaxTopicNum", + "values": [1, 1000, 10000], + "except_values": [0, 10001] + }, + { + "name": "transPullupInterval", + "alias": "tsTransPullupInterval", + "values": [1, 1000, 10000], + "except_values": [0, 10001] + }, + { + "name": "mqRebalanceInterval", + "alias": "tsMqRebalanceInterval", + "values": [1, 1000, 10000], + "except_values": [0, 10001] + }, + { + "name": "checkpointInterval", + "alias": "tsStreamCheckpointInterval", + "values": [60, 1000, 1200], + "except_values": [59, 1201] + }, + { + "name": "trimVDbIntervalSec", + "alias": "tsTrimVDbIntervalSec", + "values": [1, 1000, 100000], + "except_values": [0, 100001] + }, + { + "name": "disableStream", + "alias": "tsDisableStream", + "values": [0, 1], + "except_values": [-1] + }, + { + "name": "maxStreamBackendCache", + "alias": "tsMaxStreamBackendCache", + "values": [16, 512, 1024], + "except_values": [15, 1025] + }, + { + "name": "numOfLogLines", + "alias": "tsNumOfLogLines", + "values": [1000, 50000, 2000000000], + "except_values": [999, 2000000001] + }, + { + "name": "logKeepDays", + "alias": "tsLogKeepDays", + "values": [-365000, 10, 365000], + "except_values": [-365001, 365001] + } + ] + } + + def get_param_value_with_gdb(self, config_name, process_name): + res = subprocess.Popen("gdb -q -nx -p `pidof {}` --batch -ex 'set height 0' -ex 'p {}'".format(process_name, config_name), shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + r_lines = res.stdout.read() + if r_lines: + for line in r_lines.decode().split("\n"): + if "$1 = " in line: + tdLog.debug("gdb result: {}".format(line)) + return line.split(" = ")[1] + + def cli_check(self, name, values, except_values=False): + if not except_values: + for v in values: + tdLog.debug("Set {} to {}".format(name, v)) + tdSql.execute(f'alter local "{name} {v}";') + else: + for v in values: + tdLog.debug("Set {} to {}".format(name, v)) + tdSql.error(f'alter local "{name} {v}";') + + def svr_check(self, name, alias, values, except_values=False): + p_list = ["dnode 1", "all dnodes"] + # check bool param value + if len(values) == 2 and [0, 1] == values and name != "queryRspPolicy": + is_bool = True + else: + is_bool = False + tdLog.debug(f"{name} is_bool: {is_bool}") + if not except_values: + for v in values: + dnode = random.choice(p_list) + tdSql.execute(f'alter {dnode} "{name} {v}";') + value = self.get_param_value_with_gdb(alias, "taosd") + if value: + tdLog.debug(f"value: {value}") + assert(value == str(bool(v)).lower() if is_bool else str(v)) + else: + for v in values: + dnode = random.choice(p_list) + tdSql.error(f'alter {dnode} "{name} {v}";') + + def run(self): + for key in self.configration_dic: + if "cli" == key: + for item in self.configration_dic[key]: + self.cli_check(item["name"], item["values"]) + if "except_values" in item: + self.cli_check(item["name"], item["except_values"], True) + elif "svr" == key: + for item in self.configration_dic[key]: + self.svr_check(item["name"], item["alias"], item["values"]) + if "except_values" in item: + self.svr_check(item["name"], item["alias"], item["except_values"], True) + else: + raise Exception(f"unknown key: {key}") + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/To_iso8601.py b/tests/system-test/2-query/To_iso8601.py index 92aacbb350..160473ffce 100644 --- a/tests/system-test/2-query/To_iso8601.py +++ b/tests/system-test/2-query/To_iso8601.py @@ -30,17 +30,16 @@ class TDTestCase: tdSql.query(f'select to_iso8601(ts) from {self.ntbname}') for i in range(self.rowNum): tdSql.checkEqual(tdSql.queryResult[i][0],f'2022-01-01T00:00:00.00{i}{time_zone}') - timezone_list = ['+0000','+0100','+0200','+0300','+0330','+0400','+0500','+0530','+0600','+0700','+0800','+0900','+1000','+1100','+1200',\ - '+00','+01','+02','+03','+04','+05','+06','+07','+08','+09','+10','+11','+12',\ - '+00:00','+01:00','+02:00','+03:00','+03:30','+04:00','+05:00','+05:30','+06:00','+07:00','+08:00','+09:00','+10:00','+11:00','+12:00',\ - '-0000','-0100','-0200','-0300','-0400','-0500','-0600','-0700','-0800','-0900','-1000','-1100','-1200',\ - '-00','-01','-02','-03','-04','-05','-06','-07','-08','-09','-10','-11','-12',\ - '-00:00','-01:00','-02:00','-03:00','-04:00','-05:00','-06:00','-07:00','-08:00','-09:00','-10:00','-11:00','-12:00',\ - 'z','Z'] - for j in timezone_list: - tdSql.query(f'select to_iso8601(ts,"{j}") from {self.ntbname}') - for i in range(self.rowNum): - tdSql.checkEqual(tdSql.queryResult[i][0],f'2022-01-01T00:00:00.00{i}{j}') + + tz_list = ['+0000','+0530', '+00', '+06', '+00:00', '+12:00', '-0000', '-0900', '-00', '-05', '-00:00', '-03:00','z', 'Z'] + res_list = ['2021-12-31T16:00:00.000+0000', '2021-12-31T21:30:00.000+0530', '2021-12-31T16:00:00.000+00', '2021-12-31T22:00:00.000+06',\ + '2021-12-31T16:00:00.000+00:00', '2022-01-01T04:00:00.000+12:00','2021-12-31T16:00:00.000-0000','2021-12-31T07:00:00.000-0900',\ + '2021-12-31T16:00:00.000-00', '2021-12-31T11:00:00.000-05','2021-12-31T16:00:00.000-00:00','2021-12-31T13:00:00.000-03:00',\ + '2021-12-31T16:00:00.000z', '2021-12-31T16:00:00.000Z'] + for i in range(len(tz_list)): + tdSql.query(f'select to_iso8601(ts,"{tz_list[i]}") from {self.ntbname} where c1==1') + tdSql.checkEqual(tdSql.queryResult[0][0],res_list[i]) + error_param_list = [0,100.5,'a','!'] for i in error_param_list: tdSql.error(f'select to_iso8601(ts,"{i}") from {self.ntbname}') diff --git a/tests/system-test/2-query/limit.py b/tests/system-test/2-query/limit.py index fb5595a8be..961cff5087 100644 --- a/tests/system-test/2-query/limit.py +++ b/tests/system-test/2-query/limit.py @@ -338,11 +338,38 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 1 end ...... ") + # + def checkVGroups(self): + + # db2 + tdSql.execute("create database db2 vgroups 2;") + tdSql.execute("use db2;") + tdSql.execute("create table st(ts timestamp, age int) tags(area int);") + tdSql.execute("create table t1 using st tags(1);") + tdSql.query("select distinct(tbname) from st limit 1 offset 100;") + tdSql.checkRows(0) + tdLog.info("check db2 vgroups 2 limit 1 offset 100 successfully!") + + + # db1 + tdSql.execute("create database db1 vgroups 1;") + tdSql.execute("use db1;") + tdSql.execute("create table st(ts timestamp, age int) tags(area int);") + tdSql.execute("create table t1 using st tags(1);") + tdSql.query("select distinct(tbname) from st limit 1 offset 100;") + tdSql.checkRows(0) + tdLog.info("check db1 vgroups 1 limit 1 offset 100 successfully!") + + def run(self): # tdSql.prepare() self.prepareTestEnv() self.tmqCase1() + # one vgroup diff more than one vgroup check + self.checkVGroups() + + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/ts-4233.py b/tests/system-test/2-query/ts-4233.py new file mode 100644 index 0000000000..9b0a2f175c --- /dev/null +++ b/tests/system-test/2-query/ts-4233.py @@ -0,0 +1,46 @@ + +import taos + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), True) + + def checksql(self, sql): + result = os.popen("taos -s '%s'" %sql) + res = result.read() + print(res) + if ("Query OK" in res): + tdLog.info(f"checkEqual success") + else : + tdLog.exit(f"checkEqual error") + + def check(self): + conn = taos.connect() + sql = "select 'a;b' as x" + tdSql.query(f"%s" %sql) + tdSql.checkRows(1) + + self.checksql('select "a;b" as x\G') + self.checksql('select "a;b" as x >> /tmp/res.txt') + return + + def run(self): + tdSql.prepare() + self.check() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index 57415f8335..1c885b151c 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -80,6 +80,7 @@ typedef struct { #ifdef WEBSOCKET bool restful; bool cloud; + bool local; char* dsn; int32_t timeout; #endif diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 52cb524e3b..4817b23029 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -410,7 +410,7 @@ int32_t shellParseArgs(int32_t argc, char *argv[]) { shellInitArgs(argc, argv); shell.info.clientVersion = "Welcome to the %s Command Line Interface, Client Version:%s\r\n" - "Copyright (c) 2022 by %s, all rights reserved.\r\n\r\n"; + "Copyright (c) 2023 by %s, all rights reserved.\r\n\r\n"; #ifdef CUS_NAME strcpy(shell.info.cusName, CUS_NAME); #else diff --git a/tools/shell/src/shellAuto.c b/tools/shell/src/shellAuto.c index 60d6388faa..bd5329d810 100644 --- a/tools/shell/src/shellAuto.c +++ b/tools/shell/src/shellAuto.c @@ -83,6 +83,11 @@ SWords shellCommands[] = { {"alter local \"asynclog\" \"1\";", 0, 0, NULL}, {"alter topic", 0, 0, NULL}, {"alter user ;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE + {"balance vgroup;", 0, 0, NULL}, + {"balance vgroup leader ", 0, 0, NULL}, +#endif + // 20 {"create table using tags(", 0, 0, NULL}, {"create database " @@ -127,9 +132,12 @@ SWords shellCommands[] = { {"kill query ", 0, 0, NULL}, {"kill transaction ", 0, 0, NULL}, #ifdef TD_ENTERPRISE - {"merge vgroup ", 0, 0, NULL}, + {"merge vgroup ", 0, 0, NULL}, #endif {"pause stream ;", 0, 0, NULL}, +#ifdef TD_ENTERPRISE + {"redistribute vgroup dnode ;", 0, 0, NULL}, +#endif {"resume stream ;", 0, 0, NULL}, {"reset query cache;", 0, 0, NULL}, {"restore dnode ;", 0, 0, NULL}, @@ -187,7 +195,7 @@ SWords shellCommands[] = { {"show consumers;", 0, 0, NULL}, {"show grants;", 0, 0, NULL}, #ifdef TD_ENTERPRISE - {"split vgroup ", 0, 0, NULL}, + {"split vgroup ", 0, 0, NULL}, #endif {"insert into values(", 0, 0, NULL}, {"insert into using tags(", 0, 0, NULL}, @@ -268,7 +276,9 @@ char* db_options[] = {"keep ", "wal_retention_size ", "wal_segment_size "}; -char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "cachesize ", "wal_fsync_period ", "wal_level "}; +char* alter_db_options[] = {"cachemodel ", "replica ", "keep ", "stt_trigger ", + "wal_retention_period ", "wal_retention_size ", "cachesize ", + "wal_fsync_period ", "buffer ", "pages " ,"wal_level "}; char* data_types[] = {"timestamp", "int", "int unsigned", "varchar(16)", @@ -312,26 +322,27 @@ bool waitAutoFill = false; #define WT_VAR_TOPIC 5 #define WT_VAR_STREAM 6 #define WT_VAR_UDFNAME 7 +#define WT_VAR_VGROUPID 8 -#define WT_FROM_DB_MAX 7 // max get content from db +#define WT_FROM_DB_MAX 8 // max get content from db #define WT_FROM_DB_CNT (WT_FROM_DB_MAX + 1) -#define WT_VAR_ALLTABLE 8 -#define WT_VAR_FUNC 9 -#define WT_VAR_KEYWORD 10 -#define WT_VAR_TBACTION 11 -#define WT_VAR_DBOPTION 12 -#define WT_VAR_ALTER_DBOPTION 13 -#define WT_VAR_DATATYPE 14 -#define WT_VAR_KEYTAGS 15 -#define WT_VAR_ANYWORD 16 -#define WT_VAR_TBOPTION 17 -#define WT_VAR_USERACTION 18 -#define WT_VAR_KEYSELECT 19 -#define WT_VAR_SYSTABLE 20 -#define WT_VAR_LANGUAGE 21 +#define WT_VAR_ALLTABLE 9 +#define WT_VAR_FUNC 10 +#define WT_VAR_KEYWORD 11 +#define WT_VAR_TBACTION 12 +#define WT_VAR_DBOPTION 13 +#define WT_VAR_ALTER_DBOPTION 14 +#define WT_VAR_DATATYPE 15 +#define WT_VAR_KEYTAGS 16 +#define WT_VAR_ANYWORD 17 +#define WT_VAR_TBOPTION 18 +#define WT_VAR_USERACTION 19 +#define WT_VAR_KEYSELECT 20 +#define WT_VAR_SYSTABLE 21 +#define WT_VAR_LANGUAGE 22 -#define WT_VAR_CNT 22 +#define WT_VAR_CNT 23 #define WT_TEXT 0xFF @@ -345,11 +356,11 @@ TdThread* threads[WT_FROM_DB_CNT]; // obtain var name with sql from server char varTypes[WT_VAR_CNT][64] = { "", "", "", "", "", "", "", - "", "", "", "", "", "", "", + "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""}; char varSqls[WT_FROM_DB_CNT][64] = {"show databases;", "show stables;", "show tables;", "show dnodes;", - "show users;", "show topics;", "show streams;", "show functions;"}; + "show users;", "show topics;", "show streams;", "show functions;", "show vgroups;"}; // var words current cursor, if user press any one key except tab, cursorVar can be reset to -1 int cursorVar = -1; @@ -520,7 +531,10 @@ void showHelp() { printf( "\n\n\ ----- special commands on enterpise version ----- \n\ + balance vgroup; \n\ + balance vgroup leader \n\ compact database ; \n\ + redistribute vgroup dnode ;\n\ split vgroup ;"); #endif @@ -675,9 +689,9 @@ bool shellAutoInit() { // generate varType GenerateVarType(WT_VAR_FUNC, functions, sizeof(functions) / sizeof(char*)); GenerateVarType(WT_VAR_KEYWORD, keywords, sizeof(keywords) / sizeof(char*)); + GenerateVarType(WT_VAR_TBACTION, tb_actions, sizeof(tb_actions) / sizeof(char*)); GenerateVarType(WT_VAR_DBOPTION, db_options, sizeof(db_options) / sizeof(char*)); GenerateVarType(WT_VAR_ALTER_DBOPTION, alter_db_options, sizeof(alter_db_options) / sizeof(char*)); - GenerateVarType(WT_VAR_TBACTION, tb_actions, sizeof(tb_actions) / sizeof(char*)); GenerateVarType(WT_VAR_DATATYPE, data_types, sizeof(data_types) / sizeof(char*)); GenerateVarType(WT_VAR_KEYTAGS, key_tags, sizeof(key_tags) / sizeof(char*)); GenerateVarType(WT_VAR_TBOPTION, tb_options, sizeof(tb_options) / sizeof(char*)); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 3b150230e7..115abdcd36 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -62,6 +62,8 @@ static void shellCleanup(void *arg); static void *shellCancelHandler(void *arg); static void *shellThreadLoop(void *arg); +static bool shellCmdkilled = false; + bool shellIsEmptyCommand(const char *cmd) { for (char c = *cmd++; c != 0; c = *cmd++) { if (c != ' ' && c != '\t' && c != ';') { @@ -72,6 +74,8 @@ bool shellIsEmptyCommand(const char *cmd) { } int32_t shellRunSingleCommand(char *command) { + shellCmdkilled = false; + if (shellIsEmptyCommand(command)) { return 0; } @@ -199,22 +203,17 @@ void shellRunSingleCommandImp(char *command) { bool printMode = false; if ((sptr = strstr(command, ">>")) != NULL) { - cptr = strstr(command, ";"); - if (cptr != NULL) { - *cptr = '\0'; - } - fname = sptr + 2; while (*fname == ' ') fname++; *sptr = '\0'; - } - if ((sptr = strstr(command, "\\G")) != NULL) { - cptr = strstr(command, ";"); + cptr = strstr(fname, ";"); if (cptr != NULL) { *cptr = '\0'; } + } + if ((sptr = strstr(command, "\\G")) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. } @@ -262,7 +261,8 @@ void shellRunSingleCommandImp(char *command) { if (error_no == 0) { printf("Query OK, %"PRId64 " row(s) in set (%.6fs)\r\n", numOfRows, (et - st) / 1E6); } else { - printf("Query interrupted (%s), %"PRId64 " row(s) in set (%.6fs)\r\n", taos_errstr(pSql), numOfRows, (et - st) / 1E6); + terrno = error_no; + printf("Query interrupted (%s), %"PRId64 " row(s) in set (%.6fs)\r\n", taos_errstr(NULL), numOfRows, (et - st) / 1E6); } taos_free_result(pSql); } else { @@ -957,7 +957,11 @@ void shellDumpResultCallback(void *param, TAOS_RES *tres, int num_of_rows) { } } dump_info->numOfAllRows += num_of_rows; - taos_fetch_rows_a(tres, shellDumpResultCallback, param); + if (!shellCmdkilled) { + taos_fetch_rows_a(tres, shellDumpResultCallback, param); + } else { + tsem_post(&dump_info->sem); + } } else { if (num_of_rows < 0) { printf("\033[31masync retrieve failed, code: %d\033[0m\n", num_of_rows); @@ -972,13 +976,15 @@ int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool ver num_of_rows = shellDumpResultToFile(fname, tres); } else { tsDumpInfo dump_info; - init_dump_info(&dump_info, tres, sql, vertical); - taos_fetch_rows_a(tres, shellDumpResultCallback, &dump_info); - tsem_wait(&dump_info.sem); - num_of_rows = dump_info.numOfAllRows; + if (!shellCmdkilled) { + init_dump_info(&dump_info, tres, sql, vertical); + taos_fetch_rows_a(tres, shellDumpResultCallback, &dump_info); + tsem_wait(&dump_info.sem); + num_of_rows = dump_info.numOfAllRows; + } } - *error_no = taos_errno(tres); + *error_no = shellCmdkilled ? TSDB_CODE_TSC_QUERY_KILLED : taos_errno(tres); return num_of_rows; } @@ -1202,6 +1208,7 @@ void *shellCancelHandler(void *arg) { } else { #endif if (shell.conn) { + shellCmdkilled = true; taos_kill_query(shell.conn); } #ifdef WEBSOCKET diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index 795621dfdd..18f4ca21d1 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -47,6 +47,7 @@ int main(int argc, char *argv[]) { #ifdef WEBSOCKET shell.args.timeout = SHELL_WS_TIMEOUT; shell.args.cloud = true; + shell.args.local = false; #endif #if !defined(WINDOWS) diff --git a/tools/shell/src/shellUtil.c b/tools/shell/src/shellUtil.c index 93451c85a9..0350d934a7 100644 --- a/tools/shell/src/shellUtil.c +++ b/tools/shell/src/shellUtil.c @@ -130,11 +130,21 @@ void shellCheckConnectMode() { } if (shell.args.cloud) { shell.args.dsn = getenv("TDENGINE_CLOUD_DSN"); - if (shell.args.dsn) { + if (shell.args.dsn && strlen(shell.args.dsn) > 4) { shell.args.cloud = true; + shell.args.local = false; shell.args.restful = false; return; } + + shell.args.dsn = getenv("TDENGINE_DSN"); + if (shell.args.dsn && strlen(shell.args.dsn) > 4) { + shell.args.cloud = true; + shell.args.local = true; + shell.args.restful = false; + return; + } + if (shell.args.restful) { if (!shell.args.host) { shell.args.host = "localhost"; diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index e83ceff099..bff2ef7592 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -59,7 +59,17 @@ int shell_conn_ws_server(bool first) { fprintf(stdout, "successfully connected to %s\n\n", shell.args.dsn); } else if (first && shell.args.cloud) { - fprintf(stdout, "successfully connected to cloud service\n"); + if(shell.args.local) { + const char* host = strstr(shell.args.dsn, "@"); + if(host) { + host += 1; + } else { + host = shell.args.dsn; + } + fprintf(stdout, "successfully connected to %s\n", host); + } else { + fprintf(stdout, "successfully connected to cloud service\n"); + } } fflush(stdout); @@ -236,22 +246,17 @@ void shellRunSingleCommandWebsocketImp(char *command) { bool printMode = false; if ((sptr = strstr(command, ">>")) != NULL) { - cptr = strstr(command, ";"); - if (cptr != NULL) { - *cptr = '\0'; - } - fname = sptr + 2; while (*fname == ' ') fname++; *sptr = '\0'; - } - if ((sptr = strstr(command, "\\G")) != NULL) { - cptr = strstr(command, ";"); + cptr = strstr(fname, ";"); if (cptr != NULL) { *cptr = '\0'; } + } + if ((sptr = strstr(command, "\\G")) != NULL) { *sptr = '\0'; printMode = true; // When output to a file, the switch does not work. }