diff --git a/.travis.yml b/.travis.yml index 4afdb6154e..6e49709c85 100644 --- a/.travis.yml +++ b/.travis.yml @@ -190,7 +190,7 @@ matrix: script: - cmake .. > /dev/null - - make > /dev/null + - make > /dev/null - os: linux arch: arm64 diff --git a/CMakeLists.txt b/CMakeLists.txt index 8dd99f0b58..bc6a888f9d 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,14 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) +IF (CMAKE_VERSION VERSION_LESS 3.0) + PROJECT(TDengine CXX) + SET(PROJECT_VERSION_MAJOR "${LIB_MAJOR_VERSION}") + SET(PROJECT_VERSION_MINOR "${LIB_MINOR_VERSION}") + SET(PROJECT_VERSION_PATCH"${LIB_PATCH_VERSION}") + SET(PROJECT_VERSION "${LIB_VERSION_STRING}") +ELSE () + CMAKE_POLICY(SET CMP0048 NEW) + PROJECT(TDengine VERSION "${LIB_VERSION_STRING}" LANGUAGES CXX) +ENDIF () SET(TD_ACCOUNT FALSE) SET(TD_ADMIN FALSE) diff --git a/cmake/define.inc b/cmake/define.inc index 1bb1692a2b..f6438c0873 100755 --- a/cmake/define.inc +++ b/cmake/define.inc @@ -53,14 +53,18 @@ IF (TD_LINUX_64) ADD_DEFINITIONS(-D_M_X64) ADD_DEFINITIONS(-D_TD_LINUX_64) IF (NOT TD_ARM) - IF (${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ELSE () - SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") - ENDIF () + #IF ((${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") OR (${CMAKE_CXX_COMPILER_ID} MATCHES "clang")) + # SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -Wno-missing-braces -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + #ELSE () + # SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -malign-double -g3 -gdwarf-2 -malign-stringops -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") + #ENDIF () + SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g3 -gdwarf-2 -msse4.2 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ELSE () SET(COMMON_FLAGS "-std=gnu99 -Wall -Werror -fPIC -g -fsigned-char -fpack-struct=8 -D_FILE_OFFSET_BITS=64 -D_LARGE_FILE") ENDIF () + MESSAGE(STATUS "CMAKE_CXX_COMPILER_ID: " ${CMAKE_CXX_COMPILER_ID}) + MESSAGE(STATUS "COMMON_FLAGS: " ${COMMON_FLAGS}) + FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) ADD_DEFINITIONS(-DUSE_LIBICONV) diff --git a/cmake/platform.inc b/cmake/platform.inc index a0668de7b5..488786b178 100755 --- a/cmake/platform.inc +++ b/cmake/platform.inc @@ -52,14 +52,16 @@ ELSE () MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) ENDIF () -# -# Get OS information and store in variable TD_OS_INFO. -# -execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh) -execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO) -MESSAGE(STATUS "The current os is " ${TD_OS_INFO}) + IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux") + # + # Get OS information and store in variable TD_OS_INFO. + # + execute_process(COMMAND chmod 777 ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh) + execute_process(COMMAND ${TD_COMMUNITY_DIR}/packaging/tools/get_os.sh "" OUTPUT_VARIABLE TD_OS_INFO) + MESSAGE(STATUS "The current os is " ${TD_OS_INFO}) + SET(TD_LINUX TRUE) IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8) SET(TD_LINUX_64 TRUE) diff --git a/documentation20/webdocs/assets/dnode.png b/documentation20/webdocs/assets/dnode.png new file mode 100644 index 0000000000..cea87dcccb Binary files /dev/null and b/documentation20/webdocs/assets/dnode.png differ diff --git a/documentation20/webdocs/assets/modules.png b/documentation20/webdocs/assets/modules.png new file mode 100644 index 0000000000..10ae4703a6 Binary files /dev/null and b/documentation20/webdocs/assets/modules.png differ diff --git a/documentation20/webdocs/assets/replica-forward.jpg b/documentation20/webdocs/assets/replica-forward.jpg new file mode 100644 index 0000000000..00b8c357b9 Binary files /dev/null and b/documentation20/webdocs/assets/replica-forward.jpg differ diff --git a/documentation20/webdocs/assets/replica-master.png b/documentation20/webdocs/assets/replica-master.png new file mode 100644 index 0000000000..cb33f1ce98 Binary files /dev/null and b/documentation20/webdocs/assets/replica-master.png differ diff --git a/documentation20/webdocs/assets/replica-restore.jpg b/documentation20/webdocs/assets/replica-restore.jpg new file mode 100644 index 0000000000..6da1bed4cc Binary files /dev/null and b/documentation20/webdocs/assets/replica-restore.jpg differ diff --git a/documentation20/webdocs/assets/vnode.png b/documentation20/webdocs/assets/vnode.png index 5247717f62..e6148d4907 100644 Binary files a/documentation20/webdocs/assets/vnode.png and b/documentation20/webdocs/assets/vnode.png differ diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md index b6dbad07dc..12b1b1bd3a 100644 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md @@ -1,117 +1,128 @@ -#TDengine文档 +# TDengine文档 -TDengine是一个高效的存储、查询、分析时序大数据的平台，专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它，但建议您在使用前仔细阅读一遍下面的文档，特别是[数据模型](data-model-and-architecture)与数据建模一节。除本文档之外，欢迎[下载产品白皮书](https://www.taosdata.com/downloads/TDengine%20White%20Paper.pdf)。 +TDengine是一个高效的存储、查询、分析时序大数据的平台，专为物联网、车联网、工业互联网、运维监测等优化而设计。您可以像使用关系型数据库MySQL一样来使用它，但建议您在使用前仔细阅读一遍下面的文档，特别是[数据模型](https://www.taosdata.com/cn/documentation20/data-model-and-architecture)与[数据建模](https://www.taosdata.com/cn/documentation20/model)一节。除本文档之外，欢迎[下载产品白皮书](https://www.taosdata.com/downloads/TDengine White Paper.pdf)。如需查阅TDengine 1.6 文档，请点击[这里](https://www.taosdata.com/cn/documentation16/)访问。 + +## TDengine介绍 -##TDengine 介绍 - TDengine 简介及特色 - TDengine 适用场景 -- TDengine 性能指标介绍和验证方法 -##立即开始 -- 快捷安装：可通过源码、安装包或docker安装，三秒钟搞定 -- 轻松启动：使用systemctl 启停TDengine -- 命令行程序TAOS：访问TDengine的简便方式 -- [极速体验](https://www.taosdata.com/cn/getting-started/#TDengine-极速体验)：运行示例程序，快速体验高效的数据插入、查询 +## [立即开始](https://www.taosdata.com/cn/getting-started) -##数据模型和整体架构 -- 数据模型：关系型数据库模型，但要求每个采集点单独建表 -- 集群与基本逻辑单元：吸取NoSQL优点，支持水平扩展，支持高可靠 -- 存储模型与数据分区：标签数据与时序数据完全分离，按vnode和时间两个维度对数据切分 -- 数据写入与复制流程：先写入WAL、之后写入缓存，再给应用确认，支持多副本 -- 缓存与持久化：最新数据缓存在内存中，但落盘时采用列式存储、超高压缩比 -- 高效查询：支持各种函数、时间轴聚合、插值、多表聚合 +- [快捷安装](https://www.taosdata.com/cn/documentation20/getting-started/#快捷安装)：可通过源码、安装包或docker安装，三秒钟搞定 +- [轻松启动](https://www.taosdata.com/cn/documentation20/getting-started/#轻松启动)：使用systemctl 启停TDengine +- [命令行程序TAOS](https://www.taosdata.com/cn/documentation20/getting-started/#TDengine命令行程序)：访问TDengine的简便方式 +- [极速体验](https://www.taosdata.com/cn/documentation20/getting-started/#TDengine-极速体验)：运行示例程序，快速体验高效的数据插入、查询 -##数据建模 -- 创建库：为具有相似数据特征的数据采集点创建一个库 -- 创建超级表：为同一类型的数据采集点创建一个超级表 -- 创建表：使用超级表做模板，为每一个具体的数据采集点单独建表 +## [数据模型和整体架构](https://www.taosdata.com/cn/documentation20/architecture) -##高效写入数据 -- SQL写入：使用SQL insert命令向一张或多张表写入单条或多条记录 -- Telegraf 写入：配置Telegraf, 不用任何代码，将采集数据直接写入 -- Prometheus写入：配置Prometheus, 不用任何代码，将数据直接写入 -- EMQ X Broker：配置EMQ X，不用任何代码，就可将MQTT数据直接写入 +- [数据模型](https://www.taosdata.com/cn/documentation20/architecture/#数据模型)：关系型数据库模型，但要求每个采集点单独建表 +- [集群与基本逻辑单元](https://www.taosdata.com/cn/documentation20/architecture/#集群与基本逻辑单元)：吸取NoSQL优点，支持水平扩展，支持高可靠 +- [存储模型与数据分区、分片](https://www.taosdata.com/cn/documentation20/architecture/#存储模型与数据分区、分片)：标签数据与时序数据完全分离，按vnode和时间两个维度对数据切分 +- [数据写入与复制流程](https://www.taosdata.com/cn/documentation20/architecture/#数据写入与复制流程)：先写入WAL、之后写入缓存，再给应用确认，支持多副本 +- [缓存与持久化](https://www.taosdata.com/cn/documentation20/architecture/#缓存与持久化)：最新数据缓存在内存中，但落盘时采用列式存储、超高压缩比 +- [数据查询](https://www.taosdata.com/cn/documentation20/architecture/#数据查询)：支持各种函数、时间轴聚合、插值、多表聚合 -##高效查询数据 -- 主要查询功能：支持各种标准函数，设置过滤条件，时间段查询 -- 多表聚合查询：使用超级表，设置标签过滤条件，进行高效聚合查询 -- 降采样查询：按时间段分段聚合，支持插值 +## [数据建模](https://www.taosdata.com/cn/documentation20/model) -##高级功能 -- 连续查询(Continuous Query)：基于滑动窗口，定时自动的对数据流进行查询计算 -- 数据订阅(Publisher/Subscriber)：象典型的消息队列，应用可订阅接收到的最新数据 -- [缓存 (Cache)](https://www.taosdata.com/cn/documentation/advanced-features/#缓存-(Cache))：每个设备最新的数据都会缓存在内存中，可快速获取 -- [报警监测(Alarm monitoring)](https://www.taosdata.com/blog/2020/04/14/1438.html/)：根据配置规则，自动监测超限行为数据，并主动推送 +- [创建库](https://www.taosdata.com/cn/documentation20/model/#创建库)：为具有相似数据特征的数据采集点创建一个库 +- [创建超级表](https://www.taosdata.com/cn/documentation20/model/#创建超级表)：为同一类型的数据采集点创建一个超级表 +- [创建表](https://www.taosdata.com/cn/documentation20/model/#创建表)：使用超级表做模板，为每一个具体的数据采集点单独建表 -##连接器 -- C/C++ Connector：通过libtaos客户端的库，连接TDengine服务器的主要方法 -- Java Connector(JDBC)：通过标准的JDBC API，给Java应用提供到TDengine的连接 -- Python Connector：给Python应用提供一个连接TDengine服务器的驱动 -- RESTful Connector：提供一最简单的连接TDengine服务器的方式 -- Go Connector：给Go应用提供一个连接TDengine服务器的驱动 -- Node.js Connector：给node应用提供一个链接TDengine服务器的驱动 +## [高效写入数据](https://www.taosdata.com/cn/documentation20/insert) -##与其他工具的连接 -- Grafana：获取并可视化保存在TDengine的数据 -- Matlab：通过配置Matlab的JDBC数据源访问保存在TDengine的数据 -- R：通过配置R的JDBC数据源访问保存在TDengine的数据 +- [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入)：使用SQL insert命令向一张或多张表写入单条或多条记录 +- [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入)：配置Telegraf, 不用任何代码，将采集数据直接写入 +- [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入)：配置Prometheus, 不用任何代码，将数据直接写入 +- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将MQTT数据直接写入 -## TDengine集群的安装、管理 +## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries) -- 安装：与单节点的安装一样，但要设好配置文件里的参数first -- 节点管理：增加、删除、查看集群的节点 -- mnode的管理：系统自动创建、无需任何人工干预 -- 负载均衡：一旦节点个数或负载有变化，自动进行 -- 节点离线处理：节点离线超过一定时长，将从集群中剔除 -- Arbitrator：对于偶数个副本的情形，使用它可以防止split brain。 +- [主要查询功能](https://www.taosdata.com/cn/documentation20/queries/#主要查询功能)：支持各种标准函数，设置过滤条件，时间段查询 +- [多表聚合查询](https://www.taosdata.com/cn/documentation20/queries/#多表聚合查询)：使用超级表，设置标签过滤条件，进行高效聚合查询 +- [降采样查询值](https://www.taosdata.com/cn/documentation20/queries/#降采样查询、插值)：按时间段分段聚合，支持插值 -##TDengine的运营和维护 +## [高级功能](https://www.taosdata.com/cn/documentation20/advanced-features) -- 容量规划：根据场景，估算硬件资源 -- 容错和灾备：设置正确的WAL和数据副本数 -- 系统配置：端口，缓存大小，文件块大小和其他系统配置 -- 用户管理：添加、删除TDengine用户，修改用户密码 -- 数据导入：可按脚本文件导入，也可按数据文件导入 -- 数据导出：从shell按表导出，也可用taosdump工具做各种导出 -- 系统监控：检查系统现有的连接、查询、流式计算，日志和事件等 -- 文件目录结构：TDengine数据文件、配置文件等所在目录 Hui Li +- [连续查询(Continuous Query)](https://www.taosdata.com/cn/documentation20/advanced-features/#连续查询(Continuous-Query))：基于滑动窗口，定时自动的对数据流进行查询计算 +- [数据订阅(Publisher/Subscriber)](https://www.taosdata.com/cn/documentation20/advanced-features/#数据订阅(Publisher/Subscriber))：象典型的消息队列，应用可订阅接收到的最新数据 +- [缓存(Cache)](https://www.taosdata.com/cn/documentation20/advanced-features/#缓存(Cache))：每个设备最新的数据都会缓存在内存中，可快速获取 +- [报警监测](https://www.taosdata.com/cn/documentation20/advanced-features/#报警监测(Alert))：根据配置规则，自动监测超限行为数据，并主动推送 -##TAOS SQL -- 支持的数据类型：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 -- 数据库管理：添加、删除、查看数据库 -- 表管理：添加、删除、查看、修改表 -- 超级表管理：添加、删除、查看、修改超级表 -- 标签管理：增加、删除、修改标签 -- 数据写入：支持单表单条、多条、多表多条写入，支持历史数据写入 -- 数据查询：支持时间段、值过滤、排序、查询结果手动分页等 -- SQL函数：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等 -- 时间维度聚合：将表中数据按照时间段进行切割后聚合，降维处理 +## [连接器](https://www.taosdata.com/cn/documentation20/connector) + +- [C/C++ Connector](https://www.taosdata.com/cn/documentation20/connector/#C/C++-Connector)：通过libtaos客户端的库，连接TDengine服务器的主要方法 +- [Java Connector(JDBC)](https://www.taosdata.com/cn/documentation20/connector/#Java-Connector)：通过标准的JDBC API，给Java应用提供到TDengine的连接 +- [Python Connector](https://www.taosdata.com/cn/documentation20/connector/#Python-Connector)：给Python应用提供一个连接TDengine服务器的驱动 +- [RESTful Connector](https://www.taosdata.com/cn/documentation20/connector/#RESTful-Connector)：提供一最简单的连接TDengine服务器的方式 +- [Go Connector](https://www.taosdata.com/cn/documentation20/connector/#Go-Connector)：给Go应用提供一个连接TDengine服务器的驱动 +- [Node.js Connector](https://www.taosdata.com/cn/documentation20/connector/#Node.js-Connector)：给node应用提供一个链接TDengine服务器的驱动 + +## [与其他工具的连接](https://www.taosdata.com/cn/documentation20/connections-with-other-tools) + +- [Grafana](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#Grafana)：获取并可视化保存在TDengine的数据 +- [Matlab](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#Matlab)：通过配置Matlab的JDBC数据源访问保存在TDengine的数据 +- [R](https://www.taosdata.com/cn/documentation20/connections-with-other-tools/#R)：通过配置R的JDBC数据源访问保存在TDengine的数据 + +## [TDengine集群的安装、管理](https://www.taosdata.com/cn/documentation20/cluster) + +- [安装](https://www.taosdata.com/cn/documentation20/cluster/#创建第一个节点)：与单节点的安装一样，但要设好配置文件里的参数first +- [节点管理](https://www.taosdata.com/cn/documentation20/cluster/#节点管理)：增加、删除、查看集群的节点 +- [mnode的管理](https://www.taosdata.com/cn/documentation20/cluster/#Mnode的高可用)：系统自动创建、无需任何人工干预 +- [负载均衡](https://www.taosdata.com/cn/documentation20/cluster/#负载均衡)：一旦节点个数或负载有变化，自动进行 +- [节点离线处理](https://www.taosdata.com/cn/documentation20/cluster/#节点离线处理)：节点离线超过一定时长，将从集群中剔除 +- [Arbitrator](https://www.taosdata.com/cn/documentation20/cluster/#Arbitrator的使用)：对于偶数个副本的情形，使用它可以防止split brain + +## [TDengine的运营和维护](https://www.taosdata.com/cn/documentation20/administrator) + +- [容量规划](https://www.taosdata.com/cn/documentation20/administrator/#容量规划)：根据场景，估算硬件资源 +- [容错和灾备](https://www.taosdata.com/cn/documentation20/administrator/#容错和灾备)：设置正确的WAL和数据副本数 +- [系统配置](https://www.taosdata.com/cn/documentation20/administrator/#服务端配置)：端口，缓存大小，文件块大小和其他系统配置 +- [用户管理](https://www.taosdata.com/cn/documentation20/administrator/#用户管理)：添加、删除TDengine用户，修改用户密码 +- [数据导入](https://www.taosdata.com/cn/documentation20/administrator/#数据导入)：可按脚本文件导入，也可按数据文件导入 +- [数据导出](https://www.taosdata.com/cn/documentation20/administrator/#数据导出)：从shell按表导出，也可用taosdump工具做各种导出 +- [系统监控](https://www.taosdata.com/cn/documentation20/administrator/#系统监控)：检查系统现有的连接、查询、流式计算，日志和事件等 +- [文件目录结构](https://www.taosdata.com/cn/documentation20/administrator/#文件目录结构)：TDengine数据文件、配置文件等所在目录 + +## [TAOS SQL](https://www.taosdata.com/cn/documentation20/taos-sql) + +- [支持的数据类型](https://www.taosdata.com/cn/documentation20/taos-sql/#支持的数据类型)：支持时间戳、整型、浮点型、布尔型、字符型等多种数据类型 +- [数据库管理](https://www.taosdata.com/cn/documentation20/taos-sql/#数据库管理)：添加、删除、查看数据库 +- [表管理](https://www.taosdata.com/cn/documentation20/taos-sql/#表管理)：添加、删除、查看、修改表 +- [超级表管理](https://www.taosdata.com/cn/documentation20/taos-sql/#超级表STable管理)：添加、删除、查看、修改超级表 +- [标签管理](https://www.taosdata.com/cn/documentation20/taos-sql/#超级表-STable-中-TAG-管理)：增加、删除、修改标签 +- [数据写入](https://www.taosdata.com/cn/documentation20/taos-sql/#数据写入)：支持单表单条、多条、多表多条写入，支持历史数据写入 +- [数据查询](https://www.taosdata.com/cn/documentation20/taos-sql/#数据查询)：支持时间段、值过滤、排序、查询结果手动分页等 +- [SQL函数](https://www.taosdata.com/cn/documentation20/taos-sql/#SQL函数)：支持各种聚合函数、选择函数、计算函数，如avg, min, diff等 +- [时间维度聚合](https://www.taosdata.com/cn/documentation20/taos-sql/#时间维度聚合)：将表中数据按照时间段进行切割后聚合，降维处理 + +## TDengine的技术设计 -##TDengine的技术设计 - 系统模块：taosd的功能和模块划分 -- 技术博客：更多的技术分析和架构设计文章 +- 数据复制：支持实时同步、异步复制，保证系统的High Availibility +- [技术博客](https://www.taosdata.com/cn/blog/?categories=3)：更多的技术分析和架构设计文章 ## 常用工具 -- [TDengine样例数据导入工具](https://www.taosdata.com/cn/documentation/blog/2020/01/18/如何快速验证性能和主要功能？tdengine样例数据导入工/) -- [TDengine性能对比测试工具](https://www.taosdata.com/cn/documentation/blog/2020/01/13/用influxdb开源的性能测试工具对比influxdb和tdengine/) +- [TDengine样例导入工具](https://www.taosdata.com/blog/2020/01/18/1166.html) +- [TDengine性能对比测试工具](https://www.taosdata.com/blog/2020/01/18/1166.html) -##TDengine与其他数据库的对比测试 +## TDengine与其他数据库的对比测试 -- [用InfluxDB开源的性能测试工具对比InfluxDB和TDengine](https://www.taosdata.com/cn/documentation/blog/2020/01/13/用influxdb开源的性能测试工具对比influxdb和tdengine/) -- [TDengine与OpenTSDB对比测试](https://www.taosdata.com/cn/documentation/blog/2019/08/21/tdengine与opentsdb对比测试/) -- [TDengine与Cassandra对比测试](https://www.taosdata.com/cn/documentation/blog/2019/08/14/tdengine与cassandra对比测试/) -- [TDengine与InfluxDB对比测试](https://www.taosdata.com/cn/documentation/blog/2019/07/19/tdengine与influxdb对比测试/) +- [用InfluxDB开源的性能测试工具对比InfluxDB和TDengine](https://www.taosdata.com/blog/2020/01/13/1105.html) +- [TDengine与OpenTSDB对比测试](https://www.taosdata.com/blog/2019/08/21/621.html) +- [TDengine与Cassandra对比测试](https://www.taosdata.com/blog/2019/08/14/573.html) +- [TDengine与InfluxDB对比测试](https://www.taosdata.com/blog/2019/07/19/419.html) - [TDengine与InfluxDB、OpenTSDB、Cassandra、MySQL、ClickHouse等数据库的对比测试报告](https://www.taosdata.com/downloads/TDengine_Testing_Report_cn.pdf) ##物联网大数据 -- [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/物联网、工业互联网大数据的特点/) -- [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/物联网大数据平台应具备的功能和特点/) -- [通用大数据架构为什么不适合处理物联网数据？](https://www.taosdata.com/blog/2019/07/09/通用互联网大数据处理架构为什么不适合处理物联/) -- [物联网、车联网、工业互联网大数据平台，为什么推荐使用TDengine？](https://www.taosdata.com/blog/2019/07/09/物联网、车联网、工业互联网大数据平台，为什么/) -##培训和FAQ -- FAQ：常见问题与答案 -- 应用案列：一些使用实例来解释如何使用TDengine +- [物联网、工业互联网大数据的特点](https://www.taosdata.com/blog/2019/07/09/105.html) +- [物联网大数据平台应具备的功能和特点](https://www.taosdata.com/blog/2019/07/29/542.html) +- [通用大数据架构为什么不适合处理物联网数据？](https://www.taosdata.com/blog/2019/07/09/107.html) +- [物联网、车联网、工业互联网大数据平台，为什么推荐使用TDengine？](https://www.taosdata.com/blog/2019/07/09/109.html) - \ No newline at end of file +## [培训和FAQ](https://www.taosdata.com/cn/faq) + +- [FAQ](https://www.taosdata.com/cn/documentation20/faq)：常见问题与答案 +- [应用案列](https://www.taosdata.com/cn/blog/?categories=4)：一些使用实例来解释如何使用TDengine \ No newline at end of file diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/webdocs/markdowndocs/Evaluation-ch.md index 411cb2dfb9..5b501b7f9f 100644 --- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md +++ b/documentation20/webdocs/markdowndocs/Evaluation-ch.md @@ -1,6 +1,7 @@ # TDengine 介绍 ## TDengine 简介 + TDengine是涛思数据面对高速增长的物联网大数据市场和技术挑战推出的创新性的大数据处理产品，它不依赖任何第三方软件，也不是优化或包装了一个开源的数据库或流式计算产品，而是在吸取众多传统关系型数据库、NoSQL数据库、流式计算引擎、消息队列等软件的优点之后自主开发的产品，在时序空间大数据处理上，有着自己独到的优势。 TDengine的模块之一是时序数据库。但除此之外，为减少研发的复杂度、系统维护的难度，TDengine还提供缓存、消息队列、订阅、流式计算等功能，为物联网、工业互联网大数据的处理提供全栈的技术方案，是一个高效易用的物联网大数据平台。与Hadoop等典型的大数据平台相比，它具有如下鲜明的特点： @@ -15,46 +16,45 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 ## TDengine 总体适用场景 -作为一个IoT大数据平台，TDengine的典型适用场景是在IoT范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如CRM，ERP等，不在本文讨论范围内。 + +作为一个IOT大数据平台，TDengine的典型适用场景是在IOT范畴，而且用户有一定的数据量。本文后续的介绍主要针对这个范畴里面的系统。范畴之外的系统，比如CRM，ERP等，不在本文讨论范围内。 + ### 数据源特点和需求 -从数据源角度，设计人员可以从如下几个角度分析TDengine在目标应用系统里面的适用性。 +从数据源角度，设计人员可以从已经角度分析TDengine在目标应用系统里面的适用性。 |数据源特点和需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|总体数据量巨大| | | ✅ |TDengine在容量方面提供出色的水平扩展功能，并且具备匹配高压缩的存储结构和多级存储，达到业界最优的存储效率。| -|数据输入速度偶尔或者持续巨大| | | ✅ | TDengine的性能大大超过同类产品，可以在同样的硬件环境下持续处理大量的输入数据，并且提供很容易在用户环境里面运行的性能评估工具。| -|数据源数目巨大| | | ✅ |TDengine设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。| +|总体数据量巨大| | | √ |TDengine在容量方面提供出色的水平扩展功能，并且具备匹配高压缩的存储结构，达到业界最优的存储效率。| +|数据输入速度偶尔或者持续巨大| | | √ | TDengine的性能大大超过同类产品，可以在同样的硬件环境下持续处理大量的输入数据，并且提供很容易在用户环境里面运行的性能评估工具。| +|数据源数目巨大| | | √ |TDengine设计中包含专门针对大量数据源的优化，包括数据的写入和查询，尤其适合高效处理海量（千万或者更多量级）的数据源。| -###系统架构要求 +### 系统架构要求 |系统架构要求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求简单可靠的系统架构| | | ✅ |TDengine的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。| -|要求容错和高可靠| | | ✅ |TDengine的集群功能，自动提供容错灾备等高可靠功能| -|标准化规范| | | ✅ |TDengine使用标准的SQL语言提供主要功能，遵守标准化规范| +|要求简单可靠的系统架构| | | √ |TDengine的系统架构非常简单可靠，自带消息队列，缓存，流式计算，监控等功能，无需集成额外的第三方产品。| +|要求容错和高可靠| | | √ |TDengine的集群功能，自动提供容错灾备等高可靠功能| +|标准化规范| | | √ |TDengine使用标准的SQL语言提供主要功能，遵守标准化规范| -###系统功能需求 +### 系统功能需求 |系统功能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求完整的内置数据处理算法| | ✅ | |TDengine的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。| -|需要大量的交叉查询处理| | ✅ | |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑TDengine和关系型数据系统配合实现系统功能| +|要求完整的内置数据处理算法| | √ | |TDengine的实现了通用的数据处理算法，但是还没有做到妥善处理各行各业的所有要求，因此特殊类型的处理还需要应用层面处理。| +|需要大量的交叉查询处理| | √ | |这种类型的处理更多应该用关系型数据系统处理，或者应该考虑TDengine和关系型数据系统配合实现系统功能| -###系统性能需求 +### 系统性能需求 |系统性能需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求较大的总体处理能力| | | ✅ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。| -|要求高速处理数据 | | | ✅ |TDengine的专门为IOT优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。| -|要求快速处理小粒度数据| | | ✅ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。| +|要求较大的总体处理能力| | | √ |TDengine的集群功能可以轻松地让多服务器配合达成处理能力的提升。| +|要求高速处理数据 | | | √ |TDengine的专门为IOT优化的存储和数据处理的设计，一般可以让系统得到超出同类产品多倍数的处理速度提升。| +|要求快速处理小粒度数据| | | √ |这方面TDengine性能可以完全对标关系型和NoSQL型数据处理系统。| -###系统维护需求 +### 系统维护需求 |系统维护需求|不适用|可能适用|非常适用|简单说明| |---|---|---|---|---| -|要求系统可靠运行| | | ✅ |TDengine的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。| -|要求运维学习成本可控| | | ✅ |同上| -|要求市场有大量人才储备| ✅ | | |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务| - -##TDengine 性能指标介绍和验证方法 - - +|要求系统可靠运行| | | √ |TDengine的系统架构非常稳定可靠，日常维护也简单便捷，对维护人员的要求简洁明了，最大程度上杜绝人为错误和事故。| +|要求运维学习成本可控| | | √ |同上| +|要求市场有大量人才储备| √ | | |TDengine作为新一代产品，目前人才市场里面有经验的人员还有限。但是学习成本低，我们作为厂家也提供运维的培训和辅助服务| +## TDengine 性能指标介绍和验证方法 diff --git a/documentation20/webdocs/markdowndocs/Getting Started-ch.md b/documentation20/webdocs/markdowndocs/Getting Started-ch.md index 4b109aa8e6..041ad72ac5 100644 --- a/documentation20/webdocs/markdowndocs/Getting Started-ch.md +++ b/documentation20/webdocs/markdowndocs/Getting Started-ch.md @@ -38,7 +38,7 @@ which systemd 如果系统中不存在`systemd`命令，请考虑[通过源码安装](#通过源码安装)TDengine。 -具体的安装过程，请参见[`TDengine多种安装包的安装和卸载`](https://www.taosdata.com/blog/2019/08/09/566.html) +具体的安装过程，请参见TDengine多种安装包的安装和卸载。 ## 轻松启动 diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index dfb079d13d..8a8d4191ed 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -4,14 +4,14 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个具体的应用场景，需要考虑库的设计，超级表和普通表的设计。本节不讨论细致的语法规则，只介绍概念。 -##创建库 +## 创建库 不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小等等。为让各种场景下TDengine都能最大效率的工作，TDengine建议将不同数据特征的表创建在不同的库里，因为每个库可以配置不同的存储策略。创建一个库时，除SQL标准的选项外，应用还可以指定保留时长、副本数、内存块个数、时间精度、文件块里最大最小记录条数、是否压缩、一个数据文件覆盖的天数等多种参数。比如： ```cmd CREATE DATABASE power KEEP 365 DAYS 10 REPLICA 3 BLOCKS 4; ``` -上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，副本数为3, 内存块数为4。详细的语法及参数请见TAOS SQL。 +上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，副本数为3, 内存块数为4。详细的语法及参数请见TAOS SQL 注意：任何一张表或超级表是属于一个库的，在创建表之前，必须先创建库。 @@ -20,7 +20,7 @@ CREATE DATABASE power KEEP 365 DAYS 10 REPLICA 3 BLOCKS 4; ```cmd CREATE TABLE meters (ts timestamp, current float, voltage int, phase float) TAGS (location binary(64), groupdId int); ``` -与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义，为采集的物理量（示例中为ts, current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL一节。 +与创建普通表一样，创建表时，需要提供表名（示例中为meters），表结构Schema，即数据列的定义，为采集的物理量（示例中为ts, current, voltage, phase)，数据类型可以为整型、浮点型、字符串等。除此之外，还需要提供标签的schema (示例中为location, groupId)，标签的数据类型可以为整型、浮点型、字符串等。采集点的静态属性往往可以作为标签，比如采集点的地理位置、设备型号、设备组ID、管理员ID等等。标签的schema可以事后增加、删除、修改。具体定义以及细节请见 TAOS SQL 一节。 每一种类型的数据采集点需要建立一个超级表，因此一个物联网系统，往往会有多个超级表。一个系统可以有多个DB，一个DB里可以有一到多个超级表。 @@ -31,7 +31,7 @@ CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2); ``` 其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 TAOS SQL。 -TDengine建议将数据采集点的全局唯一ID作为表名。但对于有的场景，并没有唯一的ID，可以将多个ID组合成一个唯一的ID。不建议将具有唯一性的ID作为标签值。 +TDengine建议将数据采集点的全局唯一ID作为表名(比如设备序列号）。但对于有的场景，并没有唯一的ID，可以将多个ID组合成一个唯一的ID。不建议将具有唯一性的ID作为标签值。 **自动建表**：在某些特殊场景中，用户在写数据时并不确定某个数据采集点的表是否存在，此时可在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。比如： @@ -40,4 +40,5 @@ INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 21 ``` 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。 -**多列模型**：TDengine支持多列模型，只要这些物理量是同时采集的，这些量就可以作为不同列放在同一张表里。有的数据采集点有多组采集量，每一组的数据采集时间是不一样的，这时需要对同一个采集点建多张表。但还有一种极限的设计，单列模型，无论是否同时采集，每个采集的物理量单独建表。TDengine建议，只要采集时间一致，就采用多列模型，因为插入效率以及存储效率更高。 \ No newline at end of file +**多列模型**：TDengine支持多列模型，只要这些物理量是同时采集的，这些量就可以作为不同列放在同一张表里。有的数据采集点有多组采集量，每一组的数据采集时间是不一样的，这时需要对同一个采集点建多张表。但还有一种极限的设计，单列模型，无论是否同时采集，每个采集的物理量单独建表。TDengine建议，只要采集时间一致，就采用多列模型，因为插入效率以及存储效率更高。 + diff --git a/documentation20/webdocs/markdowndocs/Queries-ch.md b/documentation20/webdocs/markdowndocs/Queries-ch.md index 0f4124f1fc..5cc0779551 100644 --- a/documentation20/webdocs/markdowndocs/Queries-ch.md +++ b/documentation20/webdocs/markdowndocs/Queries-ch.md @@ -6,17 +6,16 @@ ## 主要查询功能 -TDengine采用SQL作为查询语言，应用程序可以通过C/C++, JDBC, GO, Python连接器发送SQL查询语句，用户还可以通过TAOS Shell直接手动执行SQL即席查询，十分方便。支持如下查询功能： +TDengine 采用 SQL 作为查询语言。应用程序可以通过 C/C++, Java, Go, Python 连接器发送 SQL 语句，用户可以通过 TDengine 提供的命令行（Command Line Interface, CLI）工具 TAOS Shell 手动执行 SQL 即席查询（Ad-Hoc Query）。TDengine 支持如下查询功能： -- 查询单列、或多列查询 -- 支持值过滤条件：\>, \<, =, \<> 大于，小于，等于，不等于等等 -- 支持对标签的模糊匹配 -- 支持Group by, Order by, Limit, Offset -- 支持列之间的四则运算 -- 支持时间戳对齐的JOIN操作 -- 支持多种函数: count, max, min, avg, sum, twa, stddev, leastsquares, top, bottom, first, last, percentile, apercentile, last_row, spread, diff +- 单列、多列数据查询 +- 标签和数值的多种过滤条件：\>, \<, =, \<>, like 等 +- 聚合结果的分组（Group by）、排序（Order by）、约束输出（Limit/Offset） +- 数值列及聚合结果的四则运算 +- 时间戳对齐的连接查询（Join Query）操作 +- 多种聚合/计算函数: count, max, min, avg, sum, twa, stddev, leastsquares, top, bottom, first, last, percentile, apercentile, last_row, spread, diff等 -例如：在TAOS Shell中，从表d1001中查询出vlotage >215的记录，按时间降序排列，仅仅输出2条。 +例如：在TAOS Shell中，从表d1001中查询出vlotage > 215的记录，按时间降序排列，仅仅输出2条。 ```mysql taos> select * from d1001 where voltage > 215 order by ts desc limit 2; ts | current | voltage | phase | @@ -27,11 +26,11 @@ Query OK, 2 row(s) in set (0.001100s) ``` 为满足物联网场景的需求，TDengine支持几个特殊的函数，比如twa(时间加权平均)，spread (最大值与最小值的差)，last_row(最后一条记录)等，更多与物联网场景相关的函数将添加进来。TDengine还支持连续查询。 -具体的查询语法请看TAOS SQL。 +具体的查询语法请看TAOS SQL 。 ## 多表聚合查询 -TDengine对每个数据采集点单独建表，但应用经常需要对数据点之间进行聚合。为高效的进行聚合操作，TDengine引入超级表（STable）的概念。超级表用来代表一特定类型的数据采集点，它是表的集合，包含多张表。这集合里每张表的Schema是一样的，但每张表都带有自己的静态标签，标签可以多个，可以随时增加、删除和修改。 +TDengine对每个数据采集点单独建表，但在实际应用中经常需要对不同的采集点数据进行聚合。为高效的进行聚合操作，TDengine引入超级表（STable）的概念。超级表用来代表一特定类型的数据采集点，它是包含多张表的表集合，集合里每张表的模式（schema）完全一致，但每张表都带有自己的静态标签，标签可以多个，可以随时增加、删除和修改。 应用可通过指定标签的过滤条件，对一个STable下的全部或部分表进行聚合或统计操作，这样大大简化应用的开发。其具体流程如下图所示： @@ -39,16 +38,16 @@ TDengine对每个数据采集点单独建表，但应用经常需要对数据点
多表聚合查询原理图
-1：应用将一个查询条件发往系统；2: taosc将超级表的名字发往Meta Node（管理节点)；3：管理节点将超级表所拥有的vnode列表发回taosc；4：taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点；5：每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合，然后扫描存储的时序数据，完成相应的聚合计算，将结果返回给taosc；6：taosc将多个数据节点返回的结果做最后的聚合，将其返回给应用。 +1：应用将一个查询条件发往系统；2: taosc将超级表的名字发往 Meta Node（管理节点)；3：管理节点将超级表所拥有的 vnode 列表发回 taosc；4：taosc将计算的请求连同标签过滤条件发往这些vnode对应的多个数据节点；5：每个vnode先在内存里查找出自己节点里符合标签过滤条件的表的集合，然后扫描存储的时序数据，完成相应的聚合计算，将结果返回给taosc；6：taosc将多个数据节点返回的结果做最后的聚合，将其返回给应用。 -由于TDengine在vnode内将标签数据与时序数据分离存储，通过先在内存里过滤标签数据，将需要扫描的数据集大幅减少，大幅提升了聚合计算速度。同时，由于数据分布在多个vnode/dnode，聚合计算操作在多个vnode里并发进行，又进一步提升了聚合的速度。 +由于TDengine在vnode内将标签数据与时序数据分离存储，通过先在内存里过滤标签数据，将需要扫描的数据集大幅减少，大幅提升聚合计算速度。同时，由于数据分布在多个vnode/dnode，聚合计算操作在多个vnode里并发进行，又进一步提升了聚合的速度。 -对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样，细节请看TAOS SQL。 +对普通表的聚合函数以及绝大部分操作都适用于超级表，语法完全一样，细节请看 TAOS SQL。 比如：在TAOS Shell，查找所有智能电表采集的电压平均值，并按照location分组 ```mysql -taos> select avg(voltage) from meters group by location; +taos> SELECT AVG(voltage) FROM meters GROUP BY location; avg(voltage) | location | ============================================================= 222.000000000 | Beijing.Haidian | @@ -58,18 +57,18 @@ Query OK, 2 row(s) in set (0.002136s) ## 降采样查询、插值 -物联网场景里，经常需要做down sampling，需要将采集的数据按时间段进行聚合。TDengine提供了一个简便的关键词interval让操作变得极为简单。比如：将智能电表d1001采集的电流值每10秒钟求和 +物联网场景里，经常需要通过降采样（down sampling）将采集的数据按时间段进行聚合。TDengine 提供了一个简便的关键词 interval 让按照时间窗口的查询操作变得极为简单。比如，将智能电表 d1001 采集的电流值每10秒钟求和 ```mysql -taos> SELECT sum(current) FROM d1001 interval(10s) ; +taos> SELECT sum(current) FROM d1001 INTERVAL(10s) ; ts | sum(current) | ====================================================== 2018-10-03 14:38:00.000 | 10.300000191 | 2018-10-03 14:38:10.000 | 24.900000572 | Query OK, 2 row(s) in set (0.000883s) ``` -降采样操作还适用于超级表，比如：将所有智能电表采集的电流值每秒钟求和 +降采样操作也适用于超级表，比如：将所有智能电表采集的电流值每秒钟求和 ```mysql -taos> SELECT sum(current) FROM meters interval(1s) ; +taos> SELECT SUM(current) FROM meters INTERVAL(1s) ; ts | sum(current) | ====================================================== 2018-10-03 14:38:04.000 | 10.199999809 | @@ -82,5 +81,5 @@ Query OK, 5 row(s) in set (0.001538s) 物联网场景里，每个数据采集点采集数据的时间是难同步的，但很多分析算法(比如FFT)需要把采集的数据严格按照时间等间隔的对齐，在很多系统里，需要应用自己写程序来处理，但使用TDengine的降采样操作就轻松解决。如果一个时间间隔里，没有采集的数据，TDengine还提供插值计算的功能。 -语法规则细节请见TAOS SQL。 +语法规则细节请见TAOS SQL 。 diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 06a18de144..6535647981 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -1,33 +1,35 @@ -#系统管理 +# TDengine的运营与维护 ## 容量规划 -一个系统的处理能力是有限的，但通过对TDengine配置参数的调整，可以做到资源的最佳配置。 +使用TDengine来搭建一个物联网大数据平台，计算资源、存储资源需要根据业务场景进行规划。下面分别讨论系统运行所需要的内存、CPU以及硬盘空间。 -###内存需求 +### 内存需求 -每个Database可以创建固定数目的Vnode，默认与CPU核数相同，可通过maxVgroupsPerDb配置；每个vnode会占用固定大小的内存（大小与数据库的配置参数blocks和cache有关)；每个Table会占用与Tag总大小有关的内存；此外，系统会有一些固定的内存开销。因此，每个Database需要的系统内存可通过如下公式计算： +每个DB可以创建固定数目的vnode，默认与CPU核数相同，可通过maxVgroupsPerDb配置；每个vnode会占用固定大小的内存（大小与数据库的配置参数blocks和cache有关)；每个Table会占用与标签总长度有关的内存；此外，系统会有一些固定的内存开销。因此，每个DB需要的系统内存可通过如下公式计算： ``` Memory Size = maxVgroupsPerDb * (blocks * cache + 10Mb) + numOfTables * (tagSizePerTable + 0.5Kb) ``` -示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则从的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M +示例：假设是4核机器，cache是缺省大小16M, blocks是缺省值6，假设有10万张表，标签总长度是256字节，则总的内存需求为：4\*(16\*6+10) + 100000*(0.25+0.5)/1000 = 499M。 + +实际运行的系统往往会根据数据特点的不同，将数据存放在不同的DB里。因此做规划时，也需要考虑。 如果内存充裕，可以加大Blocks的配置，这样更多数据将保存在内存里，提高查询速度。 -###CPU需求 +### CPU需求 CPU的需求取决于如下两方面： -- 数据插入：TDengine单核每秒能至少处理一万个插入请求。每个插入请求可以带多条记录。条数越大，插入效率越高。但对前端数据采集的要求越高，因为需要缓存记录，然后一批插入。 -- 查询需求：TDengine提供高效的查询，但是每个场景的查询差异很大，查询频次变化也很大，难以给出客观数字。需要用户针对自己的场景，写一些查询语句，才能确定。 +* __数据插入__ TDengine单核每秒能至少处理一万个插入请求。每个插入请求可以带多条记录，一次插入一条记录与插入10条记录，消耗的计算资源差别很小。因此每次插入，条数越大，插入效率越高。如果一个插入请求带200条以上记录，单核就能达到每秒插入100万条记录的速度。但对前端数据采集的要求越高，因为需要缓存记录，然后一批插入。 +* __查询需求__ TDengine提供高效的查询，但是每个场景的查询差异很大，查询频次变化也很大，难以给出客观数字。需要用户针对自己的场景，写一些查询语句，才能确定。 因此仅对数据插入而言，CPU是可以估算出来的，但查询所耗的计算资源无法估算。在实际运营过程中，不建议CPU使用率超过50%，超过后，需要增加新的节点，以获得更多计算资源。 -###存储需求 +### 存储需求 -TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景下，TDengine的压缩比不会低于5倍，有的场合，压缩比可达到10倍以上，取决于数据特征。压缩前的原始数据大小可通过如下方式计算： +TDengine相对于通用数据库，有超高的压缩比，在绝大多数场景下，TDengine的压缩比不会低于5倍，有的场合，压缩比可达到10倍以上，取决于实际场景的数据特征。压缩前的原始数据大小可通过如下方式计算： ``` Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable @@ -39,6 +41,12 @@ Raw DataSize = numOfTables * rowSizePerTable * rowsPerTable 为提高速度，可以配置多快硬盘，这样可以并发写入或读取数据。 +### 物理机或虚拟机台数 + +根据上面的内存、CPU、存储的预估，就可以知道整个系统需要多少核、多少内存、多少存储空间。如果数据副本数不为1，总需求量需要再乘以副本数。 + +因为TDengine具有很好的水平扩展能力，根据总量，再根据单个物理机或虚拟机的资源，就可以轻松决定需要购置多少台物理机或虚拟机了。 + ## 容错和灾备 ### 容错 @@ -47,15 +55,16 @@ TDengine支持**WAL**（Write Ahead Log）机制，实现数据的容错能力 TDengine接收到应用的请求数据包时，先将请求的原始数据包写入数据库日志文件，等数据成功写入数据库数据文件后，再删除相应的WAL。这样保证了TDengine能够在断电等因素导致的服务重启时从数据库日志文件中恢复数据，避免数据的丢失。 -涉及的系统配置参数有两个. +涉及的系统配置参数有两个： -walLevel：WAL级别，0：不写wal; 1：写wal, 但不执行fsync; 2：写wal, 而且执行fsync。 +- walLevel：WAL级别，0：不写wal; 1：写wal, 但不执行fsync; 2：写wal, 而且执行fsync。 +- fsync：当walLevel设置为2时，执行fsync的周期。设置为0，表示每次写入，立即执行fsync。 -fsync：当walLevel设置为2时，执行fsync的周期。设置为0，表示每次写入，立即执行fsync。 +如果要100%的保证数据不丢失，需要将walLevel设置为2，fsync设置为0。这时写入速度将会下降。但如果应用侧启动的写数据的线程数达到一定的数量(超过50)，那么写入数据的性能也会很不错，只会比fsync设置为3000毫秒下降30%左右。 -**灾备** +### 灾备 -TDengine的集群通过多个副本的机制，来提供系统的高可靠性，实现灾备能力。 +TDengine的集群通过多个副本的机制，来提供系统的高可用性，实现灾备能力。 TDengine集群是由mnode负责管理的，为保证mnode的高可靠，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，为了支持高可靠，需要设置大于1。为保证元数据的强一致性，mnode副本之间通过同步方式进行数据复制，保证了元数据的强一致性。 @@ -63,33 +72,7 @@ TDengine集群中的时序数据的副本数是与数据库关联的，一个集 TDengine集群的节点数必须大于等于副本数，否则创建表时将报错。 -当TDengine集群中的节点部署在不同的物理机上（比如不同的机架、或不同的IDC），并设置多个副本数时，就实现了异地容灾，从而提供系统的高可靠性，无需再使用其他软件或工具。 - -## 文件目录结构 - -安装TDengine后，默认会在操作系统中生成下列目录或文件： - -| 目录/文件 | 说明 | -| ---------------------- | :------------------------------------------------| -| /usr/local/taos/bin | TDengine可执行文件目录。其中的执行文件都会软链接到/usr/bin目录下。 | -| /usr/local/taos/connector | TDengine各种连接器目录。 | -| /usr/local/taos/driver | TDengine动态链接库目录。会软链接到/usr/lib目录下。 | -| /usr/local/taos/examples | TDengine各种语言应用示例目录。 | -| /usr/local/taos/include | TDengine对外提供的C语言接口的头文件。 | -| /etc/taos/taos.cfg | TDengine默认[配置文件] | -| /var/lib/taos | TDengine默认数据文件目录,可通过[配置文件]修改位置. | -| /var/log/taos | TDengine默认日志文件目录,可通过[配置文件]修改位置 | - -**可执行文件** - -TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下。其中包括： - -- _taosd_：TDengine服务端可执行文件 -- _taos_： TDengine Shell可执行文件 -- _taosdump_：数据导入导出工具 -- remove.sh：卸载TDengine的脚本, 请谨慎执行，链接到/usr/bin目录下的rmtaos命令。会删除TDengine的安装目录/usr/local/taos，但会保留/etc/taos、/var/lib/taos、/var/log/taos。 - -您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 +当TDengine集群中的节点部署在不同的物理机上，并设置多个副本数时，就实现了系统的高可靠性，无需再使用其他软件或工具。TDengine企业版还可以将副本部署在不同机房，从而实现异地容灾。 ## 服务端配置 @@ -97,8 +80,8 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 下面仅仅列出一些重要的配置参数，更多的参数请看配置文件里的说明。各个参数的详细介绍及作用请看前述章节。**注意：配置修改后，需要重启*taosd*服务才能生效。** -- first: taosd启动时，主动连接的集群中第一个dnode的end point, 缺省值为 localhost:6030。 -- second: taosd启动时，如果first连接不上，尝试连接集群中第二个dnode的end point, 缺省值为空。 +- firstEp: taosd启动时，主动连接的集群中第一个dnode的end point, 缺省值为 localhost:6030。 +- secondEp: taosd启动时，如果first连接不上，尝试连接集群中第二个dnode的end point, 缺省值为空。 - fqdn：数据节点的FQDN。如果为空，将自动获取操作系统配置的第一个, 缺省值为空。 - serverPort：taosd启动后，对外服务的端口号，默认值为6030。 - httpPort: RESTful服务使用的端口号，所有的HTTP请求（TCP）都需要向该接口发起查询/写入请求。 @@ -142,7 +125,7 @@ TDengine集群中加入一个新的dnode时，涉及集群相关的一些参数 - statusInterval: dnode向mnode报告状态时长。单位为秒，默认值：1。 - maxTablesPerVnode: 每个vnode中能够创建的最大表个数。默认值：1000000。 - maxVgroupsPerDb: 每个数据库中能够使用的最大vnode个数。 -- arbitrator: 系统中裁决器的end point。 +- arbitrator: 系统中裁决器的end point，缺省为空 - timezone：时区。从系统中动态获取当前的时区设置。 - locale：系统区位信息及编码格式。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。 - charset：字符集编码。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。 @@ -153,8 +136,8 @@ TDengine系统的前台交互客户端应用程序为taos，它与taosd共享同 客户端配置参数列表及解释 -- first: taos启动时，主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。 -- second: taos启动时，如果first连接不上，尝试连接集群中第二个taosd实例的end point, 缺省值为空。 +- firstEp: taos启动时，主动连接的集群中第一个taosd实例的end point, 缺省值为 localhost:6030。 +- secondEp: taos启动时，如果first连接不上，尝试连接集群中第二个taosd实例的end point, 缺省值为空。 - charset：字符集编码。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。 - locale：系统区位信息及编码格式。系统中动态获取，如果自动获取失败，需要用户在配置文件设置或通过API设置。 @@ -243,9 +226,7 @@ Query OK, 9 row(s) affected (0.004763s) **taosdump工具导入** -TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见博客： - -[TDengine DUMP工具使用指南]: https://www.taosdata.com/blog/2020/03/09/1334.html +TDengine提供了方便的数据库导入导出工具taosdump。用户可以将taosdump从一个系统导出的数据，导入到其他系统中。具体使用方法，请参见博客：TDengine DUMP工具使用指南 ## 数据导出 @@ -263,9 +244,7 @@ select * from >> data.csv **用taosdump导出数据** -TDengine提供了方便的数据库导出工具taosdump。用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表,所有数据或一时间段的数据，甚至仅仅表的定义。具体使用方法，请参见博客： - -[TDengine DUMP工具使用指南]: https://www.taosdata.com/blog/2020/03/09/1334.html +TDengine提供了方便的数据库导出工具taosdump。用户可以根据需要选择导出所有数据库、一个数据库或者数据库中的一张表,所有数据或一时间段的数据，甚至仅仅表的定义。具体使用方法，请参见博客：TDengine DUMP工具使用指南 ## 系统连接、任务查询管理 @@ -313,3 +292,31 @@ TDengine启动后，会自动创建一个监测数据库SYS，并自动将服务 这些监测信息的采集缺省是打开的，但可以修改配置文件里的选项enableMonitor将其关闭或打开。 +## 文件目录结构 + +安装TDengine后，默认会在操作系统中生成下列目录或文件： + +| 目录/文件 | 说明 | +| ------------------------- | :----------------------------------------------------------- | +| /usr/local/taos/bin | TDengine可执行文件目录。其中的执行文件都会软链接到/usr/bin目录下。 | +| /usr/local/taos/connector | TDengine各种连接器目录。 | +| /usr/local/taos/driver | TDengine动态链接库目录。会软链接到/usr/lib目录下。 | +| /usr/local/taos/examples | TDengine各种语言应用示例目录。 | +| /usr/local/taos/include | TDengine对外提供的C语言接口的头文件。 | +| /etc/taos/taos.cfg | TDengine默认[配置文件] | +| /var/lib/taos | TDengine默认数据文件目录,可通过[配置文件]修改位置. | +| /var/log/taos | TDengine默认日志文件目录,可通过[配置文件]修改位置 | + +**可执行文件** + +TDengine的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下。其中包括： + +- _taosd_：TDengine服务端可执行文件 +- _taos_： TDengine Shell可执行文件 +- _taosdump_：数据导入导出工具 +- remove.sh：卸载TDengine的脚本, 请谨慎执行，链接到/usr/bin目录下的rmtaos命令。会删除TDengine的安装目录/usr/local/taos，但会保留/etc/taos、/var/lib/taos、/var/log/taos。 + +您可以通过修改系统配置文件taos.cfg来配置不同的数据目录和日志目录。 + + + diff --git a/documentation20/webdocs/markdowndocs/advanced features-ch.md b/documentation20/webdocs/markdowndocs/advanced features-ch.md index c767daddce..60bd6ddfdb 100644 --- a/documentation20/webdocs/markdowndocs/advanced features-ch.md +++ b/documentation20/webdocs/markdowndocs/advanced features-ch.md @@ -119,9 +119,9 @@ taos_consume taos_unsubscribe ``` -这些API的文档请见 [C/C++ 数据订阅接口](TODO: update link)， +这些API的文档请见 [C/C++ 数据订阅接口](connector/#C/C++-Connector)， 下面仍以智能电表场景为例介绍一下它们的具体用法（超级表和子表结构请参考上一节“连续查询”）， -完整的示例代码可以在[这里](TODO: update link)找到。 +完整的示例代码可以在 [这里](https://github.com/taosdata/TDengine/blob/master/tests/examples/c/subscribe.c) 找到。 如果我们希望当某个电表的电流超过一定限制（比如10A）后能得到通知并进行一些处理， 有两种方法： 一是分别对每张子表进行查询，每次查询后记录最后一条数据的时间戳，后续只查询这个时间戳之后的数据： diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index c4e6070761..5401c64685 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -4,46 +4,48 @@ ### 物联网典型场景 在典型的物联网、车联网、运维监测场景中，往往有多种不同类型的数据采集设备，采集一个到多个不同的物理量。而同一种采集设备类型，往往又有多个具体的采集设备分布在不同的地点。大数据处理系统就是要将各种采集的数据汇总，然后进行计算和分析。对于同一类设备，其采集的数据都是很规则的。以智能电表为例，假设每个智能电表采集电流、电压、相位三个量，其采集的数据类似如下的表格： -| Device ID | Time Stamp | current | voltage | phase | | location | groupId | -| :-------: | :-----------: | :-----: | :-----: | :---: | :--- | :--------------: | :-----: | -| D1001 | 1538548685000 | 10.3 | 219 | 0.31 | | Beijing.Chaoyang | 2 | -| D1002 | 1538548684000 | 10.2 | 220 | 0.23 | | Beijing.Chaoyang | 3 | -| D1003 | 1538548686500 | 11.5 | 221 | 0.35 | | Beijing.Haidian | 3 | -| D1004 | 1538548685500 | 13.4 | 223 | 0.29 | | Beijing.Haidian | 2 | -| D1001 | 1538548695000 | 12.6 | 218 | 0.33 | | Beijing.Chaoyang | 2 | -| D1004 | 1538548696600 | 11.8 | 221 | 0.28 | | Beijing.Haidian | 2 | -| D1002 | 1538548696650 | 10.3 | 218 | 0.25 | | Beijing.Chaoyang | 3 | -| D1001 | 1538548696800 | 12.3 | 221 | 0.31 | | Beijing.Chaoyang | 2 | +| Device ID | Time Stamp | current | voltage | phase | location | groupId | +| :-------: | :-----------: | :-----: | :-----: | :---: | :--------------: | :-----: | +| d1001 | 1538548685000 | 10.3 | 219 | 0.31 | Beijing.Chaoyang | 2 | +| d1002 | 1538548684000 | 10.2 | 220 | 0.23 | Beijing.Chaoyang | 3 | +| d1003 | 1538548686500 | 11.5 | 221 | 0.35 | Beijing.Haidian | 3 | +| d1004 | 1538548685500 | 13.4 | 223 | 0.29 | Beijing.Haidian | 2 | +| d1001 | 1538548695000 | 12.6 | 218 | 0.33 | Beijing.Chaoyang | 2 | +| d1004 | 1538548696600 | 11.8 | 221 | 0.28 | Beijing.Haidian | 2 | +| d1002 | 1538548696650 | 10.3 | 218 | 0.25 | Beijing.Chaoyang | 3 | +| d1001 | 1538548696800 | 12.3 | 221 | 0.31 | Beijing.Chaoyang | 2 | +
表1：智能电表数据示例
+ 每一条记录都有设备ID，时间戳，采集的物理量(如上图中的电流、电压、相位），还有与每个设备相关的静态标签（如上述表一中的位置Location和分组groupId）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。 ### 数据特征 除时序特征外，仔细研究发现，物联网、车联网、运维监测类数据还具有很多其他明显的特征。 -1. 数据是结构化的； +1. 数据高度结构化； 2. 数据极少有更新或删除操作； 3. 无需传统数据库的事务处理； 4. 相对互联网应用，写多读少； 5. 流量平稳，根据设备数量和采集频次，可以预测出来； 6. 用户关注的是一段时间的趋势，而不是某一特点时间点的值； -7. 数据是有保留期限的； -8. 数据的查询分析一定是基于时间段和地理区域的； -9. 除存储查询外，还往往需要各种统计和实时计算操作； +7. 数据有保留期限； +8. 数据的查询分析一定是基于时间段和地理区域； +9. 除存储查询外，还需要各种统计和实时计算操作； 10. 数据量巨大，一天采集的数据就可以超过100亿条。 -充分利用上述特征，TDengine采取了一特殊的优化的存储和计算设计来处理时序数据，能将系统处理能力显著提高。 +充分利用上述特征，TDengine 采取了特殊的优化的存储和计算设计来处理时序数据，能将系统处理能力显著提高。 ### 关系型数据库模型 因为采集的数据一般是结构化数据，而且为降低学习门槛，TDengine采用传统的关系型数据库模型管理数据。因此用户需要先创建库，然后创建表，之后才能插入或查询数据。TDengine采用的是结构化存储，而不是NoSQL的key-value存储。 ### 一个数据采集点一张表 -为充分利用其数据的时序性和其他数据特点，TDengine要求**对每个数据采集点单独建表**（比如有一千万个智能电表，就需创建一千万张表，上述表格中的D1001, D1002, D1003, D1004都需单独建表），用来存储这个采集点所采集的时序数据。这种设计有几大优点： +为充分利用其数据的时序性和其他数据特点，TDengine要求**对每个数据采集点单独建表**（比如有一千万个智能电表，就需创建一千万张表，上述表格中的d1001, d1002, d1003, d1004都需单独建表），用来存储这个采集点所采集的时序数据。这种设计有几大优点： 1. 能保证一个采集点的数据在存储介质上是一块一块连续的。如果读取一个时间段的数据，它能大幅减少随机读取操作，成数量级的提升读取和查询速度。 2. 由于不同采集设备产生数据的过程完全独立，每个设备的数据源是唯一的，一张表也就只有一个写入者，这样就可采用无锁方式来写，写入速度就能大幅提升。 3. 对于一个数据采集点而言，其产生的数据是时序的，因此写的操作可用追加的方式实现，进一步大幅提高数据写入速度。 -如果采用传统的方式，将多个设备的数据写入一张表，由于网络延时不可控，不同设备的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个设备的数据是难以保证连续存储在一起的。**采用一个采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的。** +如果采用传统的方式，将多个设备的数据写入一张表，由于网络延时不可控，不同设备的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个设备的数据是难以保证连续存储在一起的。**采用一个数据采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的。** TDengine 建议用数据采集点的名字(如上表中的D1001)来做表名。每个数据采集点可能同时采集多个物理量(如上表中的curent, voltage, phase)，每个物理量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 timestamp。对采集的数据，TDengine将自动按照时间戳建立索引，但对采集的物理量不建任何索引。数据是用列式存储方式保存。 @@ -56,13 +58,12 @@ TDengine 建议用数据采集点的名字(如上表中的D1001)来做表名。 当对多个具有相同数据类型的数据采集点进行聚合操作时，TDengine将先把满足标签过滤条件的表从超级表的中查找出来，然后再扫描这些表的时序数据，进行聚合操作，这样能将需要扫描的数据集大幅减少，从而大幅提高聚合计算的性能。 -##集群与基本逻辑单元 +## 集群与基本逻辑单元 TDengine 的设计是基于单个硬件、软件系统不可靠，基于任何单台计算机都无法提供足够计算能力和存储能力处理海量数据的假设进行设计的。因此 TDengine 从研发的第一天起，就按照分布式高可靠架构进行设计，是支持水平扩展的，这样任何单台或多台服务器发生硬件故障或软件错误都不影响系统的可用性和可靠性。同时，通过节点虚拟化并辅以自动化负载均衡技术，TDengine 能最高效率地利用异构集群中的计算和存储资源降低硬件投资。 -###主要逻辑单元 +### 主要逻辑单元 TDengine 分布式架构的逻辑结构图如下：
-
图 1 TDengine架构示意图
一个完整的 TDengine 系统是运行在一到多个物理节点上的，逻辑上，它包含数据节点(dnode)、TDengine客户端(taosc)以及应用(app)。系统中存在一到多个数据节点，这些数据节点组成一个集群(cluster)。应用通过taosc的API与TDengine集群进行互动。下面对每个逻辑单元进行简要介绍。 @@ -70,11 +71,11 @@ TDengine 分布式架构的逻辑结构图如下： **数据节点(dnode):** dnode 是 TDengine 服务器侧执行代码 taosd 在物理节点上的一个运行实例，一个工作的系统必须有至少一个数据节点。dnode包含零到多个逻辑的虚拟节点(VNODE)，零或者至多一个逻辑的管理节点(mnode). dnode在系统中的唯一标识由实例的End Point(EP)决定。EP是dnode所在物理节点的FQDN(Fully Qualified Domain Name)和系统所配置的网络端口号(Port)的组合。通过配置不同的端口，一个物理节点(一台物理机、虚拟机或容器）可以运行多个实例，或有多个数据节点。 -**虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点(vnode，图中V)。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB，但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的SCHEMA、标签值等。一个虚拟节点由所属的数据节点的EP，以及所属的Vgroup ID在系统内唯一标识，是由管理节点创建并管理的。 +**虚拟节点(vnode)**: 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点(vnode，图中V2, V3, V4等)。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个DB，但一个DB可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的SCHEMA、标签值等。一个虚拟节点由所属的数据节点的EP，以及所属的Vgroup ID在系统内唯一标识，是由管理节点创建并管理的。 -**管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode，它们自动构建成为一个管理节点集群(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。 +**管理节点(mnode):** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡(图中M)。同时，管理节点也负责元数据(包括用户、数据库、表、静态标签等)的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个(最多不超过5个) mnode，它们自动构建成为一个虚拟管理节点组(图中M0, M1, M2)。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步, 任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个dnode上至多有一个mnode，由所属的数据节点的EP来唯一标识。每个dnode通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的EP。 -**虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N，系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定，缺省为1。使用 TDengine 的多副本特性,可以不再需要昂贵的磁盘阵列等存储设备，获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一系统唯一的ID，vnode group ID。如果两个虚拟节点的vnode group ID相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。Vnode group ID是永远不变的，即使一个虚拟节点组被删除，它的ID也不会被收回重复利用。 +**虚拟节点组(VGroup):** 不同数据节点上的 vnode 可以组成一个虚拟节点组(vnode group)来保证系统的高可靠。虚拟节点组内采取master/slave的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个DB的副本数为N，系统必须有至少N个数据节点。副本数在创建DB时通过参数 replica 可以指定，缺省为1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一系统唯一的ID，vnode group ID。如果两个虚拟节点的vnode group ID相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。Vnode group ID是永远不变的，即使一个虚拟节点组被删除，它的ID也不会被收回重复利用。 **TAOSC:** taosc是TDengine给应用提供的驱动程序(driver)，负责处理应用与集群的接口交互，内嵌于JDBC、ODBC driver中，或者C、Python、Go语言连接库里。应用都是通过taosc，而不是直接连接集群中的数据节点与整个集群进行交互的。这个模块负责获取并缓存元数据；将插入、查询等请求转发到正确的数据节点；在把结果返回给应用时，还需要负责最后一级的聚合、排序、过滤等操作。对于JDBC, ODBC, C/C++接口而言，这个模块是在应用所处的物理节点上运行，但消耗的资源很小。同时，为支持全分布式的RESTful接口，taosc在TDengine集群的每个dnode上都有一运行实例。 @@ -95,12 +96,10 @@ TDengine 分布式架构的逻辑结构图如下： **重定向**：无论是dnode还是taosc，最先都是要发起与mnode的链接，但mnode是系统自动创建并维护的，因此对于用户来说，并不知道哪个dnode在运行mnode。TDengine只要求向系统中任何一个工作的dnode发起链接即可。因为任何一个正在运行的dnode，都维护有目前运行的mnode EP List。当收到一个来自新启动的dnode或taosc的链接请求，如果自己不是mnode，则将mnode EP List回复给对方，taosc或新启动的dnode收到这个list, 就重新尝试建立链接。当mnode EP List发生改变，通过节点之间的消息交互，各个数据节点就很快获取最新列表，并通知taosc。 -###一典型的操作流程 +### 一典型的操作流程 为解释vnode, mnode, taosc和应用之间的关系以及各自扮演的角色，下面对写入数据这个典型操作的流程进行剖析。
-
图 2 TDengine典型的操作流程
- 1. 应用通过JDBC、ODBC或其他API接口发起插入数据的请求。 2. taosc会检查缓存，看是有保存有该表的meta data。如果有，直接到第4步。如果没有，taosc将向mnode发出get meta-data请求。 3. mnode将该表的meta-data返回给taosc。Meta-data包含有该表的schema, 而且还有该表所属的vgroup信息（vnode ID以及所在的dnode的End Point，如果副本数为N，就有N组End Point)。如果taosc迟迟得不到mnode回应，而且存在多个mnode, taosc将向下一个mnode发出请求。 @@ -118,7 +117,7 @@ TDengine 分布式架构的逻辑结构图如下： ## 存储模型与数据分区、分片 -###存储模型 +### 存储模型 TDengine存储的数据包括采集的时序数据以及库、表相关的元数据、标签数据等，这些数据具体分为三部分： - 时序数据：存放于vnode里，由data、head和last三个文件组成，数据量大，查询量取决于应用场景。容许乱序写入，但暂时不支持删除和更新操作。通过采用一个采集点一张表的模型，一个时间段的数据是连续存储，对单张表的写入是简单的追加操作，一次读，可以读到多条记录，这样保证对单个采集点的插入和查询操作，性能达到最优。 @@ -130,7 +129,7 @@ TDengine存储的数据包括采集的时序数据以及库、表相关的元数 - 能够极大地降低标签数据存储的冗余度：一般的NoSQL数据库或时序数据库，采用的K-V存储，其中的Key包含时间戳、设备ID、各种标签。每条记录都带有这些重复的内容，浪费存储空间。而且如果应用要在历史数据上增加、修改或删除标签，需要遍历数据，重写一遍，操作成本极其昂贵。 - 能够实现极为高效的多表之间的聚合查询：做多表之间聚合查询时，先把符合标签过滤条件的表查找出来，然后再查找这些表相应的数据块，这样大幅减少要扫描的数据集，从而大幅提高查询效率。而且标签数据采用全内存的结构进行管理和维护，千万级别规模的标签数据查询可以在毫秒级别返回。 -###数据分片 +### 数据分片 对于海量的数据管理，为实现水平扩展，一般都需要采取分片(Sharding)分区(Partitioning)策略。TDengine是通过vnode来实现数据分片的，通过一个时间段一个数据文件来实现时序数据分区的。 vnode(虚拟数据节点)负责为采集的时序数据提供写入、查询和计算功能。为便于负载均衡、数据恢复、支持异构环境，TDengine将一个数据节点根据其计算和存储资源切分为多个vnode。这些vnode的管理是TDengine自动完成的，对应用完全透明。 @@ -141,12 +140,12 @@ vnode(虚拟数据节点)负责为采集的时序数据提供写入、查询和 每张表的meda data（包含schema, 标签等）也存放于vnode里，而不是集中存放于mnode，实际上这是对Meta数据的分片，这样便于高效并行的进行标签过滤操作。 -###数据分区 -TDengine除vnode分片之外，还按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由DB的配置参数days决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数keep)，将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。 +### 数据分区 +TDengine除vnode分片之外，还对时序数据按照时间段进行分区。每个数据文件只包含一个时间段的时序数据，时间段的长度由DB的配置参数days决定。这种按时间段分区的方法还便于高效实现数据的保留策略，只要数据文件超过规定的天数（系统配置参数keep)，将被自动删除。而且不同的时间段可以存放于不同的路径和存储介质，以便于大数据的冷热管理，实现多级存储。 总的来说，**TDengine是通过vnode以及时间两个维度，对大数据进行切分**，便于并行高效的管理，实现水平扩展。 -###负载均衡 +### 负载均衡 每个dnode都定时向 mnode(虚拟管理节点)报告其状态（包括硬盘空间、内存大小、CPU、网络、虚拟节点个数等），因此mnode了解整个集群的状态。基于整体状态，当mnode发现某个dnode负载过重，它会将dnode上的一个或多个vnode挪到其他dnode。在挪动过程中，对外服务继续进行，数据插入、查询和计算操作都不受影响。 如果mnode一段时间没有收到dnode的状态报告，mnode会认为这个dnode已经离线。如果离线时间超过一定时长（时长由配置参数offlineThreshold决定），该dnode将被mnode强制剔除出集群。该dnode上的vnodes如果副本数大于一，系统将自动在其他dnode上创建新的副本，以保证数据的副本数。如果该dnode上还有mnode, 而且mnode的副本数大于一，系统也将自动在其他dnode上创建新的mnode, 以保证mnode的副本数。 @@ -155,14 +154,13 @@ TDengine除vnode分片之外，还按照时间段进行分区。每个数据文 负载均衡过程无需任何人工干预，应用也无需重启，将自动连接新的节点，完全透明。 -##数据写入与复制流程 +## 数据写入与复制流程 如果一个数据库有N个副本，那一个虚拟节点组就有N个虚拟节点，但是只有一个是Master，其他都是slave。当应用将新的记录写入系统时，只有Master vnode能接受写的请求。如果slave vnode收到写的请求，系统将通知taosc需要重新定向。 -###Master vnode写入流程 +### Master vnode写入流程 Master Vnode遵循下面的写入流程：
图 3 TDengine Master写入流程
- 1. Master vnode收到应用的数据插入请求，验证OK，进入下一步； 2. 如果系统配置参数walLevel打开（设置为2），vnode将把该请求的原始数据包写入数据库日志文件WAL，以保证TDengine能够在断电等因素导致的服务重启时从数据库日志文件中恢复数据，避免数据的丢失； 3. 如果有多个副本，vnode将把数据包转发给同一虚拟节点组内slave vnodes, 该转发包带有数据的版本号(version) @@ -175,7 +173,6 @@ Master Vnode遵循下面的写入流程：
图 4 TDengine Slave写入流程
- 1. Slave vnode收到Master vnode转发了的数据插入请求。 2. 如果系统配置参数walLevl设置为2，vnode将把该请求的原始数据包写入日志(WAL)； 3. 写入内存，更新内存中的skip list。 @@ -207,24 +204,24 @@ Vnode会保持一个数据版本号(Version)，对内存数据进行持久化存 3. 在线的虚拟节点数过半，而且有虚拟节点是slave的话，该虚拟节点自动成为master 4. 对于2和3，如果多个虚拟节点满足成为master的要求，那么虚拟节点组的节点列表里，最前面的选为master -更多的关于数据复制的流程，请见[TDengine 2.0 数据复制模块设计](https://jira.taosdata.com:18090/pages/viewpage.action?pageId=6266055)。 +更多的关于数据复制的流程，请见TDengine 2.0数据复制模块设计。 -###同步复制 +### 同步复制 对于数据一致性要求更高的场景，异步数据复制无法满足要求，因为有极小的概率丢失数据，因此TDengine提供同步复制的机制供用户选择。在创建数据库时，除指定副本数replica之外，用户还需要指定新的参数quorum。如果quorum大于一，它表示每次Master转发给副本时，需要等待quorum-1个回复确认，才能通知应用，数据在slave已经写入成功。如果在一定的时间内，得不到quorum-1个回复确认，master vnode将返回错误给应用。 -采用同步复制，系统的性能会有所下降，而且latency会增加。因为元数据要强一致，Mnode之间的数据同步就是采用的同步复制。 +采用同步复制，系统的性能会有所下降，而且latency会增加。因为元数据要强一致，mnode之间的数据同步缺省就是采用的同步复制。 注：vnode之间的同步复制仅仅企业版支持 -##缓存与持久化 -###缓存 +## 缓存与持久化 +### 缓存 TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO），又称为写驱动的缓存管理机制。这种策略有别于读驱动的数据缓存模式（Least-Recent-Used，LRU），直接将最近写入的数据保存在系统的缓存中。当缓存达到临界值的时候，将最早的数据批量写入磁盘。一般意义上来说，对于物联网数据的使用，用户最为关心的是刚产生的数据，即当前状态。TDengine充分利用这一特性，将最近到达的（当前状态）数据保存在缓存中。 TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，**可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署Redis或其他额外的缓存系统**，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。 每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当一半以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有一半内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。 -###持久化存储 +### 持久化存储 TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时，为了不阻塞后续数据的写入，TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件，在落盘成功后则会删除老的数据库日志文件，避免日志文件无限制的增长。 为充分利用时序数据特点，TDengine将一个vnode保存在持久化存储的数据切分成多个文件，每个文件只保存固定天数的数据，这个天数由系统配置参数days决定。切分成多个文件后，给定查询的起止日期，无需任何索引，就可以立即定位需要打开哪些数据文件，大大加快读取速度。 @@ -239,7 +236,7 @@ TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久 数据写入磁盘时，根据系统配置参数comp决定是否压缩数据。TDengine提供了三种压缩选项：无压缩、一阶段压缩和两阶段压缩，分别对应comp值为0、1和2的情况。一阶段压缩根据数据的类型进行了相应的压缩，压缩算法包括delta-delta编码、simple 8B方法、zig-zag编码、LZ4等算法。二阶段压缩在一阶段压缩的基础上又用通用压缩算法进行了压缩，压缩率更高。 -###多级存储 +### 多级存储 在默认配置下，TDengine会将所有数据保存在/var/lib/taos目录下，而且每个vnode的数据文件保存在该目录下的不同目录。为扩大存储空间，尽量减少文件读取的瓶颈，提高数据吞吐率 TDengine可通过配置系统参数dataDir让多个挂载的硬盘被系统同时使用。除此之外，TDengine也提供了数据分级存储的功能，即根据数据文件的新老程度存储在不同的存储介质上。比如最新的数据存储在SSD上，超过一周的数据存储在本地硬盘上，超过4周的数据存储在网络存储设备上，这样来降低存储成本，而又保证高效的访问数据。数据在不同存储介质上的移动是由系统自动完成的，对应用是完全透明的。数据的分级存储也是通过系统参数dataDir来配置。 dataDir的配置格式如下： @@ -263,13 +260,40 @@ dataDir /mnt/disk6/taos 2 注：多级存储功能仅企业版支持 -##数据查询 +## 数据查询 TDengine提供了多种多样针对表和超级表的查询处理功能，除了常规的聚合查询之外，还提供针对时序数据的窗口查询、统计聚合等功能。TDengine的查询处理需要客户端、vnode, mnode节点协同完成。 -5.1 单表查询 +### 单表查询 +SQL语句的解析和校验工作在客户端完成。解析SQL语句并生成抽象语法树(Abstract Syntax Tree, AST)，然后对其进行校验和检查。以及向管理节点(mnode)请求查询中指定表的元数据信息(table metadata)。 -5.4 时间轴聚合、插值 +根据元数据信息中的End Point信息，将查询请求序列化后发送到该表所在的数据节点（dnode）。dnode接收到查询请求后，识别出该查询请求指向的虚拟节点（vnode），将消息转发到vnode的查询执行队列。vnode的查询执行线程建立基础的查询执行环境，并立即返回该查询请求，同时开始执行该查询。 -5.2 多表聚合查询 +客户端在获取查询结果的时候，dnode的查询执行队列中的工作线程会等待vnode执行线程执行完成，才能将查询结果返回到请求的客户端。 -5.3 预计算 \ No newline at end of file +### 按时间轴聚合、降采样、插值 + +时序数据有别于普通数据的显著特征是每条记录均具有时间戳，因此针对具有时间戳数据在时间轴上进行聚合是不同于普通数据库的重要功能。从这点上来看，与流计算引擎的窗口查询有相似的地方。 + +在TDengine中引入关键词interval来进行时间轴上固定长度时间窗口的切分，并按照时间窗口对数据进行聚合，对窗口范围内的数据按需进行聚合。例如： +```mysql +select count(*) from d1001 interval(1h) +``` + +针对d1001设备采集的数据，按照1小时的时间窗口返回每小时存储的记录数量。 + +在需要连续获得查询结果的应用场景下，如果给定的时间区间存在数据缺失，会导致该区间数据结果也丢失。TDengine提供策略针对时间轴聚合计算的结果进行插值，通过使用关键词Fill就能够对时间轴聚合结果进行插值。例如： +```mysql +select count(*) from d1001 interval(1h) fill(prev) +``` + +针对d1001设备采集数据统计每小时记录数，如果某一个小时不存在数据，这返回之前一个小时的统计数据。TDengine提供前向插值(prev)、线性插值(linear)、NULL值填充(NULL)、特定值填充(value)。 + +### 多表聚合查询 +多表聚合查询与单表查询的整体流程相同，但是存在如下的差异： + +- 由于多表可能分布在不同的节点(dnode)，因此多表的聚合查询需要首先获得表所在的全部数据节点的信息，并且同时向相关的dnode发出查询请求。 +- 每个vnode的计算获得的中间结果(partial results)需要进行第二阶段的聚合才能形成最终结果，第二阶段的聚合过程在客户端完成。 +- 由于表标签信息存储在vnode中，因此针对标签信息的查询也需要vnode完成。客户端将标签的过滤表达式封装在查询请求结构体中发送给vnode，由vnode的查询执行线程从中抽取出标签查询条件，然后执行查询。标签查询与过滤是在针对表的查询之前完成。标签查询完成以后，将符合条件的表纳入到接下来的查询处理流程中。 + +### 预计算 +为有效提升查询处理的性能，针对物联网数据的不可更改的特点，在数据块头部记录该数据块中存储数据的统计信息：包括最大值、最小值、和。我们称之为预计算单元。如果查询处理涉及整个数据块的全部数据，直接使用预计算结果，完全不需要读取数据块的内容。由于预计算数据量远小于磁盘上存储的数据块数据的大小，对于磁盘IO为瓶颈的查询处理，使用预计算结果可以极大地减小读取IO压力，加速查询处理的流程。预计算机制与Postgre SQL的索引BRIN（block range index）有异曲同工之妙。 diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md new file mode 100644 index 0000000000..6102d5b020 --- /dev/null +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -0,0 +1,144 @@ +#TDengine 集群安装、管理 + +多个taosd的运行实例可以组成一个集群，以保证TDengine的高可靠运行，并提供水平扩展能力。要了解TDengine 2.0的集群管理，需要对集群的基本概念有所了解，请看TDengine 2.0整体架构一章。 + +集群的每个节点是由End Point来唯一标识的，End Point是由FQDN(Fully Qualified Domain Name)外加Port组成，比如 h1.taosdata.com:6030。一般FQDN就是服务器的hostname，可通过Linux命令“hostname"获取。端口是这个节点对外服务的端口号，缺省是6030，但可以通过taos.cfg里配置参数serverPort进行修改。 + +TDengine的集群管理极其简单，除添加和删除节点需要人工干预之外，其他全部是自动完成，最大程度的降低了运维的工作量。本章对集群管理的操作做详细的描述。 + +##安装、创建第一个节点 + +集群是由一个一个dnode组成的，是从一个dnode的创建开始的。创建第一个节点很简单，就按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装、启动即可。 + +启动后，请执行taos, 启动taos shell，从shell里执行命令"show dnodes;"，如下所示： + ``` +Welcome to the TDengine shell from Linux, Client Version:2.0.0.0 +Copyright (c) 2017 by TAOS Data, Inc. All rights reserved. + +taos> show dnodes; + id | end_point | vnodes | cores | status | role | create_time | +===================================================================================== + 1 | h1.taos.com:6030 | 0 | 2 | ready | any | 2020-07-31 03:49:29.202 | +Query OK, 1 row(s) in set (0.006385s) + +taos> + ``` +上述命令里，可以看到这个刚启动的这个节点的End Point是：h1.taos.com:6030 + +## 安装、创建后续节点 + +将新的节点添加到现有集群，具体有以下几步： + +1. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法进行安装，**但不要启动taosd** + +2. 如果是使用涛思数据的官方安装包进行安装，在安装结束时，会询问集群的End Port, 输入第一个节点的End Point即可。如果是源码安装，请编辑配置文件taos.cfg(缺省是在/etc/taos/目录)，增加一行： + + ``` + firstEp h1.taos.com:6030 + ``` + + 请注意将示例的“h1.taos.com:6030" 替换为你自己第一个节点的End Point + +3. 按照["立即开始“](https://www.taosdata.com/cn/getting-started/)一章的方法启动taosd + +4. 在Linux shell里执行命令"hostname"找出本机的FQDN, 假设为h2.taos.com。如果无法找到，可以查看taosd日志文件taosdlog.0里前面几行日志(一般在/var/log/taos目录)，fqdn以及port都会打印出来。 + +5. 在第一个节点，使用CLI程序taos, 登录进TDengine系统, 使用命令: + + ``` + CREATE DNODE "h2.taos.com:6030"； + ``` + + 将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。请注意将示例的“h2.taos.com:6030" 替换为你自己第一个节点的End Point + +6. 使用命令 + + ``` + SHOW DNODES； + ``` + + 查看新节点是否被成功加入。 + +按照上述步骤可以源源不断的将新的节点加入到集群。 + +**提示：** + +- firstEp, secondEp这两个参数仅仅在该节点第一次加入集群时有作用，加入集群后，该节点会保存最新的mnode的End Point列表，不再依赖这两个参数。 +- 两个没有配置firstEp, secondEp参数的dnode启动后，会独立运行起来。这个时候，无法将其中一个节点加入到另外一个节点，形成集群。**无法将两个独立的集群合并成为新的集群**。 + +##节点管理 + +###添加节点 +执行CLI程序taos, 使用root账号登录进系统, 执行: +``` +CREATE DNODE "fqdn:port"； +``` +将新节点的End Point添加进集群的EP列表。**"fqdn:port"需要用双引号引起来**，否则出错。一个节点对外服务的fqdn和port可以通过配置文件taos.cfg进行配置，缺省是自动获取。 + +###删除节点 +执行CLI程序taos, 使用root账号登录进TDengine系统，执行： +``` +DROP DNODE "fqdn:port"; +``` +其中fqdn是被删除的节点的FQDN，port是其对外服务器的端口号 + +###查看节点 +执行CLI程序taos,使用root账号登录进TDengine系统，执行： +``` +SHOW DNODES; +``` +它将列出集群中所有的dnode,每个dnode的fqdn:port, 状态(ready, offline等），vnode数目，还未使用的vnode数目等信息。在添加或删除一个节点后，可以使用该命令查看。 + +###查看虚拟节点组 + +为充分利用多核技术，并提供scalability，数据需要分片处理。因此TDengine会将一个DB的数据切分成多份，存放在多个vnode里。这些vnode可能分布在多个dnode里，这样就实现了水平扩展。一个vnode仅仅属于一个DB，但一个DB可以有多个vnode。vnode的是mnode根据当前系统资源的情况，自动进行分配的，无需任何人工干预。 + +执行CLI程序taos,使用root账号登录进TDengine系统，执行： +``` +SHOW VGROUPS; +``` +##高可用性 +TDengine通过多副本的机制来提供系统的高可用性。副本数是与DB关联的，一个集群里可以有多个DB，根据运营的需求，每个DB可以配置不同的副本数。创建数据库时，通过参数replica 指定副本数（缺省为1）。如果副本数为1，系统的可靠性无法保证，只要数据所在的节点宕机，就将无法提供服务。集群的节点数必须大于等于副本数，否则创建表时将返回错误“more dnodes are needed"。比如下面的命令将创建副本数为3的数据库demo： +``` +CREATE DATABASE demo replica 3; +``` +一个DB里的数据会被切片分到多个vnode group，vnode group里的vnode数目就是DB的副本数，同一个vnode group里各vnode的数据是完全一致的。为保证高可用性，vnode group里的vnode一定要分布在不同的dnode里（实际部署时，需要在不同的物理机上），只要一个vgroup里超过半数的vnode处于工作状态，这个vgroup就能正常的对外服务。 + +一个dnode里可能有多个DB的数据，因此一个dnode离线时，可能会影响到多个DB。如果一个vnode group里的一半或一半以上的vnode不工作，那么该vnode group就无法对外服务，无法插入或读取数据，这样会影响到它所属的DB的一部分表的d读写操作。 + +因为vnode的引入，无法简单的给出结论：“集群中过半dnode工作，集群就应该工作”。但是对于简单的情形，很好下结论。比如副本数为3，只有三个dnode，那如果仅有一个节点不工作，整个集群还是可以正常工作的，但如果有两个节点不工作，那整个集群就无法正常工作了。 + +##Mnode的高可用 +TDengine集群是由mnode (taosd的一个模块，逻辑节点) 负责管理的，为保证mnode的高可用，可以配置多个mnode副本，副本数由系统配置参数numOfMnodes决定，有效范围为1-3。为保证元数据的强一致性，mnode副本之间是通过同步的方式进行数据复制的。 + +一个集群有多个dnode, 但一个dnode至多运行一个mnode实例。多个dnode情况下，哪个dnode可以作为mnode呢？这是完全由系统根据整个系统资源情况，自动指定的。用户可通过CLI程序taos，在TDengine的console里，执行如下命令： +``` +SHOW MNODES; +``` +来查看mnode列表，该列表将列出mnode所处的dnode的End Point和角色(master, slave, unsynced 或offline)。 +当集群中第一个节点启动时，该节点一定会运行一个mnode实例，否则该dnode无法正常工作，因为一个系统是必须有至少一个mnode的。如果numOfMnodes配置为2，启动第二个dnode时，该dnode也将运行一个mnode实例。 + +为保证mnode服务的高可用性，numOfMnodes必须设置为2或更大。因为mnode保存的元数据必须是强一致的，如果numOfMnodes大于2，复制参数quorum自动设为2，也就是说，至少要保证有两个副本写入数据成功，才通知客户端应用写入成功。 + +##负载均衡 + +有三种情况，将触发负载均衡，而且都无需人工干预。 + +- 当一个新节点添加进集群时，系统将自动触发负载均衡，一些节点上的数据将被自动转移到新节点上，无需任何人工干预。 +- 当一个节点从集群中移除时，系统将自动把该节点上的数据转移到其他节点，无需任何人工干预。 +- 如果一个节点过热（数据量过大），系统将自动进行负载均衡，将该节点的一些vnode自动挪到其他节点。 + +当上述三种情况发生时，系统将启动一各个节点的负载计算，从而决定如何挪动。 + +##节点离线处理 +如果一个节点离线，TDengine集群将自动检测到。有如下两种情况： +- 改节点离线超过一定时间（taos.cfg里配置参数offlineThreshold控制时长)，系统将自动把该节点删除，产生系统报警信息，触发负载均衡流程。如果该被删除的节点重现上线时，它将无法加入集群，需要系统管理员重新将其添加进集群才会开始工作。 +- 离线后，在offlineThreshold的时长内重新上线，系统将自动启动数据恢复流程，等数据完全恢复后，该节点将开始正常工作。 + +**注意：**如果一个虚拟节点组（包括mnode组）里每个节点都处于离线或unsynced状态，必须等该虚拟节点组里的所有节点都上线、都能交换状态信息后，才能选出Master，该虚拟节点组才能对外提供服务。比如整个集群有3个节点，副本数为3，如果3个节点都宕机，然后2个节点重启，是无法工作的，只有等3个节点都重启成功，才能对外服务。 + +##Arbitrator的使用 + +如果副本数为偶数，当一个vnode group里一半或超过一半的vnode不工作时，是无法从中选出master的。同理，一半或超过一半的mnode不工作时，是无法选出mnode的master的，因为存在“split brain”问题。为解决这个问题，TDengine引入了arbitrator的概念。Arbitrator模拟一个vnode或mnode在工作，但只简单的负责网络连接，不处理任何数据插入或访问。只要包含arbitrator在内，超过半数的vnode或mnode工作，那么该vnode group或mnode组就可以正常的提供数据插入或查询服务。比如对于副本数为2的情形，如果一个节点A离线，但另外一个节点B正常，而且能连接到arbitrator, 那么节点B就能正常工作。 + +TDengine安装包里带有一个执行程序tarbitrator, 找任何一台Linux服务器运行它即可。该程序对系统资源几乎没有要求，只需要保证有网络连接即可。该应用的命令行参数`-p`可以指定其对外服务的端口号，缺省是6030。配置每个taosd实例时，可以在配置文件taos.cfg里将参数arbitrator设置为arbitrator的End Point。如果该参数配置了，当副本数为偶数数，系统将自动连接配置的arbitrator。 diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md index 71f1dd31d3..04f7c2bc5c 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-ch.md @@ -45,11 +45,11 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine 创建数据库连接，初始化连接上下文。其中需要用户提供的参数包含： - * ip：TDengine管理主节点的IP地址 - * user：用户名 - * pass：密码 - * db：数据库名字，如果用户没有提供，也可以正常连接，用户可以通过该连接创建新的数据库，如果用户提供了数据库名字，则说明该数据库用户已经创建好，缺省使用该数据库 - * port：端口号 + - ip：TDengine管理主节点的IP地址 + - user：用户名 + - pass：密码 + - db：数据库名字，如果用户没有提供，也可以正常连接，用户可以通过该连接创建新的数据库，如果用户提供了数据库名字，则说明该数据库用户已经创建好，缺省使用该数据库 + - port：端口号 返回值为空表示失败。应用程序需要保存返回的参数，以便后续API调用。 diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 574e69ccd9..bca9cff8e6 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -5,19 +5,19 @@ 2.0版本在之前版本的基础上，进行了完全的重构，配置文件和数据文件是不兼容的。在升级之前务必进行如下操作： 1. 删除配置文件，执行 sudo rm -rf /etc/taos/taos.cfg - 2. 删除日志文件，执行 sudo rm -rf /var/log/taos/ - 3. 确保数据已经不再需要的前提下，删除数据文件，执行 sudo rm -rf /var/lib/taos/ - 4. 安装最新稳定版本的TDengine -5. 如果数据需要迁移数据或者数据文件损坏，请联系涛思数据官方技术支持团队，进行协助解决 +5. 如果数据需要迁移数据或者数据文件损坏，请联系涛思数据官方技术支持团队，进行协助解决 -#### 2. Windows平台下JDBCDriver找不到动态链接库 +#### 2. Windows平台下JDBCDriver找不到动态链接库，怎么办？ +请看为此问题撰写的技术博客 -#### 3. 创建数据表时提示more dnodes are needed +#### 3. 创建数据表时提示more dnodes are needed +请看为此问题撰写的技术博客 -#### 4. TDengine crash时生成core文件的方法 +#### 4. 如何让TDengine crash时生成core文件？ +请看为此问题撰写的技术博客 #### 5. 遇到错误"failed to connect to server", 我怎么办？ @@ -68,7 +68,8 @@ windows下插入nchar类的数据中如果有中文，请先确认系统的地 ​ Connection = DriverManager.getConnection(url, properties); -#### 12. TDengine GO windows驱动的编译 +#### 12.TDengine GO windows驱动的如何编译？ +请看为此问题撰写的技术博客 diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md index 2fd869cfe6..b74e7a16ca 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/webdocs/markdowndocs/insert-ch.md @@ -18,9 +18,12 @@ TDengine也支持一次向多个表写入数据，比如下面这条命令就向 INSERT INTO d1001 VALUES (1538548685000, 10.3, 219, 0.31) (1538548695000, 12.6, 218, 0.33) d1002 VALUES (1538548696800, 12.3, 221, 0.31); ``` -**Tips:** 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置）。 +详细的SQL INSERT语法规则请见TAOS SQL -详细的SQL INSERT语法规则请见TAOS SQL +**Tips:** + +- 要提高写入效率，需要批量写入。一批写入的记录条数越多，插入效率就越高。但一条记录不能超过16K，一条SQL语句总长度不能超过64K（可通过参数maxSQLLength配置）。 +- TDengine支持多线程同时写入，要进一步提高写入速度，一个客户端需要打开20个以上的线程同时写。但线程数达到一定数量后，无法再提高，甚至还会下降，因为线程切频繁切换，带来额外开销。 ## Prometheus直接写入 [Prometheus](https://www.prometheus.io/)作为Cloud Native Computing Fundation毕业的项目，在性能监控以及K8S性能监控领域有着非常广泛的应用。TDengine提供一个小工具[Bailongma](https://github.com/taosdata/Bailongma)，只需在Prometheus做简单配置，无需任何代码，就可将Prometheus采集的数据直接写入TDengine，并按规则在TDengine自动创建库和相关表项。博文[用Docker容器快速搭建一个Devops监控Demo](https://www.taosdata.com/blog/2020/02/03/1189.html)即是采用bailongma将Prometheus和Telegraf的数据写入TDengine中的示例，可以参考。 diff --git a/documentation20/webdocs/markdowndocs/replica-ch.md b/documentation20/webdocs/markdowndocs/replica-ch.md new file mode 100644 index 0000000000..c32928b42d --- /dev/null +++ b/documentation20/webdocs/markdowndocs/replica-ch.md @@ -0,0 +1,253 @@ +# TDengine 2.0 数据复制模块设计 + +## 数据复制概述 + +数据复制(Replication)是指同一份数据在多个物理地点保存。它的目的是防止数据丢失，提高系统的高可用性(High Availability)，而且通过应用访问多个副本，提升数据查询性能。 + +在高可靠的大数据系统里，数据复制是必不可少的一大功能。数据复制又分为实时复制与非实时复制。实时复制是指任何数据的更新(包括数据的增加、删除、修改）操作，会被实时的复制到所有副本，这样任何一台机器宕机或网络出现故障，整个系统还能提供最新的数据，保证系统的正常工作。而非实时复制，是指传统的数据备份操作，按照固定的时间周期，将一份数据全量或增量复制到其他地方。如果主节点宕机，副本是很大可能没有最新数据，因此在有些场景是无法满足要求的。 + +TDengine面向的是物联网场景，需要支持数据的实时复制，来最大程度保证系统的可靠性。实时复制有两种方式，一种是异步复制，一种是同步复制。异步复制(Asynchronous Replication)是指数据由Master转发给Slave后，Master并不需要等待Slave回复确认，这种方式效率高，但有极小的概率会丢失数据。同步复制是指Master将数据转发给Slave后，需要等待Slave的回复确认，才会通知应用写入成功，这种方式效率偏低，但能保证数据绝不丢失。 + +数据复制是与数据存储（写入、读取）密切相关的，但两者又是相对独立，可以完全脱耦的。在TDengine系统中，有两种不同类型的数据，一种是时序数据，由TSDB模块负责；一种是元数据(Meta Data), 由MNODE负责。这两种性质不同的数据都需要同步功能。数据复制模块通过不同的实例启动配置参数，为这两种类型数据都提供同步功能。 + +在阅读本文之前，请先阅读《TDengine 2.0 整体架构》，了解TDengine的集群设计和基本概念 + +特别注明：本文中提到数据更新操作包括数据的增加、删除与修改。 + +## 基本概念和定义 + +TDengine里存在vnode, mnode, vnode用来存储时序数据，mnode用来存储元数据。但从同步数据复制的模块来看，两者没有本质的区别，因此本文里的虚拟节点不仅包括vnode, 也包括mnode, vgoup也指mnode group, 除非特别注明。 + +**版本(version)**： + +一个虚拟节点组里多个虚拟节点互为备份，来保证数据的有效与可靠，是依靠虚拟节点组的数据版本号来维持的。TDengine2.0设计里，对于版本的定义如下：客户端发起增加、删除、修改的流程，无论是一条记录还是多条，只要是在一个请求里，这个数据更新请求被TDengine的一个虚拟节点收到后，经过合法性检查后，可以被写入系统时，就会被分配一个版本号。这个版本号在一个虚拟节点里从1开始，是单调连续递增的。无论这条记录是采集的时序数据还是meta data, 一样处理。当Master转发一个写入请求到slave时，必须带上版本号。一个虚拟节点将一数据更新请求写入WAL时，需要带上版本号。 + +不同虚拟节点组的数据版本号是完全独立的，互不相干的。版本号本质上是数据更新记录的transaction ID，但用来标识数据集的版本。 + +**角色(role)：** + +一个虚拟节点可以是master, slave, unsynced或offline状态。 + +- master： 具有最新的数据，容许客户端往里写入数据，一个虚拟节点组，至多一个master. +- slave：与master是同步的，但不容许客户端往里写入数据，根据配置，可以容许客户端对其进行查询。 +- unsynced: 节点处于非同步状态，比如虚拟节点刚启动、或与其他虚拟节点的连接出现故障等。处于该状态时，该虚拟节点既不能提供写入，也不能提供查询服务 +- offline: 由于宕机或网络原因，无法访问到某虚拟节点时，其他虚拟节点将该虚拟节点标为离线。但请注意，该虚拟节点本身的状态可能是unsynced或其他，但不会是离线。 + +**Quorum:** + +指数据写入成功所需要的确认数。对于异步复制，quorum设为1，具有master角色的虚拟节点自己确认即可。对于同步复制，需要至少大于等于2。原则上，Quorum >=1 并且 Quorum <= replication(副本数）。这个参数在启动一个同步模块实例时需要提供。 + +**WAL：** + +TDengine的WAL(Write Ahead Log)与cassandra的commit log, mySQL的bin log, Postgres的WAL没本质区别。没有写入数据库文件，还保存在内存的数据都会先存在WAL。当数据已经成功写入数据库数据文件，相应的WAL会被删除。但需要特别指明的是，在TDengine系统里，有几点： + +- 每个虚拟节点有自己独立的wal +- WAL里包含而且仅仅包含来自客户端的数据更新操作，每个更新操作都会被打上一个版本号 + +**复制实例：** + +复制模块只是一可执行的代码，复制实例是指正在运行的复制模块的一个实例，一个节点里，可以存在多个实例。原则上，一个节点有多少虚拟节点，就可以启动多少实例。对于副本数为1的场景，应用可以决定是否需要启动同步实例。应用启动一个同步模块的实例时，需要提供的就是虚拟节点组的配置信息，包括： + +- 虚拟节点个数，即replication number +- 各虚拟节点所在节点的信息，包括node的end point +- quorum, 需要的数据写入成功的确认数 +- 虚拟节点的初始版本号 + +## 数据复制模块的基本工作原理 + +TDengine采取的是Master-Slave模式进行同步，与流行的RAFT一致性算法比较一致。总结下来，有几点： + +1. 一个vgroup里有一到多个虚拟节点，每个虚拟节点都有自己的角色 +2. 客户端只能向角色是master的虚拟节点发起数据更新操作，因为master具有最新版本的数据，如果向非Master发起数据更新操作，会直接收到错误 +3. 客户端可以向master, 也可以向角色是Slave的虚拟节点发起查询操作，但不能对unsynced的虚拟节点发起任何操作 +4. 如果master不存在，这个vgroup是不能对外提供数据更新和查询服务的 +5. master收到客户端的数据更新操作时，会将其转发给slave节点 +6. 一个虚拟节点的版本号比master低的时候，会发起数据恢复流程，成功后，才会成为slave + +数据实时复制有三个主要流程：选主、数据转发、数据恢复。后续做详细讨论。 + +## 虚拟节点之间的网络链接 + +虚拟节点之间通过TCP进行链接，节点之间的状态交换、数据包的转发都是通过这个TCP链接(peerFd)进行。为避免竞争，两个虚拟节点之间的TCP链接，总是由IP地址(UINT32)小的节点作为TCP客户端发起。一旦TCP链接被中断，虚拟节点能通过TCP socket自动检测到，将对方标为offline。如果监测到任何错误（比如数据恢复流程），虚拟节点将主动重置该链接。 + +一旦作为客户端的节点链接不成或中断，它将周期性的每隔一秒钟去试图去链接一次。因为TCP本身有心跳机制，虚拟节点之间不再另行提供心跳。 + +如果一个unsynced节点要发起数据恢复流程，它与Master将建立起专有的TCP链接(syncFd)。数据恢复完成后，该链接会被关闭。而且为限制资源的使用，系统只容许一定数量(配置参数tsMaxSyncNum)的数据恢复的socket存在。如果超过这个数字，系统会将新的数据恢复请求延后处理。 + +任意一个节点，无论有多少虚拟节点，都会启动而且只会启动一个TCP server, 来接受来自其他虚拟节点的上述两类TCP的链接请求。当TCP socket建立起来，客户端侧发送的消息体里会带有vgId（全局唯一的vgroup ID), TCP 服务器侧会检查该vgId是否已经在该节点启动运行。如果已经启动运行，就接受其请求。如果不存在，就直接将链接请求关闭。在TDengine代码里，mnode group的vgId设置为1。 + +## 选主流程 + +当同一组的两个虚拟节点之间(vnode A, vnode B)建立连接后，他们互换status消息。status消息里包含本地存储的同一虚拟节点组内所有虚拟节点的role和version。 + +如果一个虚拟节点(vnode A)检测到与同一虚拟节点组内另外一虚拟节点（vnode B）的链接中断，vnode A将立即把vnode B的role设置为offline。无论是接收到另外一虚拟节点发来的status消息，还是检测与另外一虚拟节点的链接中断，该虚拟节点都将进入状态处理流程。状态处理流程的规则如下： + +1. 如果检测到在线的节点数没有超过一半，则将自己的状态设置为unsynced. +2. 如果在线的虚拟节点数超过一半，会检查master节点是否存在，如果存在，则会决定是否将自己状态改为slave或启动数据恢复流程 +3. 如果master不存在，则会检查自己保存的各虚拟节点的状态信息与从另一节点接收到的是否一致，如果一致，说明节点组里状态已经稳定一致，则会触发选举流程。如果不一致，说明状态还没趋于一致，即使master不存在，也不进行选主。由于要求状态信息一致才进行选举，每个虚拟节点根据同样的信息，会选出同一个虚拟节点做master，无需投票表决。 +4. 自己的状态是根据规则自己决定并修改的，并不需要其他节点同意，包括成为master。一个节点无权修改其他节点的状态。 +5. 如果一个虚拟节点检测到自己或其他虚拟节点的role发生改变，该节点会广播它自己保存的各个虚拟节点的状态信息（role和version). + +具体的流程图如下： + +
+ +选择Master的具体规则如下： + +1. 如果只有一个副本，该副本永远就是master +2. 所有副本都在线时，版本最高的被选为master +3. 在线的虚拟节点数过半，而且有虚拟节点是slave的话，该虚拟节点自动成为master +4. 对于2和3，如果多个虚拟节点满足成为master的要求，那么虚拟节点组的节点列表里，最前面的选为master + +按照上面的规则，如果所有虚拟节点都是unsynced(比如全部重启），只有所有虚拟节点上线，才能选出master，该虚拟节点组才能开始对外提供服务。当一个虚拟节点的role发生改变时，sync模块回通过回调函数notifyRole通知应用。 + +## 数据转发流程 + +如果vnode A是master, vnode B是slave, vnode A能接受客户端的写请求，而vnode B不能。当vnode A收到写的请求后，遵循下面的流程： + +
+ +1. 应用对写请求做基本的合法性检查，通过，则给改请求包打上一个版本号(version, 单调递增） +2. 应用将打上版本号的写请求封装一个WAL Head, 写入WAL(Write Ahead Log) +3. 应用调用API syncForwardToPeer，如多vnode B是slave状态，sync模块将包含WAL Head的数据包通过Forward消息发送给vnode B，否则就不转发。 +4. vnode B收到Forward消息后，调用回调函数writeToCache, 交给应用处理 +5. vnode B应用在写入成功后，都需要调用syncAckForward通知sync模块已经写入成功。 +6. 如果quorum大于1，vnode B需要等待应用的回复确认，收到确认后，vnode B发送Forward Response消息给node A。 +7. 如果quorum大于1，vnode A需要等待vnode B或其他副本对Forward消息的确认。 +8. 如果quorum大于1，vnode A收到quorum-1条确认消息后，调用回调函数confirmForward，通知应用写入成功。 +9. 如果quorum为1，上述6，7，8步不会发生。 +10. 如果要等待slave的确认，master会启动2秒的定时器（可配置），如果超时，则认为失败。 + +对于回复确认，sync模块提供的是异步回调函数，因此APP在调用syncForwardToPeer之后，无需等待，可以处理下一个操作。在Master与Slave的TCP链接管道里，可能有多个Forward消息，这些消息是严格按照应用提供的顺序排好的。对于Forward Response也是一样，TCP管道里存在多个，但都是排序好的。这个顺序，SYNC模块并没有做特别的事情，是由APP单线程顺序写来保证的(TDengine里每个vnode的写数据，都是单线程）。 + +## 数据恢复流程 + +如果一虚拟节点(vnode B) 处于unsynced状态，master存在（vnode A)，而且其版本号比master的低，它将立即启动数据恢复流程。在理解恢复流程时，需要澄清几个关于文件的概念和处理规则。 + +1. 每个文件（无论是archived data的file还是wal)都有一个index, 这需要应用来维护(vnode里，该index就是fileId*3 + 0/1/2, 对应data, head与last三个文件）。如果index为0，表示系统里最老的数据文件。对于mnode里的文件，数量是固定的，对应于acct, user, db, table等文件。 +2. 任何一个数据文件(file)有名字、大小，还有一个magic number。只有文件名、大小与magic number一致时，两个文件才判断是一样的，无需同步。Magic number可以是checksum, 也可以是简单的文件大小。怎么计算magic，换句话说，如何检测数据文件是否有效，完全由应用决定。 +3. 文件名的处理有点复杂，因为每台服务器的路径可能不一致。比如node A的TDengine的数据文件存放在 /etc/taos目录下，而node B的数据存放在 /home/jhtao目录下。因此同步模块需要应用在启动一个同步实例时提供一个path，这样两台服务器的绝对路径可以不一样，但仍然可以做对比，做同步。 +4. 当sync模块调用回调函数getFileInfo获得数据文件信息时，有如下的规则 + 1. index 为0，表示获取最老的文件，同时修改index返回给sync模块。如果index不为0，表示获取指定位置的文件。 + 2. 如果name为空，表示sync想获取位于index位置的文件信息，包括magic, size。Master节点会这么调用 + 3. 如果name不为空，表示sync想获取指定文件名和index的信息，slave节点会这么调用 + 4. 如果某个index的文件不存在，magic返回0，表示文件已经是最后一个。因此整个系统里，文件的index必须是连续的一段整数。 +5. 当sync模块调用回调函数getWalInfo获得wal信息时，有如下规则 + 1. index为0，表示获得最老的WAL文件, 返回时，index更新为具体的数字 + 2. 如果返回0，表示这是最新的一个WAL文件，如果返回值是1，表示后面还有更新的WAL文件 + 3. 返回的文件名为空，那表示没有WAL文件 +6. 无论是getFileInfo, 还是getWalInfo, 只要获取出错（不是文件不存在），返回-1即可，系统会报错，停止同步 + +整个数据恢复流程分为两大步骤，第一步，先恢复archived data(file), 然后恢复wal。具体流程如下： + +
+ +1. 通过已经建立的TCP链接，发送sync req给master节点 +2. master收到sync req后，以client的身份，向vnode B主动建立一新的专用于同步的TCP链接（syncFd) +3. 新的TCP链接建立成功后，master将开始retrieve流程，对应的，vnode B将同步启动restore流程 +4. Retrieve/Restore流程里，先处理所有archived data (vnode里的data, head, last文件），后处理WAL data。 +5. 对于archived data，master将通过回调函数getFileInfo获取数据文件的基本信息，包括文件名、magic以及文件大小。 +6. master 将获得的文件名、magic以及文件大小发给vnode B +7. vnode B将回调函数getFile获得magic和文件大小，如果两者一致，就认为无需同步，如果两者不一致 ，就认为需要同步。vnode B将结果通过消息FileAck发回master +8. 如果文件需要同步，master就调用sendfile把整个文件发往vnode B +9. 如果文件不需要同步，master(vnode A)就重复5，6，7，8，直到所有文件被处理完 + +对于WAL同步，流程如下： + +1. master节点调用回调函数getWalInfo，获取WAL的文件名。 +2. 如果getWalInfo返回值大于0，表示该文件还不是最后一个WAL，因此master调用sendfile一下把该文件发送给vnode B +3. 如果getWalInfo返回时为0，表示该文件是最后一个WAL，因为文件可能还处于写的状态中，sync模块要根据WAL Head的定义逐条读出记录，然后发往vnode B。 +4. vnode A读取TCP链接传来的数据，按照WAL Head，逐条读取，如果版本号比现有的大，调用回调函数writeToCache，交给应用处理。如果小，直接扔掉。 +5. 上述流程循环，直到所有WAL文件都被处理完。处理完后，master就会将新来的数据包通过Forward消息转发给slave。 + +从同步文件启动起，sync模块会通过inotify监控所有处理过的file以及wal。一旦发现被处理过的文件有更新变化，同步流程将中止，会重新启动。因为有可能落盘操作正在进行（比如历史数据导入，内存数据落盘），把已经处理过的文件进行了修改，需要重新同步才行。 + +对于最后一个WAL (LastWal)的处理逻辑有点复杂，因为这个文件往往是打开写的状态，有很多场景需要考虑，比如： + +- LastWal文件size在增长，需要重新读； +- LastWal文件虽然已经打开写，但内容为空； +- LastWal文件已经被关闭，应用生成了新的Last WAL文件; +- LastWal文件没有被关闭，但数据落盘的原因，没有读到完整的一条记录; +- LastWal文件没有被关闭，但数据落盘的原因，还有部分记录暂时读取不到； + +sync模块通过inotify监控LastWal文件的更新和关闭操作。而且在确认已经尽可能读完LastWal的数据后，会将对方同步状态设置为SYNC_CACHE。该状态下，master节点会将新的记录转发给vnode B，而此时vnode B并没有完成同步，需要把这些转发包先存在recv buffer里，等WAL处理完后，vnode A再把recv buffer里的数据包通过回调writeToCache交给应用处理。 + +等vnode B把这些buffered forwards处理完，同步流程才算结束，vnode B正式变为slave。 + +## Master分布均匀性问题 + +因为Master负责写、转发，消耗的资源会更多，因此Master在整个集群里分布均匀比较理想。 + +但在TDengine的设计里，如果多个虚拟节点都符合master条件，TDengine选在列表中最前面的做Master, 这样是否导致在集群里，Master数量的分布不均匀问题呢？这取决于应用的设计。 + +给一个具体例子，系统里仅仅有三个节点，IP地址分别为IP1, IP2, IP3. 在各个节点上，TDengine创建了多个虚拟节点组，每个虚拟节点组都有三个副本。如果三个副本的顺序在所有虚拟节点组里都是IP1, IP2, IP3, 那毫无疑问，master将集中在IP1这个节点，这是我们不想看到的。 + +但是，如果在创建虚拟节点组时，增加随机性，这个问题就不存在了。比如在vgroup 1, 顺序是IP1, IP2, IP3, 在vgroup 2里，顺序是IP2, IP3, IP1, 在vgroup 3里，顺序是IP3, IP1, IP2。最后master的分布会是均匀的。 + +因此在创建一个虚拟节点组时，应用需要保证节点的顺序是round robin或完全随机。 + +## 少数虚拟节点写入成功的问题 + +在某种情况下，写入成功的确认数大于0，但小于配置的Quorum, 虽然有虚拟节点数据更新成功，master仍然会认为数据更新失败，并通知客户端写入失败。 + +这个时候，系统存在数据不一致的问题，因为有的虚拟节点已经写入成功，而有的写入失败。一个处理方式是，Master重置(reset)与其他虚拟节点的连接，该虚拟节点组将自动进入选举流程。按照规则，已经成功写入数据的虚拟节点将成为新的master，组内的其他虚拟节点将从master那里恢复数据。 + +因为写入失败，客户端会重新写入数据。但对于TDengine而言，是OK的。因为时序数据都是有时间戳的，时间戳相同的数据更新操作，第一次会执行，但第二次会自动扔掉。对于Meta Data(增加、删除库、表等等）的操作，也是OK的。一张表、库已经被创建或删除，再创建或删除，不会被执行的。 + +在TDengine的设计里，虚拟节点与虚拟节点之间，是一个TCP链接，是一个pipeline，数据块一个接一个按顺序在这个pipeline里等待处理。一旦某个数据块的处理失败，这个链接会被重置，后续的数据块的处理都会失败。因此不会存在Pipeline里一个数据块更新失败，但下一个数据块成功的可能。 + +## Split Brain的问题 + +选举流程中，有个强制要求，那就是一定有超过半数的虚拟节点在线。但是如果replication正好是偶数，这个时候，完全可能存在splt brain问题。 + +为解决这个问题，TDengine提供Arbitrator的解决方法。Arbitrator是一个节点，它的任务就是接受任何虚拟节点的链接请求，并保持它。 + +在启动复制模块实例时，在配置参数中，应用可以提供Arbitrator的IP地址。如果是奇数个副本，复制模块不会与这个arbitrator去建立链接，但如果是偶数个副本，就会主动去建立链接。 + +Arbitrator的程序tarbitrator.c在复制模块的同一目录, 编译整个系统时，会在bin目录生成。命令行参数“-？”查看可以配置的参数，比如绑定的IP地址，监听的端口号。 + +## 与RAFT相比的异同 + +数据一致性协议流行的有两种，Paxos与Raft. 本设计的实现与Raft有很多类同之处，下面做一些比较 + +相同之处： + +- 三大流程一致：Raft里有Leader election, replication, safety，完全对应TDengine的选举、数据转发、数据恢复三个流程 +- 节点状态定义一致：Raft里每个节点有Leader, Follower, Candidate三个状态，TDengine里是Master, Slave, Unsynced, Offline。多了一个offlince, 但本质上是一样的，因为offline是外界看一个节点的状态，但该节点本身是处于master, slave 或unsynced的 +- 数据转发流程完全一样，Master(leader)需要等待回复确认。 +- 数据恢复流程几乎一样，Raft没有涉及历史数据同步问题，只考虑了WAL数据同步 + +不同之处： + +- 选举流程不一样：Raft里任何一个节点是candidate时，主动向其他节点发出vote request, 如果超过半数回答Yes, 这个candidate就成为Leader,开始一个新的term. 而TDengine的实现里，节点上线、离线或角色改变都会触发状态消息在节点组类传播，等节点组里状态稳定一致之后才触发选举流程，因为状态稳定一致，基于同样的状态信息，每个节点做出的决定会是一致的，一旦某个节点符合成为master的条件，无需其他节点认可，它会自动将自己设为master。TDengine里，任何一个节点检测到其他节点或自己的角色发生改变，就会给节点组内其他节点进行广播的。Raft里不存在这样的机制，因此需要投票来解决。 +- 对WAL的一条记录，Raft用term + index来做唯一标识。但TDengine只用version（类似index)，在TDengine实现里，仅仅用version是完全可行的, 因为TDengine的选举机制，没有term的概念。 + +如果整个虚拟节点组全部宕机，重启，但不是所有虚拟节点都上线，这个时候TDengine是不会选出master的，因为未上线的节点有可能有最高version的数据。而RAFT协议，只要超过半数上线，就会选出Leader。 + +## Meta Data的数据复制 + +TDengine里存在时序数据，也存在Meta Data。Meta Data对数据的可靠性要求更高，那么TDengine设计能否满足要求呢？下面做个仔细分析 + +TDengine里Meta Data包括以下： + +- account 信息 +- 一个account下面，可以有多个user, 多个DB +- 一个DB下面有多个vgroup +- 一个DB下面有多个stable +- 一个vgroup下面有多个table +- 整个系统有多个mnode, dnode +- 一个dnode可以有多个vnode + +上述的account, user, DB, vgroup, table, stable, mnode, dnode都有自己的属性，这些属性是TDengine自己定义的，不会开放给用户进行修改。这些Meta Data的查询都比较简单，都可以采用key-value模型进行存储。这些Meta Data还具有几个特点： + +1. 上述的Meta Data之间有一定的层级关系，比如必须先创建DB，才能创建table, stable。只有先创建dnode，才可能创建vnode, 才可能创建vgroup。因此他们创建的顺序是绝对不能错的。 +2. 在客户端应用的数据更新操作得到TDengine服务器侧确认后，所执行的数据更新操作绝对不能丢失。否则会造成客户端应用与服务器的数据不一致。 +3. 上述的Meta Data是容许重复操作的。比如插入新记录后，再插入一次，删除一次后，再删除一次，更新一次后，再更新一次，不会对系统产生任何影响，不会改变系统任何状态。 + +对于特点1，本设计里，数据的写入是单线程的，按照到达的先后顺序，给每个数据更新操作打上版本号，版本号大的记录一定是晚于版本号小的写入系统，数据写入顺序是100%保证的，绝对不会让版本号大的记录先写入。复制过程中，数据块的转发也是严格按照顺序进行的，因此TDengine的数据复制设计是能保证Meta Data的创建顺序的。 + +对于特点2，只要Quorum数设置等于replica，那么一定能保证回复确认过的数据更新操作不会在服务器侧丢失。即使某节点永不起来，只要超过一半的节点还是online, 查询服务不会受到任何影响。这时，如果某个节点离线超过一定时长，系统可以自动补充新的节点，以保证在线的节点数在绝大部分时间是100%的。 + +对于特点3，完全可能发生，服务器确实持久化存储了某一数据更新操作，但客户端应用出了问题，认为操作不成功，它会重新发起操作。但对于Meta Data而言，没有关系，客户端可以再次发起同样的操作，不会有任何影响。 + +总结来看，只要quorum设置大于一，本数据复制的设计是能满足Meta Data的需求的。目前，还没有发现漏洞。 diff --git a/documentation20/webdocs/markdowndocs/taosd-ch.md b/documentation20/webdocs/markdowndocs/taosd-ch.md new file mode 100644 index 0000000000..8143137c79 --- /dev/null +++ b/documentation20/webdocs/markdowndocs/taosd-ch.md @@ -0,0 +1,107 @@ +# TDengine 2.0 执行代码taosd的设计 + +逻辑上，TDengine系统包含dnode, taosc和App，dnode是服务器侧执行代码taosd的一个运行实例，因此taosd是TDengine的核心，本文对taosd的设计做一简单的介绍，模块内的实现细节请见其他文档。 + +## 系统模块图 +taosd包含rpc, dnode, vnode, tsdb, query, cq, sync, wal, mnode, http, monitor等模块，具体如下图： + +
+ +taosd的启动入口是dnode模块，dnode然后启动其他模块，包括可选配置的http, monitor模块。taosc或dnode之间交互的消息都是通过rpc模块进行，dnode模块根据接收到的消息类型，将消息分发到vnode或mnode的消息队列，或由dnode模块自己消费。dnode的工作线程(worker)消费消息队列里的消息，交给mnode或vnode进行处理。下面对各个模块做简要说明。 + +## RPC模块 + +该模块负责taosd与taosc, 以及其他数据节点之间的通讯。TDengine没有采取标准的HTTP或gRPC等第三方工具，而是实现了自己的通讯模块RPC。 + +考虑到物联网场景下，数据写入的包一般不大，因此除支持TCP链接之外，RPC还支持UDP链接。当数据包小于15K时，RPC将采用UDP方式进行链接，否则将采用TCP链接。对于查询类的消息，RPC不管包的大小，总是采取TCP链接。对于UDP链接，RPC实现了自己的超时、重传、顺序检查等机制，以保证数据可靠传输。 + +RPC模块还提供数据压缩功能，如果数据包的字节数超过系统配置参数compressMsgSize, RPC在传输中将自动压缩数据，以节省带宽。 + +为保证数据的安全和数据的integrity, RPC模块采用MD5做数字签名，对数据的真实性和完整性进行认证。 + +## DNODE模块 + +该模块是整个taosd的入口，它具体负责如下任务： + +- 系统的初始化，包括 + - 从文件taos.cfg读取系统配置参数，从文件dnodeCfg.json读取数据节点的配置参数； + - 启动RPC模块，并建立起与taosc通讯的server链接，与其他数据节点通讯的server链接； + - 启动并初始化dnode的内部管理, 该模块将扫描该数据节点已有的vnode，并打开它们； + - 初始化可配置的模块，如mnode, http, monitor等。 +- 数据节点的管理，包括 + - 定时的向mnode发送status消息，报告自己的状态； + - 根据mnode的指示，创建、改变、删除vnode； + - 根据mnode的指示，修改自己的配置参数； +- 消息的分发、消费，包括 + - 为每一个vnode和mnode的创建并维护一个读队列、一个写队列； + - 将从taosc或其他数据节点来的消息，根据消息类型，将其直接分发到不同的消息队列，或由自己的管理模块直接消费； + - 维护一个读的线程池，消费读队列的消息，交给vnode或mnode处理。为支持高并发，一个读线程(Worker)可以消费多个队列的消息，一个读队列可以由多个worker消费； + - 维护一个写的线程池，消费写队列的消息，交给vnode或mnode处理。为保证写操作的序列化，一个写队列只能由一个写线程负责，但一个写线程可以负责多个写队列。 + +taosd的消息消费由dnode通过读写线程池进行控制，是系统的中枢。该模块内的结构体图如下： + +
+ +## VNODE模块 + +vnode是一独立的数据存储查询逻辑单元，但因为一个vnode只能容许一个DB，因此vnode内部没有account, DB, user等概念。为实现更好的模块化、封装以及未来的扩展，它有很多子模块，包括负责存储的TSDB，负责查询的Query, 负责数据复制的sync，负责数据库日志的的wal, 负责连续查询的cq(continuous query), 负责事件触发的流计算的event等模块，这些子模块只与vnode模块发生关系，与其他模块没有任何调用关系。模块图如下： + +
+vnode模块向下，与dnodeVRead，dnodeVWrite发生互动，向上，与子模块发生互动。它主要的功能有： + +- 协调各个子模块的互动。各个子模块之间都不直接调用，都需要通过vnode模块进行； +- 对于来自taosc或mnode的写操作，vnode模块将其分解为写日志(wal), 转发(sync), 本地存储(tsdb)子模块的操作； +- 对于查询操作，分发到query模块进行。 + +一个数据节点里有多个vnode, 因此vnode模块是有多个运行实例的。每个运行实例是完全独立的。 + +vnode与其子模块是通过API直接调用，而不是通过消息队列传递。而且各个子模块只与vnode模块有交互，不与dnode, rpc等模块发生任何直接关联。 + +## MNODE模块 + +mnode是整个系统的大脑，负责整个系统的资源调度，负责meta data的管理与存储。 + +一个运行的系统里，只有一个mnode，但它有多个副本（由系统配置参数numOfMpeers控制）。这些副本分布在不同的dnode里，目的是保证系统的高可靠运行。副本之间的数据复制是采用同步而非异步的方式，以确保数据的一致性，确保数据不会丢失。这些副本会自动选举一个Master，其他副本是slave。所有数据更新类的操作，都只能在master上进行，而查询类的可以在slave节点上进行。代码实现上，同步模块与vnode共享，但mnode被分配一个特殊的vgroup ID: 1，而且quorum大于1。整个集群系统是由多个dnode组成的，运行的mnode的副本数不可能超过dnode的个数，但不会超过配置的副本数。如果某个mnode副本宕机一段时间，只要超过半数的mnode副本仍在运行，运行的mnode会自动根据整个系统的资源情况，在其他dnode里再启动一个mnode, 以保证运行的副本数。 + +各个dnode通过信息交换，保存有mnode各个副本的End Point列表，并向其中的master节点定时（间隔由系统配置参数statusInterval控制）发送status消息，消息体里包含该dnode的CPU、内存、剩余存储空间、vnode个数，以及各个vnode的状态(存储空间、原始数据大小、记录条数、角色等）。这样mnode就了解整个系统的资源情况，如果用户创建新的表，就可以决定需要在哪个dnode创建；如果增加或删除dnode, 或者监测到某dnode数据过热、或离线太长，就可以决定需要挪动那些vnode，以实现负载均衡。 + +mnode里还负责account, user, DB, stable, table, vgroup, dnode的创建、删除与更新。mnode不仅把这些entity的meta data保存在内存，还做持久化存储。但为节省内存，各个表的标签值不保存在mnode（保存在vnode)，而且子表不维护自己的schema, 而是与stable共享。为减小mnode的查询压力，taosc会缓存table、stable的schema。对于查询类的操作，各个slave mnode也可以提供，以减轻master压力。 + +## TSDB模块 +TSDB模块是VNODE中的负责快速高并发地存储和读取属于该VNODE的表的元数据及采集的时序数据的引擎。除此之外，TSDB还提供了表结构的修改、表标签值的修改等功能。TSDB提供API供VNODE和Query等模块调用。TSDB中存储了两类数据，1：元数据信息；2：时序数据 + +### 元数据信息 +TSDB中存储的元数据包含属于其所在的VNODE中表的类型，schema的定义等。对于超级表和超级表下的子表而言，又包含了tag的schema定义以及子表的tag值等。对于元数据信息而言，TSDB就相当于一个全内存的KV型数据库，属于该VNODE的表对象全部在内存中，方便快速查询表的信息。除此之外，TSDB还对其中的子表，按照tag的第一列取值做了全内存的索引，大大加快了对于标签的过滤查询。TSDB中的元数据的最新状态在落盘时，会以追加（append-only）的形式，写入到meta文件中。meta文件只进行追加操作，即便是元数据的删除，也会以一条记录的形式写入到文件末尾。TSDB也提供了对于元数据的修改操作，如表schema的修改，tag schema的修改以及tag值的修改等。 + +### 时序数据 +每个TSDB在创建时，都会事先分配一定量的内存缓冲区，且内存缓冲区的大小可配可修改。表采集的时序数据，在写入TSDB时，首先以追加的方式写入到分配的内存缓冲区中，同时建立基于时间戳的内存索引，方便快速查询。当内存缓冲区的数据积累到一定的程度时（达到内存缓冲区总大小的1/3），则会触发落盘操作，将缓冲区中的数据持久化到硬盘文件上。时序数据在内存缓冲区中是以行（row）的形式存储的。 + +而时序数据在写入到TSDB的数据文件时，是以列（column）的形式存储的。TSDB中的数据文件包含多个数据文件组，每个数据文件组中又包含.head、.data和.last三个文件，如（v2f1801.head、v2f1801.data、v2f1801.last）数据文件组。TSDB中的数据文件组是按照时间跨度进行分片的，默认是10天一个文件组，且可通过配置文件及建库选项进行配置。分片的数据文件组又按照编号递增排列，方便快速定位某一时间段的时序数据，高效定位数据文件组。时序数据在TSDB的数据文件中是以块的形式进行列式存储的，每个块中只包含一张表的数据，且数据在一个块中是按照时间顺序递增排列的。在一个数据文件组中，.head文件负责存储数据块的索引及统计信息，如每个块的位置，压缩算法，时间戳范围等。存储在.head文件中一张表的索引信息是按照数据块中存储的数据的时间递增排列的，方便进行折半查找等工作。.head和.last文件是存储真实数据块的文件，若数据块中的数据累计到一定程度，则会写入.data文件中，否则，会写入.last文件中，等待下次落盘时合并数据写入.data文件中，从而大大减少文件中块的个数，避免数据的过度碎片化。 + +## Query模块 +该模块负责整体系统的查询处理。客户端调用该该模块进行SQL语法解析，并将查询或写入请求发送到vnode，同时负责针对超级表的查询进行二阶段的聚合操作。在Vnode端，该模块调用TSDB模块读取系统中存储的数据进行查询处理。Query模块还定义了系统能够支持的全部查询函数，查询函数的实现机制与查询框架无耦合，可以在不修改查询流程的情况下动态增加查询函数。详细的设计请参见《TDengine 2.0查询模块设计》。 + +## SYNC模块 +该模块实现数据的多副本复制，包括vnode与mnode的数据复制，支持异步和同步两种复制方式，以满足meta data与时序数据不同复制的需求。因为它为mnode与vnode共享，系统为mnode副本预留了一个特殊的vgroup ID:1。因此vnode group的ID是从2开始的。 + +每个vnode/mnode模块实例会有一对应的sync模块实例，他们是一一对应的。详细设计请见TDengine 2.0 数据复制模块设计 + +## WAL模块 +该模块负责将新插入的数据写入write ahead log(WAL), 为vnode, mnode共享。以保证服务器crash或其他故障，能从WAL中恢复数据。 + +每个vnode/mnode模块实例会有一对应的wal模块实例，是完全一一对应的。WAL的落盘操作由两个参数walLevel, fsync控制。看具体场景，如果要100%保证数据不会丢失，需要将walLevel配置为2，fsync设置为0，每条数据插入请求，都会实时落盘后，才会给应用确认 + +## HTTP模块 +该模块负责处理系统对外的RESTful接口，可以通过配置，由dnode启动或停止。 + +该模块将接收到的RESTful请求，做了各种合法性检查后，将其变成标准的SQL语句，通过taosc的异步接口，将请求发往整个系统中的任一dnode。收到处理后的结果后，再翻译成HTTP协议，返回给应用。 + +如果HTTP模块启动，就意味着启动了一个taosc的实例。任一一个dnode都可以启动该模块，以实现对RESTful请求的分布式处理。 + +## Monitor模块 +该模块负责检测一个dnode的运行状态，可以通过配置，由dnode启动或停止。原则上，每个dnode都应该启动一个monitor实例。 + +Monitor采集TDengine里的关键操作，比如创建、删除、更新账号、表、库等，而且周期性的收集CPU、内存、网络等资源的使用情况（采集周期由系统配置参数monitorInterval控制）。获得这些数据后，monitor模块将采集的数据写入系统的日志库(DB名字由系统配置参数monitorDbName控制）。 + +Monitor模块使用taosc来将采集的数据写入系统，因此每个monitor实例，都有一个taosc运行实例。 + diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index aee4bbadf6..ce5c052e97 100644 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -79,19 +79,19 @@ osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) #echo "osinfo: ${osinfo}" os_type=0 if echo $osinfo | grep -qwi "ubuntu" ; then - echo "This is ubuntu system" +# echo "This is ubuntu system" os_type=1 elif echo $osinfo | grep -qwi "debian" ; then - echo "This is debian system" +# echo "This is debian system" os_type=1 elif echo $osinfo | grep -qwi "Kylin" ; then - echo "This is Kylin system" +# echo "This is Kylin system" os_type=1 elif echo $osinfo | grep -qwi "centos" ; then - echo "This is centos system" +# echo "This is centos system" os_type=2 elif echo $osinfo | grep -qwi "fedora" ; then - echo "This is fedora system" +# echo "This is fedora system" os_type=2 else echo "${osinfo}: This is an officially unverified linux system, If there are any problems with the installation and operation, " @@ -135,7 +135,7 @@ do esac done -echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" +#echo "verType=${verType} interactiveFqdn=${interactiveFqdn}" function kill_taosd() { pid=$(ps -ef | grep "taosd" | grep -v "grep" | awk '{print $2}') @@ -240,7 +240,9 @@ function install_config() { # first full-qualified domain name (FQDN) for TDengine cluster system echo - echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join OR leave it blank to build one${NC} :" + echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" read firstEp while true; do if [ ! -z "$firstEp" ]; then @@ -670,10 +672,11 @@ function install_TDengine() { echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" fi - if [ ! -z "$firstEp" ]; then - echo - echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp ${GREEN_DARK} to login into cluster, then execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}in TAOS shell to add this new node into the clsuter${NC}" - echo + if [ ! -z "$firstEp" ]; then + echo + echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp${GREEN_DARK} to login into cluster, then${NC}" + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo fi echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" echo diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index ef66f9fa4f..cffe06e30c 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -113,7 +113,9 @@ function install_config() { # first full-qualified domain name (FQDN) for TDengine cluster system echo - echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join OR leave it blank to build one${NC} :" + echo -e -n "${GREEN}Enter FQDN:port (like h1.taosdata.com:6030) of an existing TDengine cluster node to join${NC}" + echo + echo -e -n "${GREEN}OR leave it blank to build one${NC}:" read firstEp while true; do if [ ! -z "$firstEp" ]; then @@ -265,10 +267,11 @@ function install_TDengine() { echo -e "${GREEN_DARK}To access TDengine ${NC}: use ${GREEN_UNDERLINE}taos${NC} in shell${NC}" - if [ ! -z "$firstEp" ]; then - echo - echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp ${GREEN_DARK} to login into cluster, then execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}in TAOS shell to add this new node into the clsuter${NC}" - echo + if [ ! -z "$firstEp" ]; then + echo + echo -e "${GREEN_DARK}Please run${NC}: taos -h $firstEp${GREEN_DARK} to login into cluster, then${NC}" + echo -e "${GREEN_DARK}execute ${NC}: create dnode 'newDnodeFQDN:port'; ${GREEN_DARK}to add this new node${NC}" + echo fi echo echo -e "\033[44;32;1mTDengine is installed successfully!${NC}" diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 7aed8c05ca..db5dd6a520 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -14,10 +14,10 @@ */ #define _DEFAULT_SOURCE +#include "os.h" #include "tutil.h" #include "tbalance.h" #include "tsync.h" -#include "ttime.h" #include "ttimer.h" #include "tglobal.h" #include "tdataformat.h" diff --git a/src/client/src/TSDBJNIConnector.c b/src/client/src/TSDBJNIConnector.c index 549a0e8d0d..eee255f55b 100644 --- a/src/client/src/TSDBJNIConnector.c +++ b/src/client/src/TSDBJNIConnector.c @@ -18,7 +18,6 @@ #include "tlog.h" #include "tscUtil.h" #include "tsclient.h" -#include "ttime.h" #include "com_taosdata_jdbc_TSDBJNIConnector.h" diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 17f6c97ea1..c4f768f6ac 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -27,7 +27,6 @@ #include "tscSubquery.h" #include "tscompression.h" #include "tsqlfunction.h" -#include "ttime.h" #include "tutil.h" #define GET_INPUT_CHAR(x) (((char *)((x)->aInputElemBuf)) + ((x)->startOffset) * ((x)->inputBytes)) @@ -2108,7 +2107,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { } } - tfree(pData); + taosTFree(pData); } /* diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 7f336daa91..a8f287d499 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -326,7 +326,7 @@ static void tscProcessServerVer(SSqlObj *pSql) { STR_WITH_SIZE_TO_VARSTR(vx, v, t); tscSetLocalQueryResult(pSql, vx, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - tfree(vx); + taosTFree(vx); } static void tscProcessClientVer(SSqlObj *pSql) { @@ -342,7 +342,7 @@ static void tscProcessClientVer(SSqlObj *pSql) { STR_WITH_SIZE_TO_VARSTR(v, version, t); tscSetLocalQueryResult(pSql, v, pExpr->aliasName, pExpr->resType, pExpr->resBytes); - tfree(v); + taosTFree(v); } static void tscProcessServStatus(SSqlObj *pSql) { diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index bf76b8cbe8..394b725781 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -230,7 +230,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (ds == NULL) { tscError("%p failed to create merge structure", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pReducer); + taosTFree(pReducer); return; } @@ -257,7 +257,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (ds->filePage.num == 0) { // no data in this flush, the index does not increase tscDebug("%p flush data is empty, ignore %d flush record", pSql, idx); - tfree(ds); + taosTFree(ds); continue; } @@ -267,7 +267,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // no data actually, no need to merge result. if (idx == 0) { - tfree(pReducer); + taosTFree(pReducer); return; } @@ -275,7 +275,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd SCompareParam *param = malloc(sizeof(SCompareParam)); if (param == NULL) { - tfree(pReducer); + taosTFree(pReducer); return; } param->pLocalData = pReducer->pLocalDataSrc; @@ -288,8 +288,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator); if (pReducer->pLoserTree == NULL || pRes->code != 0) { - tfree(param); - tfree(pReducer); + taosTFree(param); + taosTFree(pReducer); return; } @@ -332,14 +332,14 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL || /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { - tfree(pReducer->pTempBuffer); - tfree(pReducer->discardData); - tfree(pReducer->pResultBuf); - tfree(pReducer->pFinalRes); - tfree(pReducer->prevRowOfInput); - tfree(pReducer->pLoserTree); - tfree(param); - tfree(pReducer); + taosTFree(pReducer->pTempBuffer); + taosTFree(pReducer->discardData); + taosTFree(pReducer->pResultBuf); + taosTFree(pReducer->pFinalRes); + taosTFree(pReducer->prevRowOfInput); + taosTFree(pReducer->pLoserTree); + taosTFree(param); + taosTFree(pReducer); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return; } @@ -516,38 +516,38 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tVariantDestroy(&pCtx->tag); if (pCtx->tagInfo.pTagCtxList != NULL) { - tfree(pCtx->tagInfo.pTagCtxList); + taosTFree(pCtx->tagInfo.pTagCtxList); } } - tfree(pLocalReducer->pCtx); + taosTFree(pLocalReducer->pCtx); } - tfree(pLocalReducer->prevRowOfInput); + taosTFree(pLocalReducer->prevRowOfInput); - tfree(pLocalReducer->pTempBuffer); - tfree(pLocalReducer->pResultBuf); + taosTFree(pLocalReducer->pTempBuffer); + taosTFree(pLocalReducer->pResultBuf); if (pLocalReducer->pResInfo != NULL) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - tfree(pLocalReducer->pResInfo[i].interResultBuf); + taosTFree(pLocalReducer->pResInfo[i].interResultBuf); } - tfree(pLocalReducer->pResInfo); + taosTFree(pLocalReducer->pResInfo); } if (pLocalReducer->pLoserTree) { - tfree(pLocalReducer->pLoserTree->param); - tfree(pLocalReducer->pLoserTree); + taosTFree(pLocalReducer->pLoserTree->param); + taosTFree(pLocalReducer->pLoserTree); } - tfree(pLocalReducer->pFinalRes); - tfree(pLocalReducer->discardData); + taosTFree(pLocalReducer->pFinalRes); + taosTFree(pLocalReducer->discardData); tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel, pLocalReducer->numOfVnode); for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) { - tfree(pLocalReducer->pLocalDataSrc[i]); + taosTFree(pLocalReducer->pLocalDataSrc[i]); } pLocalReducer->numOfBuffer = 0; @@ -593,7 +593,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } *pOrderDesc = tOrderDesCreate(orderIdx, numOfGroupByCols, pModel, pQueryInfo->order.order); - tfree(orderIdx); + taosTFree(orderIdx); if (*pOrderDesc == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -705,7 +705,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - tfree(pSchema); + taosTFree(pSchema); return pRes->code; } @@ -746,7 +746,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } *pFinalModel = createColumnModel(pSchema, size, capacity); - tfree(pSchema); + taosTFree(pSchema); return TSDB_CODE_SUCCESS; } @@ -765,7 +765,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); } - tfree(pMemBuffer); + taosTFree(pMemBuffer); } /** @@ -1036,10 +1036,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO pFinalDataPage->num = 0; for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - tfree(pResPages[i]); + taosTFree(pResPages[i]); } - tfree(pResPages); + taosTFree(pResPages); } static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index ae2370cd56..2f245ee513 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -30,7 +30,6 @@ #include "tscLog.h" #include "tscSubquery.h" #include "tstoken.h" -#include "ttime.h" #include "tdataformat.h" @@ -1388,7 +1387,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { assert(taos_errno(pSql) == code); taos_free_result(pSql); - tfree(pSupporter); + taosTFree(pSupporter); fclose(fp); pParentSql->res.code = code; @@ -1453,7 +1452,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } } - tfree(tokenBuf); + taosTFree(tokenBuf); free(line); if (count > 0) { @@ -1466,7 +1465,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } else { taos_free_result(pSql); - tfree(pSupporter); + taosTFree(pSupporter); fclose(fp); pParentSql->fp = pParentSql->fetchFp; @@ -1496,7 +1495,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); - tfree(pSupporter) + taosTFree(pSupporter) tscQueueAsyncRes(pSql); return; diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 0cf69dfd46..3ddfc1c341 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -613,7 +613,7 @@ int taos_stmt_execute(TAOS_STMT* stmt) { if (sql == NULL) { ret = TSDB_CODE_TSC_OUT_OF_MEMORY; } else { - tfree(pStmt->pSql->sqlstr); + taosTFree(pStmt->pSql->sqlstr); pStmt->pSql->sqlstr = sql; SSqlObj* pSql = taos_query((TAOS*)pStmt->taos, pStmt->pSql->sqlstr); ret = taos_errno(pSql); diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index 602b4654df..363243b1d7 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -16,7 +16,6 @@ #include "os.h" #include "tscLog.h" #include "tsclient.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "taosmsg.h" diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 44236face0..4e0c238ac0 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -29,7 +29,6 @@ #include "tsclient.h" #include "tstoken.h" #include "tstrbuild.h" -#include "ttime.h" #include "ttokendef.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -737,7 +736,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableNa assert(pTableMetaInfo->pTableMeta == NULL); } - tfree(oldName); + taosTFree(oldName); return TSDB_CODE_SUCCESS; } @@ -3648,7 +3647,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { taosStringBuilderDestroy(&sb1); - tfree(segments); + taosTFree(segments); invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); return ret; @@ -3661,7 +3660,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, pQueryInfo->tagCond.tbnameCond.cond = strdup(str); taosStringBuilderDestroy(&sb1); - tfree(segments); + taosTFree(segments); return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 521280af87..3fd0aa79a6 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -23,7 +23,6 @@ #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "tlockfree.h" @@ -196,7 +195,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { pSql->cmd.command = TSDB_SQL_HB; if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { - tfree(pSql); + taosTFree(pSql); return; } @@ -1538,7 +1537,7 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pCmd->payloadLen = pMsg - (char*)pInfoMsg; pCmd->msgType = TSDB_MSG_TYPE_CM_TABLE_META; - tfree(tmpData); + taosTFree(tmpData); assert(msgLen + minMsgSize() <= pCmd->allocSize); return TSDB_CODE_SUCCESS; @@ -1572,7 +1571,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen); } - tfree(tmpData); + taosTFree(tmpData); pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SCMMultiTableInfoMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META; @@ -1962,7 +1961,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); - tfree(pTableMeta); + taosTFree(pTableMeta); return 0; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index ff050dbbbf..10720874e2 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -474,7 +474,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { pRes->rspType = 0; pSql->numOfSubs = 0; - tfree(pSql->pSubs); + taosTFree(pSql->pSubs); assert(pSql->fp == NULL); @@ -753,7 +753,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { if (sqlLen > tsMaxSQLStringLen) { tscError("%p sql too long", pSql); pRes->code = TSDB_CODE_TSC_INVALID_SQL; - tfree(pSql); + taosTFree(pSql); return pRes->code; } @@ -762,7 +762,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscError("%p failed to malloc sql string buffer", pSql); tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(taos), pObj); - tfree(pSql); + taosTFree(pSql); return pRes->code; } @@ -896,7 +896,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (tblListLen > MAX_TABLE_NAME_LENGTH) { tscError("%p tableNameList too long, length:%d, maximum allowed:%d", pSql, tblListLen, MAX_TABLE_NAME_LENGTH); pRes->code = TSDB_CODE_TSC_INVALID_SQL; - tfree(pSql); + taosTFree(pSql); return pRes->code; } @@ -904,7 +904,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { if (str == NULL) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscError("%p failed to malloc sql string buffer", pSql); - tfree(pSql); + taosTFree(pSql); return pRes->code; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index b07627c87b..20b5e86090 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -21,7 +21,6 @@ #include "tsched.h" #include "tcache.h" #include "tsclient.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" @@ -156,7 +155,7 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0); taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), true); - tfree(pTableMetaInfo->vgroupList); + taosTFree(pTableMetaInfo->vgroupList); tscSetRetryTimer(pStream, pStream->pSql, retryDelay); return; @@ -260,9 +259,9 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); - tfree(pSql->pSubs); + taosTFree(pSql->pSubs); pSql->numOfSubs = 0; - tfree(pTableMetaInfo->vgroupList); + taosTFree(pTableMetaInfo->vgroupList); tscSetNextLaunchTimer(pStream, pSql); } } @@ -592,6 +591,6 @@ void taos_close_stream(TAOS_STREAM *handle) { tscFreeSqlObj(pSql); pStream->pSql = NULL; - tfree(pStream); + taosTFree(pStream); } } diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 6e572d94d1..9569987de0 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -18,7 +18,6 @@ #include "trpc.h" #include "tsclient.h" #include "tsocket.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "tscLog.h" diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 15e02799aa..fed790b9af 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -183,7 +183,7 @@ SJoinSupporter* tscCreateJoinSupporter(SSqlObj* pSql, SSubqueryState* pState, in pSupporter->uid = pTableMetaInfo->pTableMeta->id.uid; assert (pSupporter->uid != 0); - getTmpfilePath("join-", pSupporter->path); + taosGetTmpfilePath("join-", pSupporter->path); pSupporter->f = fopen(pSupporter->path, "w"); // todo handle error @@ -215,7 +215,7 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { pSupporter->f = NULL; } - tfree(pSupporter->pIdTagList); + taosTFree(pSupporter->pIdTagList); tscTagCondRelease(&pSupporter->tagCond); free(pSupporter); } @@ -407,7 +407,7 @@ void freeJoinSubqueryObj(SSqlObj* pSql) { } } - tfree(pState); + taosTFree(pState); pSql->numOfSubs = 0; } @@ -773,7 +773,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // continue to retrieve ts-comp data from vnode if (!pRes->completed) { - getTmpfilePath("ts-join", pSupporter->path); + taosGetTmpfilePath("ts-join", pSupporter->path); pSupporter->f = fopen(pSupporter->path, "w"); pRes->row = pRes->numOfRows; @@ -797,7 +797,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow tscResetForNextRetrieve(&pSql->res); assert(pSupporter->f == NULL); - getTmpfilePath("ts-join", pSupporter->path); + taosGetTmpfilePath("ts-join", pSupporter->path); // TODO check for failure pSupporter->f = fopen(pSupporter->path, "w"); @@ -1317,12 +1317,12 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState SRetrieveSupport* pSupport = pSub->param; - tfree(pSupport->localBuffer); + taosTFree(pSupport->localBuffer); pthread_mutex_unlock(&pSupport->queryMutex); pthread_mutex_destroy(&pSupport->queryMutex); - tfree(pSupport); + taosTFree(pSupport); tscFreeSqlObj(pSub); } @@ -1358,7 +1358,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { if (ret != 0) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscQueueAsyncRes(pSql); - tfree(pMemoryBuf); + taosTFree(pMemoryBuf); return ret; } @@ -1386,7 +1386,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); if (trs->localBuffer == NULL) { tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - tfree(trs); + taosTFree(trs); break; } @@ -1404,8 +1404,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - tfree(trs->localBuffer); - tfree(trs); + taosTFree(trs->localBuffer); + taosTFree(trs); break; } @@ -1450,12 +1450,12 @@ static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { taos_free_result(pSql); - tfree(trsupport->localBuffer); + taosTFree(trsupport->localBuffer); pthread_mutex_unlock(&trsupport->queryMutex); pthread_mutex_destroy(&trsupport->queryMutex); - tfree(trsupport); + taosTFree(trsupport); } static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); @@ -1572,7 +1572,7 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO tscLocalReducerEnvDestroy(trsupport->pExtMemBuffer, trsupport->pOrderDescriptor, trsupport->pFinalColModel, pState->numOfTotal); - tfree(trsupport->pState); + taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes @@ -1651,7 +1651,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p pParentSql->res.row = 0; // only free once - tfree(trsupport->pState); + taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); // set the command flag must be after the semaphore been correctly set. @@ -1855,7 +1855,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) } taos_free_result(tres); - tfree(pSupporter); + taosTFree(pSupporter); if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { return; @@ -1864,7 +1864,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) tscDebug("%p Async insertion completed, total inserted:%" PRId64, pParentObj, pParentObj->res.numOfRows); // release data block data - tfree(pState); + taosTFree(pState); // restore user defined fp pParentObj->fp = pParentObj->fetchFp; @@ -1968,11 +1968,11 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { _error: for(int32_t j = 0; j < numOfSub; ++j) { - tfree(pSql->pSubs[j]->param); + taosTFree(pSql->pSubs[j]->param); taos_free_result(pSql->pSubs[j]); } - tfree(pState); + taosTFree(pState); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2143,7 +2143,7 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker - tfree(pRes->tsrow); + taosTFree(pRes->tsrow); return pRes->tsrow; } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 42356108b1..a252beec33 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -18,7 +18,6 @@ #include "tcache.h" #include "trpc.h" #include "tsystem.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "tsched.h" diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 73d6f0e592..9e0ff8a9bf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -245,7 +245,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { } pQueryInfo->fillType = TSDB_FILL_NONE; - tfree(pQueryInfo->fillVal); + taosTFree(pQueryInfo->fillVal); } int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { @@ -259,9 +259,9 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { - tfree(pRes->tsrow); - tfree(pRes->buffer); - tfree(pRes->length); + taosTFree(pRes->tsrow); + taosTFree(pRes->buffer); + taosTFree(pRes->length); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return pRes->code; @@ -274,23 +274,23 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { void tscDestroyResPointerInfo(SSqlRes* pRes) { if (pRes->buffer != NULL) { // free all buffers containing the multibyte string for (int i = 0; i < pRes->numOfCols; i++) { - tfree(pRes->buffer[i]); + taosTFree(pRes->buffer[i]); } pRes->numOfCols = 0; } - tfree(pRes->pRsp); - tfree(pRes->tsrow); - tfree(pRes->length); + taosTFree(pRes->pRsp); + taosTFree(pRes->tsrow); + taosTFree(pRes->length); - tfree(pRes->pGroupRec); - tfree(pRes->pColumnIndex); - tfree(pRes->buffer); + taosTFree(pRes->pGroupRec); + taosTFree(pRes->pColumnIndex); + taosTFree(pRes->buffer); if (pRes->pArithSup != NULL) { - tfree(pRes->pArithSup->data); - tfree(pRes->pArithSup); + taosTFree(pRes->pArithSup->data); + taosTFree(pRes->pArithSup); } pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free @@ -307,11 +307,11 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd) { freeQueryInfoImpl(pQueryInfo); clearAllTableMetaInfo(pQueryInfo, (const char*)addr, false); - tfree(pQueryInfo); + taosTFree(pQueryInfo); } pCmd->numOfClause = 0; - tfree(pCmd->pQueryInfo); + taosTFree(pCmd->pQueryInfo); } void tscResetSqlCmdObj(SSqlCmd* pCmd) { @@ -357,13 +357,13 @@ void tscPartiallyFreeSqlObj(SSqlObj* pSql) { // pSql->sqlstr will be used by tscBuildQueryStreamDesc if (pObj->signature == pObj) { //pthread_mutex_lock(&pObj->mutex); - tfree(pSql->sqlstr); + taosTFree(pSql->sqlstr); //pthread_mutex_unlock(&pObj->mutex); } tscFreeSqlResult(pSql); - tfree(pSql->pSubs); + taosTFree(pSql->pSubs); pSql->numOfSubs = 0; tscResetSqlCmdObj(pCmd); @@ -383,10 +383,10 @@ void tscFreeSqlObj(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; memset(pCmd->payload, 0, (size_t)pCmd->allocSize); - tfree(pCmd->payload); + taosTFree(pCmd->payload); pCmd->allocSize = 0; - tfree(pSql->sqlstr); + taosTFree(pSql->sqlstr); sem_destroy(&pSql->rspSem); free(pSql); } @@ -396,12 +396,12 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { return; } - tfree(pDataBlock->pData); - tfree(pDataBlock->params); + taosTFree(pDataBlock->pData); + taosTFree(pDataBlock->params); // free the refcount for metermeta taosCacheRelease(tscCacheHandle, (void**)&(pDataBlock->pTableMeta), false); - tfree(pDataBlock); + taosTFree(pDataBlock); } SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, @@ -679,7 +679,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); - tfree(dataBuf->pData); + taosTFree(dataBuf->pData); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -735,7 +735,7 @@ void tscCloseTscObj(STscObj* pObj) { } tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, pObj->pDnodeConn); - tfree(pObj); + taosTFree(pObj); } bool tscIsInsertData(char* sqlstr) { @@ -916,7 +916,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { if (pInfo->pArithExprInfo != NULL) { tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); - tfree(pInfo->pArithExprInfo); + taosTFree(pInfo->pArithExprInfo); } } @@ -1029,7 +1029,7 @@ void* sqlExprDestroy(SSqlExpr* pExpr) { tVariantDestroy(&pExpr->param[i]); } - tfree(pExpr); + taosTFree(pExpr); return NULL; } @@ -1116,11 +1116,11 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) { for(int32_t i = 0; i < numOfFilters; ++i) { if (pFilterInfo[i].filterstr) { - tfree(pFilterInfo[i].pz); + taosTFree(pFilterInfo[i].pz); } } - tfree(pFilterInfo); + taosTFree(pFilterInfo); } SColumn* tscColumnClone(const SColumn* src) { @@ -1351,7 +1351,7 @@ void tscTagCondRelease(STagCond* pTagCond) { size_t s = taosArrayGetSize(pTagCond->pCond); for (int32_t i = 0; i < s; ++i) { SCond* p = taosArrayGet(pTagCond->pCond, i); - tfree(p->cond); + taosTFree(p->cond); } taosArrayDestroy(pTagCond->pCond); @@ -1546,7 +1546,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); - tfree(pQueryInfo->fillVal); + taosTFree(pQueryInfo->fillVal); } void tscClearSubqueryInfo(SSqlCmd* pCmd) { @@ -1566,7 +1566,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem free(pTableMetaInfo); } - tfree(pQueryInfo->pTableMetaInfo); + taosTFree(pQueryInfo->pTableMetaInfo); } STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, @@ -1613,7 +1613,7 @@ void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) } taosCacheRelease(tscCacheHandle, (void**)&(pTableMetaInfo->pTableMeta), removeFromCache); - tfree(pTableMetaInfo->vgroupList); + taosTFree(pTableMetaInfo->vgroupList); tscColumnListDestroy(pTableMetaInfo->tagColList); pTableMetaInfo->tagColList = NULL; @@ -2084,7 +2084,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { pRes->numOfTotal = num; - tfree(pSql->pSubs); + taosTFree(pSql->pSubs); pSql->numOfSubs = 0; pSql->fp = fp; diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp index b37be0485b..bee01b5cec 100644 --- a/src/client/tests/timeParseTest.cpp +++ b/src/client/tests/timeParseTest.cpp @@ -1,10 +1,10 @@ +#include "os.h" #include #include #include #include "taos.h" #include "tstoken.h" -#include "ttime.h" #include "tutil.h" int main(int argc, char** argv) { diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 68875341ba..6c6e5e35d8 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -80,7 +80,7 @@ typedef struct { #define schemaFLen(s) ((s)->flen) #define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) tfree((s)) +#define tdFreeSchema(s) taosTFree((s)) STSchema *tdDupSchema(STSchema *pSchema); int tdEncodeSchema(void **buf, STSchema *pSchema); @@ -284,7 +284,7 @@ typedef struct { #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) tfree(r) +#define kvRowFree(r) taosTFree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) SKVRow tdKVRowDup(SKVRow row); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index adfce5580e..a185a9de7b 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -93,7 +93,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { if (pBuilder) { - tfree(pBuilder->columns); + taosTFree(pBuilder->columns); } } @@ -361,8 +361,8 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { void tdFreeDataCols(SDataCols *pCols) { if (pCols) { - tfree(pCols->buf); - tfree(pCols->cols); + taosTFree(pCols->buf); + taosTFree(pCols->cols); free(pCols); } } @@ -685,8 +685,8 @@ int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) { } void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) { - tfree(pBuilder->pColIdx); - tfree(pBuilder->buf); + taosTFree(pBuilder->pColIdx); + taosTFree(pBuilder->buf); } void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 1a510299c1..0d071454a7 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -111,7 +111,7 @@ int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; int32_t tsMaxVgroupsPerDb = 0; -int32_t tsMinTablePerVnode = 100; +int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 1b64a7aefa..2906b3f0ac 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -128,7 +128,7 @@ void tVariantDestroy(tVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { - tfree(pVar->pz); + taosTFree(pVar->pz); pVar->nLen = 0; } diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 2f0c3c78e7..da0c5b12d4 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -52,12 +52,7 @@ org.apache.commons commons-lang3 ${commons-lang3.version} - - - org.jacoco - jacoco-maven-plugin - 0.8.3 - + junit junit @@ -105,32 +100,7 @@ true - - - org.jacoco - jacoco-maven-plugin - 0.8.3 - - - com/**/* - - - - - pre-test - - prepare-agent - - - - post-test - test - - report - - - - + diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index e0f30166c4..758d620e57 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -114,7 +114,7 @@ void cqClose(void *handle) { SCqObj *pTemp = pObj; pObj = pObj->next; tdFreeSchema(pTemp->pSchema); - tfree(pTemp->sqlStr); + taosTFree(pTemp->sqlStr); free(pTemp); } diff --git a/src/dnode/CMakeLists.txt b/src/dnode/CMakeLists.txt index f7c2961352..64d69e9f01 100644 --- a/src/dnode/CMakeLists.txt +++ b/src/dnode/CMakeLists.txt @@ -16,7 +16,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(src SRC) ADD_EXECUTABLE(taosd ${SRC}) - TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4 balance sync) +# TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4 balance sync) + TARGET_LINK_LIBRARIES(taosd mnode taos monitor http mqtt tsdb twal vnode cJson lz4 balance sync) IF (TD_ACCOUNT) TARGET_LINK_LIBRARIES(taosd account) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 85230c7da9..82ed7dd179 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -75,7 +75,7 @@ void dnodeCleanupMnodePeer() { } taosCloseQset(tsMPeerQset); - tfree(tsMPeerPool.peerWorker); + taosTFree(tsMPeerPool.peerWorker); dInfo("dnode mpeer is closed"); } diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index b53c66e00c..0a305b5598 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -77,7 +77,7 @@ void dnodeCleanupMnodeWrite() { } taosCloseQset(tsMWriteQset); - tfree(tsMWritePool.writeWorker); + taosTFree(tsMWritePool.writeWorker); dInfo("dnode mwrite is closed"); } diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index ec35475d73..745ffb3b84 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -18,13 +18,11 @@ #include "cJSON.h" #include "taoserror.h" #include "taosmsg.h" -#include "ttime.h" #include "ttimer.h" #include "tsdb.h" #include "twal.h" #include "tqueue.h" #include "tsync.h" -#include "ttime.h" #include "ttimer.h" #include "tbalance.h" #include "tglobal.h" diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index c70ca8662e..c328e57453 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -22,6 +22,7 @@ extern "C" { #include #include +#include "osDef.h" #include "taos.h" #define TSDB__packed @@ -161,7 +162,7 @@ extern tDataTypeDescriptor tDataTypeDesc[11]; bool isValidDataType(int32_t type); //bool isNull(const char *val, int32_t type); -static inline __attribute__((always_inline)) bool isNull(const char *val, int32_t type) { +static FORCE_INLINE bool isNull(const char *val, int32_t type) { switch (type) { case TSDB_DATA_TYPE_BOOL: return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; diff --git a/src/kit/shell/CMakeLists.txt b/src/kit/shell/CMakeLists.txt index 01c5f5ea3a..3695f694f3 100644 --- a/src/kit/shell/CMakeLists.txt +++ b/src/kit/shell/CMakeLists.txt @@ -15,11 +15,11 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) LIST(REMOVE_ITEM SRC ./src/shellDarwin.c) ADD_EXECUTABLE(shell ${SRC}) - IF (TD_PAGMODE_LITE) +# IF (TD_PAGMODE_LITE) TARGET_LINK_LIBRARIES(shell taos) - ELSE () - TARGET_LINK_LIBRARIES(shell taos_static) - ENDIF () +# ELSE () +# TARGET_LINK_LIBRARIES(shell taos_static) +# ENDIF () SET_TARGET_PROPERTIES(shell PROPERTIES OUTPUT_NAME taos) ELSEIF (TD_WINDOWS_64) diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index 1a75a2aa85..3cb324abe9 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -229,8 +229,8 @@ void shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); + taosTFree(cmd.buffer); + taosTFree(cmd.command); return; } else { updateBuffer(&cmd); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 616a3bfd7f..5d877ba6f0 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -21,7 +21,6 @@ #include "os.h" #include "shell.h" #include "shellCommand.h" -#include "ttime.h" #include "tutil.h" #include "taosdef.h" #include "taoserror.h" @@ -176,7 +175,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { if (history.hist[history.hend] != NULL) { - tfree(history.hist[history.hend]); + taosTFree(history.hist[history.hend]); } history.hist[history.hend] = strdup(command); @@ -770,7 +769,7 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { fprintf(f, "%s\n", history.hist[i]); - tfree(history.hist[i]); + taosTFree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } diff --git a/src/kit/shell/src/shellImport.c b/src/kit/shell/src/shellImport.c index afd2d85dae..ee0a90757b 100644 --- a/src/kit/shell/src/shellImport.c +++ b/src/kit/shell/src/shellImport.c @@ -21,7 +21,6 @@ #include "shell.h" #include "shellCommand.h" #include "tglobal.h" -#include "ttime.h" #include "tutil.h" static char **shellSQLFiles = NULL; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 94f3901dd8..963afe346d 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -201,8 +201,8 @@ void shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - tfree(cmd.buffer); - tfree(cmd.command); + taosTFree(cmd.buffer); + taosTFree(cmd.command); return; } else { updateBuffer(&cmd); @@ -320,7 +320,7 @@ void *shellLoopQuery(void *arg) { reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); - tfree(command); + taosTFree(command); exitShell(); pthread_cleanup_pop(1); diff --git a/src/kit/taosdemo/CMakeLists.txt b/src/kit/taosdemo/CMakeLists.txt index a1593ce5a1..5045e8cd3f 100644 --- a/src/kit/taosdemo/CMakeLists.txt +++ b/src/kit/taosdemo/CMakeLists.txt @@ -11,10 +11,10 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdemo ${SRC}) - IF (TD_PAGMODE_LITE) +# IF (TD_PAGMODE_LITE) TARGET_LINK_LIBRARIES(taosdemo taos) - ELSE () - TARGET_LINK_LIBRARIES(taosdemo taos_static) - ENDIF () +# ELSE () +# TARGET_LINK_LIBRARIES(taosdemo taos_static) +# ENDIF () ENDIF () diff --git a/src/kit/taosdump/CMakeLists.txt b/src/kit/taosdump/CMakeLists.txt index 2a14dbfb4a..1c5dd3eda2 100644 --- a/src/kit/taosdump/CMakeLists.txt +++ b/src/kit/taosdump/CMakeLists.txt @@ -13,11 +13,11 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) AUX_SOURCE_DIRECTORY(. SRC) ADD_EXECUTABLE(taosdump ${SRC}) - IF (TD_PAGMODE_LITE) +# IF (TD_PAGMODE_LITE) TARGET_LINK_LIBRARIES(taosdump taos) - ELSE () - TARGET_LINK_LIBRARIES(taosdump taos_static) - ENDIF () +# ELSE () +# TARGET_LINK_LIBRARIES(taosdump taos_static) +# ENDIF () ENDIF () diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 7ae209bbdf..ee98c711e0 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -448,8 +448,8 @@ int main(int argc, char *argv[]) { void taosFreeDbInfos() { if (dbInfos == NULL) return; - for (int i = 0; i < 128; i++) tfree(dbInfos[i]); - tfree(dbInfos); + for (int i = 0; i < 128; i++) taosTFree(dbInfos[i]); + taosTFree(dbInfos); } // check table is normal table or super table @@ -606,11 +606,11 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu if (numOfTable >= arguments->table_batch) { numOfTable = 0; - tclose(fd); + taosClose(fd); fd = -1; } } - tclose(fd); + taosClose(fd); fd = -1; taos_free_result(result); @@ -780,14 +780,14 @@ int taosDumpOut(struct arguments *arguments) { if (retCode < 0) { if (-1 != normalTblFd){ - tclose(normalTblFd); + taosClose(normalTblFd); } goto _clean_tmp_file; } } if (-1 != normalTblFd){ - tclose(normalTblFd); + taosClose(normalTblFd); } // start multi threads to dumpout @@ -806,7 +806,7 @@ int taosDumpOut(struct arguments *arguments) { fclose(fp); taos_close(taos); taos_free_result(result); - tfree(command); + taosTFree(command); taosFreeDbInfos(); fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); return 0; @@ -815,7 +815,7 @@ _exit_failure: fclose(fp); taos_close(taos); taos_free_result(result); - tfree(command); + taosTFree(command); taosFreeDbInfos(); fprintf(stderr, "dump out rows: %" PRId64 "\n", totalDumpOutRows); return -1; @@ -1076,7 +1076,7 @@ void* taosDumpOutWorkThreadFp(void *arg) } taos_free_result(tmpResult); - tclose(fd); + taosClose(fd); fclose(fp); return NULL; @@ -1206,7 +1206,7 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) (void)taosDumpStable(tableRecord.name, fp, taosCon); } - tclose(fd); + taosClose(fd); remove(".stables.tmp"); free(tmpCommand); @@ -1288,11 +1288,11 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao if (numOfTable >= arguments->table_batch) { numOfTable = 0; - tclose(fd); + taosClose(fd); fd = -1; } } - tclose(fd); + taosClose(fd); fd = -1; taos_free_result(tmpResult); @@ -1760,13 +1760,13 @@ void taosLoadFileCharset(FILE *fp, char *fcharset) { } strcpy(fcharset, line + 2); - tfree(line); + taosTFree(line); return; _exit_no_charset: fseek(fp, 0, SEEK_SET); *fcharset = '\0'; - tfree(line); + taosTFree(line); return; } @@ -1860,9 +1860,9 @@ static void taosMallocSQLFiles() static void taosFreeSQLFiles() { for (int i = 0; i < tsSqlFileNum; i++) { - tfree(tsDumpInSqlFiles[i]); + taosTFree(tsDumpInSqlFiles[i]); } - tfree(tsDumpInSqlFiles); + taosTFree(tsDumpInSqlFiles); } static void taosGetDirectoryFileList(char *inputDir) @@ -2063,17 +2063,17 @@ int taosDumpInOneFile_old(TAOS * taos, FILE* fp, char* fcharset, char* encod } if (cd != ((iconv_t)(-1))) iconv_close(cd); - tfree(line); - tfree(command); - tfree(lcommand); + taosTFree(line); + taosTFree(command); + taosTFree(lcommand); taos_close(taos); fclose(fp); return 0; _dumpin_exit_failure: if (cd != ((iconv_t)(-1))) iconv_close(cd); - tfree(command); - tfree(lcommand); + taosTFree(command); + taosTFree(lcommand); taos_close(taos); fclose(fp); return -1; @@ -2120,8 +2120,8 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c cmd_len = 0; } - tfree(cmd); - tfree(line); + taosTFree(cmd); + taosTFree(line); fclose(fp); return 0; } diff --git a/src/kit/taosmigrate/taosmigrateVnodeCfg.c b/src/kit/taosmigrate/taosmigrateVnodeCfg.c index e80e687f02..472671c22c 100644 --- a/src/kit/taosmigrate/taosmigrateVnodeCfg.c +++ b/src/kit/taosmigrate/taosmigrateVnodeCfg.c @@ -289,7 +289,7 @@ static int32_t readVnodeCfg(SVnodeObj *pVnode, char* cfgFile) //} PARSE_OVER: - tfree(content); + taosTFree(content); cJSON_Delete(root); if (fp) fclose(fp); return ret; diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index c40a696ede..5244bc3e85 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "ttime.h" #include "dnode.h" #include "mnodeDef.h" #include "mnodeInt.h" @@ -34,7 +33,7 @@ static int32_t mnodeCreateRootAcct(); static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) { SAcctObj *pAcct = pOper->pObj; pthread_mutex_destroy(&pAcct->mutex); - tfree(pOper->pObj); + taosTFree(pOper->pObj); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 41727712b5..5231b7bcd3 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" -#include "ttime.h" #include "dnode.h" #include "mnodeDef.h" #include "mnodeInt.h" @@ -34,7 +33,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void * static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) { - tfree(pOper->pObj); + taosTFree(pOper->pObj); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 82ac0868d1..c7e3085e69 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -19,7 +19,6 @@ #include "tutil.h" #include "tgrant.h" #include "tglobal.h" -#include "ttime.h" #include "tname.h" #include "tbalance.h" #include "tdataformat.h" @@ -53,8 +52,8 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg); static void mnodeDestroyDb(SDbObj *pDb) { pthread_mutex_destroy(&pDb->mutex); - tfree(pDb->vgList); - tfree(pDb); + taosTFree(pDb->vgList); + taosTFree(pDb); } static int32_t mnodeDbActionDestroy(SSdbOper *pOper) { @@ -386,7 +385,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, void *pMs code = mnodeCheckDbCfg(&pDb->cfg); if (code != TSDB_CODE_SUCCESS) { - tfree(pDb); + taosTFree(pDb); return code; } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index be15abe354..06c343a903 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -19,7 +19,6 @@ #include "tbalance.h" #include "tglobal.h" #include "tconfig.h" -#include "ttime.h" #include "tutil.h" #include "tsocket.h" #include "tbalance.h" @@ -62,7 +61,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole); static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) { - tfree(pOper->pObj); + taosTFree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -504,7 +503,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { int dnodeId = pDnode->dnodeId; - tfree(pDnode); + taosTFree(pDnode); mError("failed to create dnode:%d, reason:%s", dnodeId, tstrerror(code)); } else { mLInfo("dnode:%d is created", pDnode->dnodeId); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 5f82a9afad..c762403525 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -21,7 +21,6 @@ #include "tsync.h" #include "tbalance.h" #include "tutil.h" -#include "ttime.h" #include "tsocket.h" #include "tdataformat.h" #include "mnodeDef.h" @@ -57,7 +56,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo #endif static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) { - tfree(pOper->pObj); + taosTFree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -279,7 +278,7 @@ int32_t mnodeAddMnode(int32_t dnodeId) { int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { - tfree(pMnode); + taosTFree(pMnode); } mnodeUpdateMnodeEpSet(); diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index ff4ddf1b2a..353dd59671 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -18,7 +18,6 @@ #include "taosmsg.h" #include "taoserror.h" #include "tutil.h" -#include "ttime.h" #include "tcache.h" #include "tglobal.h" #include "tdataformat.h" @@ -133,8 +132,8 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po static void mnodeFreeConn(void *data) { SConnObj *pConn = data; - tfree(pConn->pQueries); - tfree(pConn->pStreams); + taosTFree(pConn->pQueries); + taosTFree(pConn->pStreams); mDebug("connId:%d, is destroyed", pConn->connId); } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index f3f6e33431..a4c2c60aa3 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -934,7 +934,7 @@ void sdbCleanupWriteWorker() { } sdbFreeWritequeue(); - tfree(tsSdbPool.writeWorker); + taosTFree(tsSdbPool.writeWorker); mInfo("sdb write is closed"); } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 733bd43c74..995bfbe840 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -403,7 +403,7 @@ static void mnodeFreeShowObj(void *data) { sdbFreeIter(pShow->pIter); mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index); - tfree(pShow); + taosTFree(pShow); } static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 009633b43b..1c39ef3294 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" -#include "ttime.h" #include "tutil.h" #include "taoserror.h" #include "taosmsg.h" @@ -90,10 +89,10 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); static void mnodeDestroyChildTable(SChildTableObj *pTable) { - tfree(pTable->info.tableId); - tfree(pTable->schema); - tfree(pTable->sql); - tfree(pTable); + taosTFree(pTable->info.tableId); + taosTFree(pTable->schema); + taosTFree(pTable->sql); + taosTFree(pTable); } static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) { @@ -411,9 +410,9 @@ static void mnodeDestroySuperTable(SSuperTableObj *pStable) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; } - tfree(pStable->info.tableId); - tfree(pStable->schema); - tfree(pStable); + taosTFree(pStable->info.tableId); + taosTFree(pStable->schema); + taosTFree(pStable); } static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) { diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index a875cff4a2..765661ac71 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "trpc.h" -#include "ttime.h" #include "tutil.h" #include "tglobal.h" #include "tgrant.h" @@ -43,7 +42,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg); static int32_t mnodeUserActionDestroy(SSdbOper *pOper) { - tfree(pOper->pObj); + taosTFree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -239,7 +238,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); - tfree(pUser); + taosTFree(pUser); } else { mLInfo("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 91aa3fdec2..e46da1d892 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -20,7 +20,6 @@ #include "tsocket.h" #include "tidpool.h" #include "tsync.h" -#include "ttime.h" #include "tbalance.h" #include "tglobal.h" #include "tdataformat.h" @@ -70,7 +69,7 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) { pVgroup->idPool = NULL; } - tfree(pVgroup); + taosTFree(pVgroup); } static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) { diff --git a/src/os/CMakeLists.txt b/src/os/CMakeLists.txt index a8664669d0..b4ad4ad915 100644 --- a/src/os/CMakeLists.txt +++ b/src/os/CMakeLists.txt @@ -6,11 +6,11 @@ IF (TD_LINUX_64) ELSEIF (TD_LINUX_32) ADD_SUBDIRECTORY(src/linux32) ELSEIF (TD_DARWIN_64) - ADD_SUBDIRECTORY(src/darwin64) + ADD_SUBDIRECTORY(src/darwin) ELSEIF (TD_WINDOWS_64) - ADD_SUBDIRECTORY(src/windows64) + ADD_SUBDIRECTORY(src/windows) ELSEIF (TD_WINDOWS_32) - ADD_SUBDIRECTORY(src/windows32) + ADD_SUBDIRECTORY(src/windows) ENDIF () ADD_SUBDIRECTORY(src/detail) diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 896f3afc7a..700b29ce98 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -21,7 +21,7 @@ extern "C" { #endif #ifdef _TD_DARWIN_64 -#include "osDarwin64.h" +#include "osDarwin.h" #endif #ifdef _TD_LINUX_64 @@ -36,15 +36,26 @@ extern "C" { #include "osAlpine.h" #endif -#ifdef _TD_WINDOWS_64 -#include "osWindows64.h" +#if defined(_TD_WINDOWS_64) || defined(_TD_WINDOWS_32) +#include "osWindows.h" #endif -#ifdef _TD_WINDOWS_32 -#include "osWindows32.h" -#endif +#include "osAtomic.h" +#include "osDef.h" +#include "osDir.h" +#include "osFile.h" +#include "osLz4.h" +#include "osMath.h" +#include "osMemory.h" +#include "osRand.h" +#include "osSemphone.h" +#include "osSocket.h" +#include "osString.h" +#include "osSysinfo.h" +#include "osTime.h" +#include "osTimer.h" -#include "osSpec.h" +void osInit(); #ifdef __cplusplus } diff --git a/src/os/inc/osAtomic.h b/src/os/inc/osAtomic.h new file mode 100644 index 0000000000..803c351400 --- /dev/null +++ b/src/os/inc/osAtomic.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_ATOMIC_H +#define TDENGINE_OS_ATOMIC_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TAOS_OS_FUNC_ATOMIC + #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) + + #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap + #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap + + #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) + + #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) + #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h new file mode 100644 index 0000000000..8628a0f3ac --- /dev/null +++ b/src/os/inc/osDarwin.h @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_PLATFORM_LINUX_H +#define TDENGINE_PLATFORM_LINUX_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define TAOS_OS_FUNC_FILE_SENDIFLE + #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size) + #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) + +#define TAOS_OS_FUNC_SEMPHONE + #define tsem_t dispatch_semaphore_t + int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value); + int tsem_wait(dispatch_semaphore_t *sem); + int tsem_post(dispatch_semaphore_t *sem); + int tsem_destroy(dispatch_semaphore_t *sem); + +#define TAOS_OS_FUNC_SOCKET_SETSOCKETOPT +#define TAOS_OS_FUNC_STRING_STR2INT64 +#define TAOS_OS_FUNC_SYSINFO +#define TAOS_OS_FUNC_TIMER + +// specific +#define htobe64 htonll +typedef int(*__compar_fn_t)(const void *, const void *); + +// for send function in tsocket.c +#define MSG_NOSIGNAL 0 +#define SO_NO_CHECK 0x1234 +#define SOL_TCP 0x1234 +#define TCP_KEEPIDLE 0x1234 + +#ifndef PTHREAD_MUTEX_RECURSIVE_NP + #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osDarwin64.h b/src/os/inc/osDarwin64.h deleted file mode 100644 index 3ceb7ea8f5..0000000000 --- a/src/os/inc/osDarwin64.h +++ /dev/null @@ -1,274 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_PLATFORM_LINUX_H -#define TDENGINE_PLATFORM_LINUX_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define htobe64 htonll - -#define taosCloseSocket(x) \ - { \ - if (FD_VALID(x)) { \ - close(x); \ - x = FD_INITIALIZER; \ - } \ - } - -#define taosWriteSocket(fd, buf, len) write(fd, buf, len) -#define taosReadSocket(fd, buf, len) read(fd, buf, len) - -#define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) -#define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) -#define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) -#define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) -#define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - -#define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap -#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap -#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap -#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap -#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap - -#define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - -#define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) -#define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - -#define SWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ - } while (0) - -#define MAX(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a > __b) ? __a : __b; \ - }) - -#define MIN(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a < __b) ? __a : __b; \ - }) - -#define MILLISECOND_PER_SECOND ((int64_t)1000L) - -#define tsem_t dispatch_semaphore_t - -int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value); -int tsem_wait(dispatch_semaphore_t *sem); -int tsem_post(dispatch_semaphore_t *sem); -int tsem_destroy(dispatch_semaphore_t *sem); - -void osInit(); - -ssize_t tread(int fd, void *buf, size_t count); - -ssize_t twrite(int fd, void *buf, size_t n); - - -bool taosCheckPthreadValid(pthread_t thread); - -void taosResetPthread(pthread_t *thread); - -int64_t taosGetPthreadId(); - -int taosSetNonblocking(int sock, int on); - -int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen); - -void taosPrintOsInfo(); - -void taosPrintOsInfo(); - -void taosGetSystemInfo(); - -void taosKillSystem(); - -bool taosSkipSocketCheck(); - -bool taosGetDisk(); - -int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count); - -void taosSetCoreDump(); - -int tSystem(const char * cmd); - -typedef int(*__compar_fn_t)(const void *, const void *); - -// for send function in tsocket.c -#define MSG_NOSIGNAL 0 -#define SO_NO_CHECK 0x1234 -#define SOL_TCP 0x1234 -#define TCP_KEEPIDLE 0x1234 - -#ifndef PTHREAD_MUTEX_RECURSIVE_NP - #define PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE -#endif - -#ifndef _TD_ARM_32_ -#define BUILDIN_CLZL(val) __builtin_clzl(val) -#define BUILDIN_CTZL(val) __builtin_ctzl(val) -#else -#define BUILDIN_CLZL(val) __builtin_clzll(val) -#define BUILDIN_CTZL(val) __builtin_ctzll(val) -#endif -#define BUILDIN_CLZ(val) __builtin_clz(val) -#define BUILDIN_CTZ(val) __builtin_ctz(val) - -#undef threadlocal -#ifdef _ISOC11_SOURCE - #define threadlocal _Thread_local -#elif defined(__APPLE__) - #define threadlocal -#elif defined(__GNUC__) && !defined(threadlocal) - #define threadlocal __thread -#else - #define threadlocal -#endif - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osDef.h b/src/os/inc/osDef.h new file mode 100644 index 0000000000..81c70a58fd --- /dev/null +++ b/src/os/inc/osDef.h @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_DEF_H +#define TDENGINE_OS_DEF_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef STDERR_FILENO +#define STDERR_FILENO (2) +#endif + +#define FD_VALID(x) ((x) > STDERR_FILENO) +#define FD_INITIALIZER ((int32_t)-1) + +#define WCHAR wchar_t + +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) +#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) + +#ifndef NDEBUG +#define ASSERT(x) assert(x) +#else +#define ASSERT(x) +#endif + +#ifdef UNUSED +#undefine UNUSED +#endif +#define UNUSED(x) ((void)(x)) + +#ifdef UNUSED_FUNC +#undefine UNUSED_FUNC +#endif + +#ifdef UNUSED_PARAM +#undef UNUSED_PARAM +#endif + +#if defined(__GNUC__) +#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) +#define UNUSED_FUNC __attribute__((unused)) +#else +#define UNUSED_PARAM(x) x +#define UNUSED_FUNC +#endif + +#ifdef tListLen +#undefine tListLen +#endif +#define tListLen(x) (sizeof(x) / sizeof((x)[0])) + +#if defined(__GNUC__) +#define FORCE_INLINE inline __attribute__((always_inline)) +#else +#define FORCE_INLINE +#endif + +#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" + +#define DEFAULT_COMP(x, y) \ + do { \ + if ((x) == (y)) { \ + return 0; \ + } else { \ + return (x) < (y) ? -1 : 1; \ + } \ + } while (0) + +#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) + +// align to 8bytes +#define ALIGN8(n) ALIGN_NUM(n, 8) + +#undef threadlocal +#ifdef _ISOC11_SOURCE + #define threadlocal _Thread_local +#elif defined(__APPLE__) + #define threadlocal +#elif defined(__GNUC__) && !defined(threadlocal) + #define threadlocal __thread +#else + #define threadlocal +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osDir.h b/src/os/inc/osDir.h new file mode 100644 index 0000000000..73e4b216e6 --- /dev/null +++ b/src/os/inc/osDir.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_DIR_H +#define TDENGINE_OS_DIR_H + +#ifdef __cplusplus +extern "C" { +#endif + +// TAOS_OS_FUNC_DIR +void taosRemoveDir(char *rootDir); +int taosMkDir(const char *pathname, mode_t mode); +void taosMvDir(char* destDir, char *srcDir); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h new file mode 100644 index 0000000000..6e48e80ca4 --- /dev/null +++ b/src/os/inc/osFile.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_FILE_H +#define TDENGINE_OS_FILE_H + +#ifdef __cplusplus +extern "C" { +#endif + +ssize_t taosTReadImp(int fd, void *buf, size_t count); +ssize_t taosTWriteImp(int fd, void *buf, size_t count); + +ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); +int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count); + +#ifndef TAOS_OS_FUNC_FILE_SENDIFLE + #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) + #define taosFSendFile(outfile, infile, offset, count) taosTSendFileImp(fileno(outfile), fileno(infile), offset, size) +#endif + +#define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count) +#define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count) +#define taosLSeek(fd, offset, whence) lseek(fd, offset, whence) + +#ifdef TAOS_RANDOM_FILE_FAIL + void taosSetRandomFileFailFactor(int factor); + void taosSetRandomFileFailOutput(const char *path); + #ifdef TAOS_RANDOM_FILE_FAIL_TEST + ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); + ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); + off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); + #undef taosTRead + #undef taosTWrite + #undef taosLSeek + #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) + #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) + #endif +#endif + +int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); + +// TAOS_OS_FUNC_FILE_GETTMPFILEPATH +void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h index 1778b0e315..b62b086d0f 100644 --- a/src/os/inc/osLinux32.h +++ b/src/os/inc/osLinux32.h @@ -22,7 +22,6 @@ extern "C" { #include #include - #include #include #include @@ -78,6 +77,12 @@ extern "C" { #include #include +#define TAOS_OS_FUNC_LZ4 +#define BUILDIN_CLZL(val) __builtin_clzll(val) +#define BUILDIN_CTZL(val) __builtin_ctzll(val) +#define BUILDIN_CLZ(val) __builtin_clz(val) +#define BUILDIN_CTZ(val) __builtin_ctz(val) + #ifdef __cplusplus } #endif diff --git a/src/os/inc/osLz4.h b/src/os/inc/osLz4.h new file mode 100644 index 0000000000..a944892c48 --- /dev/null +++ b/src/os/inc/osLz4.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_LZ4_H +#define TDENGINE_OS_LZ4_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TAOS_OS_FUNC_LZ4 + #define BUILDIN_CLZL(val) __builtin_clzl(val) + #define BUILDIN_CTZL(val) __builtin_ctzl(val) + #define BUILDIN_CLZ(val) __builtin_clz(val) + #define BUILDIN_CTZ(val) __builtin_ctz(val) +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osMath.h b/src/os/inc/osMath.h new file mode 100644 index 0000000000..168f6607f0 --- /dev/null +++ b/src/os/inc/osMath.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_MATH_H +#define TDENGINE_OS_MATH_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define POW2(x) ((x) * (x)) + +#ifndef TAOS_OS_FUNC_MATH + #define SWAP(a, b, c) \ + do { \ + typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ + } while (0) + + #define MAX(a, b) \ + ({ \ + typeof(a) __a = (a); \ + typeof(b) __b = (b); \ + (__a > __b) ? __a : __b; \ + }) + + #define MIN(a, b) \ + ({ \ + typeof(a) __a = (a); \ + typeof(b) __b = (b); \ + (__a < __b) ? __a : __b; \ + }) +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h new file mode 100644 index 0000000000..aaac8c68e3 --- /dev/null +++ b/src/os/inc/osMemory.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_MEMORY_H +#define TDENGINE_OS_MEMORY_H + +#include "osString.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + TAOS_ALLOC_MODE_DEFAULT = 0, + TAOS_ALLOC_MODE_RANDOM_FAIL = 1, + TAOS_ALLOC_MODE_DETECT_LEAK = 2 +} ETaosMemoryAllocMode; + +void taosSetAllocMode(int mode, const char *path, bool autoDump); +void taosDumpMemoryLeak(); + +void * taosTMalloc(size_t size); +void * taosTCalloc(size_t nmemb, size_t size); +void * taosTRealloc(void *ptr, size_t size); +void taosTZfree(void *ptr); +size_t taosTSizeof(void *ptr); +void taosTMemset(void *ptr, int c); + +#define taosTFree(x) \ + do { \ + if (x) { \ + free((void *)(x)); \ + x = 0; \ + } \ + } while (0); + +#define taosMalloc(size) malloc(size) +#define taosCalloc(num, size) calloc(num, size) +#define taosRealloc(ptr, size) realloc(ptr, size) +#define taosFree(ptr) free(ptr) +#define taosStrdup(str) taosStrdupImp(str) +#define taosStrndup(str, size) taosStrndupImp(str, size) +#define taosGetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream) + +#ifdef TAOS_MEM_CHECK + #ifdef TAOS_MEM_CHECK_TEST + void * taos_malloc(size_t size, const char *file, uint32_t line); + void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); + void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); + void taos_free(void *ptr, const char *file, uint32_t line); + char * taos_strdup(const char *str, const char *file, uint32_t line); + char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); + ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); + #undef taosMalloc + #undef taosCalloc + #undef taosRealloc + #undef taosFree + #undef taosStrdup + #undef taosStrndup + #undef taosGetline + #define taosMalloc(size) taos_malloc(size, __FILE__, __LINE__) + #define taosCalloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) + #define taosRealloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) + #define taosFree(ptr) taos_free(ptr, __FILE__, __LINE__) + #define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__) + #define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) + #define taosGetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) + #endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osRand.h b/src/os/inc/osRand.h new file mode 100644 index 0000000000..803c0688bb --- /dev/null +++ b/src/os/inc/osRand.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_RAND_H +#define TDENGINE_OS_RAND_H + +#ifdef __cplusplus +extern "C" { +#endif + +// TAOS_OS_FUNC_RAND +uint32_t taosRand(void); +void taosRandStr(char* str, int32_t size); +uint32_t taosSafeRand(void); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h new file mode 100644 index 0000000000..300f1e8ef1 --- /dev/null +++ b/src/os/inc/osSemphone.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_SEMPHONE_H +#define TDENGINE_OS_SEMPHONE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TAOS_OS_FUNC_SEMPHONE + #define tsem_t sem_t + #define tsem_init sem_init + #define tsem_wait sem_wait + #define tsem_post sem_post + #define tsem_destroy sem_destroy +#endif + +// TAOS_OS_FUNC_SEMPHONE_PTHREAD +bool taosCheckPthreadValid(pthread_t thread); +int64_t taosGetPthreadId(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osSocket.h b/src/os/inc/osSocket.h new file mode 100644 index 0000000000..10fed022bb --- /dev/null +++ b/src/os/inc/osSocket.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_SOCKET_H +#define TDENGINE_OS_SOCKET_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TAOS_OS_FUNC_SOCKET_OP + #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) + #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen) + #define taosReadSocket(fd, buf, len) read(fd, buf, len) + #define taosWriteSocket(fd, buf, len) write(fd, buf, len) + #define taosCloseSocket(x) \ + { \ + if (FD_VALID(x)) { \ + close(x); \ + x = FD_INITIALIZER; \ + } \ + } +#endif + +#define taosClose(x) taosCloseSocket(x) + +#ifdef TAOS_RANDOM_NETWORK_FAIL + #ifdef TAOS_RANDOM_NETWORK_FAIL_TEST + ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); + ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); + ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); + ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); + #undef taosSend + #undef taosSendto + #undef taosReadSocket + #undef taosWriteSocket + #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags) + #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen) + #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len) + #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len) + #endif +#endif + +// TAOS_OS_FUNC_SOCKET +int taosSetNonblocking(int sock, int on); +void taosBlockSIGPIPE(); + +// TAOS_OS_FUNC_SOCKET_SETSOCKETOPT +int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osSpec.h b/src/os/inc/osSpec.h deleted file mode 100644 index d93a647206..0000000000 --- a/src/os/inc/osSpec.h +++ /dev/null @@ -1,391 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_SPEC_H -#define TDENGINE_OS_SPEC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define tclose(x) taosCloseSocket(x) -#define tfree(x) \ - do { \ - if (x) { \ - free((void *)(x)); \ - x = 0; \ - } \ - } while (0); - -#define tstrncpy(dst, src, size) \ - do { \ - strncpy((dst), (src), (size)); \ - (dst)[(size)-1] = 0; \ - } while (0); - -#ifndef STDERR_FILENO -#define STDERR_FILENO (2) -#endif - -#define FD_VALID(x) ((x) > STDERR_FILENO) -#define FD_INITIALIZER ((int32_t)-1) - -#define WCHAR wchar_t - -#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) -#define POINTER_DISTANCE(p1, p2) ((char *)(p1) - (char *)(p2)) - -#ifndef NDEBUG -#define ASSERT(x) assert(x) -#else -#define ASSERT(x) -#endif - -#ifdef UNUSED -#undefine UNUSED -#endif -#define UNUSED(x) ((void)(x)) - -#ifdef UNUSED_FUNC -#undefine UNUSED_FUNC -#endif - -#ifdef UNUSED_PARAM -#undef UNUSED_PARAM -#endif - -#if defined(__GNUC__) -#define UNUSED_PARAM(x) _UNUSED##x __attribute__((unused)) -#define UNUSED_FUNC __attribute__((unused)) -#else -#define UNUSED_PARAM(x) x -#define UNUSED_FUNC -#endif - -#ifdef tListLen -#undefine tListLen -#endif -#define tListLen(x) (sizeof(x) / sizeof((x)[0])) - -#if defined(__GNUC__) -#define FORCE_INLINE inline __attribute__((always_inline)) -#else -#define FORCE_INLINE -#endif - -#define DEFAULT_UNICODE_ENCODEC "UCS-4LE" - -#define DEFAULT_COMP(x, y) \ - do { \ - if ((x) == (y)) { \ - return 0; \ - } else { \ - return (x) < (y) ? -1 : 1; \ - } \ - } while (0) - -#define ALIGN_NUM(n, align) (((n) + ((align)-1)) & (~((align)-1))) - -// align to 8bytes -#define ALIGN8(n) ALIGN_NUM(n, 8) - -#define POW2(x) ((x) * (x)) - -#ifndef TAOS_OS_FUNC_MATH - #define SWAP(a, b, c) \ - do { \ - typeof(a) __tmp = (a); \ - (a) = (b); \ - (b) = __tmp; \ - } while (0) - - #define MAX(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a > __b) ? __a : __b; \ - }) - - #define MIN(a, b) \ - ({ \ - typeof(a) __a = (a); \ - typeof(b) __b = (b); \ - (__a < __b) ? __a : __b; \ - }) -#endif - -#ifndef TAOS_OS_DEF_TIME - #define MILLISECOND_PER_SECOND ((int64_t)1000L) -#endif -#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) -#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) -#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) -#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) -#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30) -#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365) - -#ifndef TAOS_OS_FUNC_SEMPHONE - #define tsem_t sem_t - #define tsem_init sem_init - #define tsem_wait sem_wait - #define tsem_post sem_post - #define tsem_destroy sem_destroy -#endif - -#ifndef TAOS_OS_FUNC_ATOMIC - #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST) - - #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap - #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap - - #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST) - - #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) - #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST) -#endif - -ssize_t taosTReadImp(int fd, void *buf, size_t count); -ssize_t taosTWriteImp(int fd, void *buf, size_t count); -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size); -#ifndef TAOS_OS_FUNC_FILE_OP - #define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count) - #define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count) - #define taosLSeek(fd, offset, whence) lseek(fd, offset, whence) - #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) -#endif - -#ifndef TAOS_OS_FUNC_NETWORK - #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) - #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen) - #define taosReadSocket(fd, buf, len) read(fd, buf, len) - #define taosWriteSocket(fd, buf, len) write(fd, buf, len) - #define taosCloseSocket(x) \ - { \ - if (FD_VALID(x)) { \ - close(x); \ - x = FD_INITIALIZER; \ - } \ - } -#endif - -#ifndef TAOS_OS_FUNC_LZ4 - #define BUILDIN_CLZL(val) __builtin_clzl(val) - #define BUILDIN_CTZL(val) __builtin_ctzl(val) - #define BUILDIN_CLZ(val) __builtin_clz(val) - #define BUILDIN_CTZ(val) __builtin_ctz(val) -#endif - -#ifndef TAOS_OS_FUNC_WCHAR - #define twcslen wcslen -#endif - -#undef threadlocal -#ifdef _ISOC11_SOURCE - #define threadlocal _Thread_local -#elif defined(__APPLE__) - #define threadlocal -#elif defined(__GNUC__) && !defined(threadlocal) - #define threadlocal __thread -#else - #define threadlocal -#endif - -void osInit(); - -// TAOS_OS_FUNC_PTHREAD -bool taosCheckPthreadValid(pthread_t thread); -int64_t taosGetPthreadId(); - -// TAOS_OS_FUNC_SOCKET -int taosSetNonblocking(int sock, int on); -int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen); -void taosBlockSIGPIPE(); - -// TAOS_OS_FUNC_SYSINFO -void taosGetSystemInfo(); -void taosPrintOsInfo(); -void taosKillSystem(); -int tSystem(const char * cmd) ; - -// TAOS_OS_FUNC_CORE -void taosSetCoreDump(); - -// TAOS_OS_FUNC_UTIL -int64_t tsosStr2int64(char *str); - -// TAOS_OS_FUNC_TIMER -void taosMsleep(int mseconds); -int taosInitTimer(void (*callback)(int), int ms); -void taosUninitTimer(); - -// TAOS_OS_FUNC_RAND -uint32_t taosRand(void); -void taosRandStr(char* str, int32_t size); -uint32_t trand(void); - -// TAOS_OS_FUNC_FILE -void getTmpfilePath(const char *fileNamePrefix, char *dstPath); -int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath); - -// USE_LIBICONV -int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); -bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len); -int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes); -bool taosValidateEncodec(const char *encodec); -char * taosCharsetReplace(char *charsetstr); - -// TAOS_OS_FUNC_MALLOC -#define TAOS_ALLOC_MODE_DEFAULT 0 -#define TAOS_ALLOC_MODE_RANDOM_FAIL 1 -#define TAOS_ALLOC_MODE_DETECT_LEAK 2 -void taosSetAllocMode(int mode, const char *path, bool autoDump); -void taosDumpMemoryLeak(); -void * tmalloc(size_t size); -void * tcalloc(size_t nmemb, size_t size); -size_t tsizeof(void *ptr); -void tmemset(void *ptr, int c); -void * trealloc(void *ptr, size_t size); -void tzfree(void *ptr); - -// TAOS_OS_FUNC_DIR -void taosRemoveDir(char *rootDir); -int taosMkDir(const char *pathname, mode_t mode); -void taosMvDir(char* destDir, char *srcDir); - -#ifdef TAOS_RANDOM_FILE_FAIL - void taosSetRandomFileFailFactor(int factor); - void taosSetRandomFileFailOutput(const char *path); - ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line); - off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line); - #undef taosTRead - #undef taosTWrite - #undef taosLSeek - #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__) - #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__) -#endif - -#ifdef TAOS_RANDOM_NETWORK_FAIL - ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags); - ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen); - ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count); - ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count); - #undef taosSend - #undef taosSendto - #undef taosReadSocket - #undef taosWriteSocket - #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags) - #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen) - #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len) - #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len) -#endif - -#ifdef TAOS_MEM_CHECK - void * taos_malloc(size_t size, const char *file, uint32_t line); - void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); - void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); - void taos_free(void *ptr, const char *file, uint32_t line); - char * taos_strdup(const char *str, const char *file, uint32_t line); - char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); - ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); - #define malloc(size) taos_malloc(size, __FILE__, __LINE__) - #define calloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) - #define realloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) - #define free(ptr) taos_free(ptr, __FILE__, __LINE__) - #define strdup(str) taos_strdup(str, __FILE__, __LINE__) - #define strndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) - #define getline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) -#endif // TAOS_MEM_CHECK - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osString.h b/src/os/inc/osString.h new file mode 100644 index 0000000000..b2846a31bc --- /dev/null +++ b/src/os/inc/osString.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_STRING_H +#define TDENGINE_OS_STRING_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef TAOS_OS_FUNC_STRING_STRDUP + #define taosStrdupImp(str) strdup(str) + #define taosStrndupImp(str, size) strndup(str, size) +#endif + +#ifndef TAOS_OS_FUNC_STRING_GETLINE + #define taosGetlineImp(lineptr, n, stream) getline(lineptr, n , stream) +#else + int taosGetlineImp(char **lineptr, size_t *n, FILE *stream); +#endif + +#ifndef TAOS_OS_FUNC_STRING_WCHAR + #define twcslen wcslen +#endif + +#define tstrncpy(dst, src, size) \ + do { \ + strncpy((dst), (src), (size)); \ + (dst)[(size)-1] = 0; \ + } while (0); + +#ifndef TAOS_OS_FUNC_STRING_STR2INT64 + int64_t tsosStr2int64(char *str); +#endif + +// USE_LIBICONV +int32_t taosUcs4ToMbs(void *ucs4, int32_t ucs4_max_len, char *mbs); +bool taosMbsToUcs4(char *mbs, size_t mbs_len, char *ucs4, int32_t ucs4_max_len, size_t *len); +int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes); +bool taosValidateEncodec(const char *encodec); +char * taosCharsetReplace(char *charsetstr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osSysinfo.h b/src/os/inc/osSysinfo.h new file mode 100644 index 0000000000..c3db406a17 --- /dev/null +++ b/src/os/inc/osSysinfo.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_SYSINFO_H +#define TDENGINE_OS_SYSINFO_H + +#ifdef __cplusplus +extern "C" { +#endif + +// TAOS_OS_FUNC_SYSINFO +void taosGetSystemInfo(); +bool taosGetProcIO(float *readKB, float *writeKB); +bool taosGetBandSpeed(float *bandSpeedKb); +bool taosGetDisk(); +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ; +bool taosGetProcMemory(float *memoryUsedMB) ; +bool taosGetSysMemory(float *memoryUsedMB); +void taosPrintOsInfo(); +int taosSystem(const char * cmd) ; +void taosKillSystem(); + +// TAOS_OS_FUNC_SYSINFO_CORE +void taosSetCoreDump(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/util/inc/ttime.h b/src/os/inc/osTime.h similarity index 79% rename from src/util/inc/ttime.h rename to src/os/inc/osTime.h index 25d1bdb23e..cd2553f753 100644 --- a/src/util/inc/ttime.h +++ b/src/os/inc/osTime.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_TTIME_H -#define TDENGINE_TTIME_H +#ifndef TDENGINE_OS_TIME_H +#define TDENGINE_OS_TIME_H #ifdef __cplusplus extern "C" { @@ -23,6 +23,16 @@ extern "C" { #include "os.h" #include "taosdef.h" +#ifndef TAOS_OS_FUNC_TIME_DEF + #define MILLISECOND_PER_SECOND ((int64_t)1000L) +#endif +#define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) +#define MILLISECOND_PER_HOUR (MILLISECOND_PER_MINUTE * 60) +#define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) +#define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) +#define MILLISECOND_PER_MONTH (MILLISECOND_PER_DAY * 30) +#define MILLISECOND_PER_YEAR (MILLISECOND_PER_DAY * 365) + //@return timestamp in second int32_t taosGetTimestampSec(); diff --git a/src/os/inc/osTimer.h b/src/os/inc/osTimer.h new file mode 100644 index 0000000000..d6deae2a6d --- /dev/null +++ b/src/os/inc/osTimer.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_OS_TIMER_H +#define TDENGINE_OS_TIMER_H + +#ifdef __cplusplus +extern "C" { +#endif + +// TAOS_OS_FUNC_TIMER +void taosMsleep(int mseconds); +int taosInitTimer(void (*callback)(int), int ms); +void taosUninitTimer(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h new file mode 100644 index 0000000000..a8c2243253 --- /dev/null +++ b/src/os/inc/osWindows.h @@ -0,0 +1,376 @@ +/* +* Copyright (c) 2019 TAOS Data, Inc. +* +* This program is free software: you can use, redistribute, and/or modify +* it under the terms of the GNU Affero General Public License, version 3 +* or later ("AGPL"), as published by the Free Software Foundation. +* +* This program is distributed in the hope that it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. +* +* You should have received a copy of the GNU Affero General Public License +* along with this program. If not, see . +*/ + +#ifndef TDENGINE_PLATFORM_WINDOWS_H +#define TDENGINE_PLATFORM_WINDOWS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "winsock2.h" +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define TAOS_OS_FUNC_ATOMIC + +#define TAOS_OS_FUNC_LZ4 + int32_t BUILDIN_CLZL(uint64_t val); + int32_t BUILDIN_CLZ(uint32_t val); + int32_t BUILDIN_CTZL(uint64_t val); + int32_t BUILDIN_CTZ(uint32_t val); + +#define TAOS_OS_FUNC_DIR + +#define TAOS_OS_FUNC_FILE +#define TAOS_OS_FUNC_FILE_SENDIFLE + #define taosFSendFile(outfile, infile, offset, count) taosFSendFileImp(outfile, infile, offset, size) + #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size) +#define TAOS_OS_FUNC_FILE_GETTMPFILEPATH + +#define TAOS_OS_FUNC_MATH + #define SWAP(a, b, c) \ + do { \ + c __tmp = (c)(a); \ + (a) = (c)(b); \ + (b) = __tmp; \ + } while (0) + #define MAX(a,b) (((a)>(b))?(a):(b)) + #define MIN(a,b) (((a)<(b))?(a):(b)) + +#define TAOS_OS_FUNC_SEMPHONE_PTHREAD + +#define TAOS_OS_FUNC_SOCKET +#define TAOS_OS_FUNC_SOCKET_SETSOCKETOPT +#define TAOS_OS_FUNC_SOCKET_OP + #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags) + #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen) + #define taosWriteSocket(fd, buf, len) send(fd, buf, len, 0) + #define taosReadSocket(fd, buf, len) recv(fd, buf, len, 0) + #define taosCloseSocket(fd) closesocket(fd) + +#define TAOS_OS_FUNC_STRING_WCHAR +#define TAOS_OS_FUNC_STRING_GETLINE +#define TAOS_OS_FUNC_STRING_STR2INT64 + #ifdef _TD_GO_DLL_ + int64_t tsosStr2int64(char *str); + uint64_t htonll(uint64_t val); + #else + #define tsosStr2int64 _atoi64 + #endif +#define TAOS_OS_FUNC_STRING_STRDUP + #define taosStrdupImp(str) _strdup(str) + #define taosStrndupImp(str, size) _strndup(str, size) + +#define TAOS_OS_FUNC_SYSINFO + +#define TAOS_OS_FUNC_TIME_DEF + #ifdef _TD_GO_DLL_ + #define MILLISECOND_PER_SECOND (1000LL) + #else + #define MILLISECOND_PER_SECOND (1000i64) + #endif + +#define TAOS_OS_FUNC_TIMER_SLEEP +#define TAOS_OS_FUNC_TIMER + +// specific +typedef int (*__compar_fn_t)(const void *, const void *); +#define ssize_t int +#define bzero(ptr, size) memset((ptr), 0, (size)) +#define mkdir(pathname, mode) _mkdir(pathname) +#define strcasecmp _stricmp +#define strncasecmp _strnicmp +#define wcsncasecmp _wcsnicmp +#define strtok_r strtok_s +#define snprintf _snprintf +#define in_addr_t unsigned long +#define socklen_t int +#define htobe64 htonll +#define twrite write +#define getpid _getpid + +int gettimeofday(struct timeval *tv, struct timezone *tz); +struct tm *localtime_r(const time_t *timep, struct tm *result); +char * strptime(const char *buf, const char *fmt, struct tm *tm); +char * strsep(char **stringp, const char *delim); +char * getpass(const char *prefix); +int flock(int fd, int option); +int fsync(int filedes); +char * strndup(const char *s, size_t n); + +// for function open in stat.h +#define S_IRWXU _S_IREAD +#define S_IRWXG _S_IWRITE +#define S_IRWXO _S_IWRITE + +// for access function in io.h +#define F_OK 00 //Existence only +#define W_OK 02 //Write - only +#define R_OK 04 //Read - only +#define X_OK 06 //Read and write + +// for send function in tsocket.c +#define MSG_NOSIGNAL 0 +#define SO_NO_CHECK 0x1234 +#define SOL_TCP 0x1234 +#define TCP_KEEPCNT 0x1234 +#define TCP_KEEPIDLE 0x1234 +#define TCP_KEEPINTVL 0x1234 + +#define LOCK_EX 1 +#define LOCK_NB 2 +#define LOCK_UN 3 + +#ifndef PATH_MAX + #define PATH_MAX 256 +#endif + +//for signal, not dispose +#define SIGALRM 1234 +typedef int sigset_t; +struct sigaction { + void (*sa_handler)(int); +}; +int sigaction(int, struct sigaction *, void *); + +typedef struct { + int we_wordc; + char **we_wordv; + int we_offs; + char wordPos[20]; +} wordexp_t; +int wordexp(const char *words, wordexp_t *pwordexp, int flags); +void wordfree(wordexp_t *pwordexp); + +#define TAOS_OS_FUNC_ATOMIC + #define atomic_load_8(ptr) (*(char volatile*)(ptr)) + #define atomic_load_16(ptr) (*(short volatile*)(ptr)) + #define atomic_load_32(ptr) (*(long volatile*)(ptr)) + #define atomic_load_64(ptr) (*(__int64 volatile*)(ptr)) + #define atomic_load_ptr(ptr) (*(void* volatile*)(ptr)) + + #define atomic_store_8(ptr, val) ((*(char volatile*)(ptr)) = (char)(val)) + #define atomic_store_16(ptr, val) ((*(short volatile*)(ptr)) = (short)(val)) + #define atomic_store_32(ptr, val) ((*(long volatile*)(ptr)) = (long)(val)) + #define atomic_store_64(ptr, val) ((*(__int64 volatile*)(ptr)) = (__int64)(val)) + #define atomic_store_ptr(ptr, val) ((*(void* volatile*)(ptr)) = (void*)(val)) + + #define atomic_exchange_8(ptr, val) _InterlockedExchange8((char volatile*)(ptr), (char)(val)) + #define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) + #define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) + #define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) + #define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) + + #ifdef _TD_GO_DLL_ + #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap + #else + #define atomic_val_compare_exchange_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval)) + #endif + #define atomic_val_compare_exchange_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval)) + #define atomic_val_compare_exchange_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval)) + #define atomic_val_compare_exchange_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval)) + #define atomic_val_compare_exchange_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)) + + char interlocked_add_fetch_8(char volatile *ptr, char val); + short interlocked_add_fetch_16(short volatile *ptr, short val); + long interlocked_add_fetch_32(long volatile *ptr, long val); + __int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val); + + #define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val)) + #define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val)) + #define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val)) + #define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) + #ifdef _WIN64 + #define atomic_add_fetch_ptr atomic_add_fetch_64 + #else + #define atomic_add_fetch_ptr atomic_add_fetch_32 + #endif + + #ifdef _TD_GO_DLL_ + #define atomic_fetch_add_8 __sync_fetch_and_ad + #define atomic_fetch_add_16 __sync_fetch_and_add + #else + #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) + #endif + #define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val)) + #define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val)) + #ifdef _WIN64 + #define atomic_fetch_add_ptr atomic_fetch_add_64 + #else + #define atomic_fetch_add_ptr atomic_fetch_add_32 + #endif + + #define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val)) + #define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val)) + #define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val)) + #define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val)) + #ifdef _WIN64 + #define atomic_sub_fetch_ptr atomic_sub_fetch_64 + #else + #define atomic_sub_fetch_ptr atomic_sub_fetch_32 + #endif + + #define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val)) + #define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val)) + #define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val)) + #define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val)) + #ifdef _WIN64 + #define atomic_fetch_sub_ptr atomic_fetch_sub_64 + #else + #define atomic_fetch_sub_ptr atomic_fetch_sub_32 + #endif + + #ifndef _TD_GO_DLL_ + char interlocked_and_fetch_8(char volatile* ptr, char val); + short interlocked_and_fetch_16(short volatile* ptr, short val); + #endif + long interlocked_and_fetch_32(long volatile* ptr, long val); + __int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val); + + #ifndef _TD_GO_DLL_ + #define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val)) + #define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val)) + #endif + #define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val)) + #define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) + #ifdef _WIN64 + #define atomic_and_fetch_ptr atomic_and_fetch_64 + #else + #define atomic_and_fetch_ptr atomic_and_fetch_32 + #endif + #ifndef _TD_GO_DLL_ + #define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val)) + #endif + #define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val)) + + #ifdef _M_IX86 + __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val); + #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val)) + #else + #define atomic_fetch_and_64(ptr, val) _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)) + #endif + + #ifdef _WIN64 + #define atomic_fetch_and_ptr atomic_fetch_and_64 + #else + #define atomic_fetch_and_ptr atomic_fetch_and_32 + #endif + #ifndef _TD_GO_DLL_ + char interlocked_or_fetch_8(char volatile* ptr, char val); + short interlocked_or_fetch_16(short volatile* ptr, short val); + #endif + long interlocked_or_fetch_32(long volatile* ptr, long val); + __int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val); + + #ifndef _TD_GO_DLL_ + #define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val)) + #define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val)) + #endif + #define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val)) + #define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) + #ifdef _WIN64 + #define atomic_or_fetch_ptr atomic_or_fetch_64 + #else + #define atomic_or_fetch_ptr atomic_or_fetch_32 + #endif + #ifndef _TD_GO_DLL_ + #define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val)) + #endif + #define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val)) + + #ifdef _M_IX86 + __int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val); + #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val)) + #else + #define atomic_fetch_or_64(ptr, val) _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)) + #endif + + #ifdef _WIN64 + #define atomic_fetch_or_ptr atomic_fetch_or_64 + #else + #define atomic_fetch_or_ptr atomic_fetch_or_32 + #endif + + #ifndef _TD_GO_DLL_ + char interlocked_xor_fetch_8(char volatile* ptr, char val); + short interlocked_xor_fetch_16(short volatile* ptr, short val); + #endif + long interlocked_xor_fetch_32(long volatile* ptr, long val); + __int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val); + + #ifndef _TD_GO_DLL_ + #define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val)) + #define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val)) + #endif + #define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val)) + #define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) + #ifdef _WIN64 + #define atomic_xor_fetch_ptr atomic_xor_fetch_64 + #else + #define atomic_xor_fetch_ptr atomic_xor_fetch_32 + #endif + + #ifndef _TD_GO_DLL_ + #define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val)) + #define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val)) + #endif + #define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val)) + + #ifdef _M_IX86 + __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val); + #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val)) + #else + #define atomic_fetch_xor_64(ptr, val) _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)) + #endif + + #ifdef _WIN64 + #define atomic_fetch_xor_ptr atomic_fetch_xor_64 + #else + #define atomic_fetch_xor_ptr atomic_fetch_xor_32 + #endif + +#ifdef __cplusplus +} +#endif +#endif \ No newline at end of file diff --git a/src/os/inc/osWindows32.h b/src/os/inc/osWindows32.h deleted file mode 100644 index 4744d4beb5..0000000000 --- a/src/os/inc/osWindows32.h +++ /dev/null @@ -1,54 +0,0 @@ -/* -* Copyright (c) 2019 TAOS Data, Inc. -* -* This program is free software: you can use, redistribute, and/or modify -* it under the terms of the GNU Affero General Public License, version 3 -* or later ("AGPL"), as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, but WITHOUT -* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -* FITNESS FOR A PARTICULAR PURPOSE. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#ifndef TDENGINE_PLATFORM_WINDOWS32_H -#define TDENGINE_PLATFORM_WINDOWS32_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "winsock2.h" -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef __cplusplus -} -#endif -#endif \ No newline at end of file diff --git a/src/os/inc/osWindows64.h b/src/os/inc/osWindows64.h deleted file mode 100644 index b4687afb82..0000000000 --- a/src/os/inc/osWindows64.h +++ /dev/null @@ -1,424 +0,0 @@ -/* -* Copyright (c) 2019 TAOS Data, Inc. -* -* This program is free software: you can use, redistribute, and/or modify -* it under the terms of the GNU Affero General Public License, version 3 -* or later ("AGPL"), as published by the Free Software Foundation. -* -* This program is distributed in the hope that it will be useful, but WITHOUT -* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -* FITNESS FOR A PARTICULAR PURPOSE. -* -* You should have received a copy of the GNU Affero General Public License -* along with this program. If not, see . -*/ - -#ifndef TDENGINE_PLATFORM_WINDOWS_H -#define TDENGINE_PLATFORM_WINDOWS_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "winsock2.h" -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -#define TAOS_OS_FUNC_WCHAR - -// for function open in stat.h -#define S_IRWXU _S_IREAD -#define S_IRWXG _S_IWRITE -#define S_IRWXO _S_IWRITE - -// for access function in io.h -#define F_OK 00 //Existence only -#define W_OK 02 //Write - only -#define R_OK 04 //Read - only -#define X_OK 06 //Read and write - -// for send function in tsocket.c -#define MSG_NOSIGNAL 0 -#define SO_NO_CHECK 0x1234 -#define SOL_TCP 0x1234 -#define TCP_KEEPCNT 0x1234 -#define TCP_KEEPIDLE 0x1234 -#define TCP_KEEPINTVL 0x1234 - -#define LOCK_EX 1 -#define LOCK_NB 2 -#define LOCK_UN 3 - -#define bzero(ptr, size) memset((ptr), 0, (size)) -#define mkdir(pathname, mode) _mkdir(pathname) -#define strcasecmp _stricmp -#define strncasecmp _strnicmp -#define wcsncasecmp _wcsnicmp -#define strtok_r strtok_s -#ifdef _TD_GO_DLL_ - int64_t tsosStr2int64(char *str); - uint64_t htonll(uint64_t val); -#else - #define tsosStr2int64 _atoi64 -#endif - -#define snprintf _snprintf -#define in_addr_t unsigned long -#define socklen_t int -#define htobe64 htonll -#define twrite write - -#ifndef PATH_MAX - #define PATH_MAX 256 -#endif - -#define taosCloseSocket(fd) closesocket(fd) -#define taosWriteSocket(fd, buf, len) send(fd, buf, len, 0) -#define taosReadSocket(fd, buf, len) recv(fd, buf, len, 0) - -#if defined(_M_ARM) || defined(_M_ARM64) - -/* the '__iso_volatile' functions does not use a memory fence, so these - * definitions are incorrect, comment out as we don't support Windows on - * ARM at present. - -#define atomic_load_8(ptr) __iso_volatile_load8((const volatile __int8*)(ptr)) -#define atomic_load_16(ptr) __iso_volatile_load16((const volatile __int16*)(ptr)) -#define atomic_load_32(ptr) __iso_volatile_load32((const volatile __int32*)(ptr)) -#define atomic_load_64(ptr) __iso_volatile_load64((const volatile __int64*)(ptr)) - -#define atomic_store_8(ptr, val) __iso_volatile_store8((volatile __int8*)(ptr), (__int8)(val)) -#define atomic_store_16(ptr, val) __iso_volatile_store16((volatile __int16*)(ptr), (__int16)(val)) -#define atomic_store_32(ptr, val) __iso_volatile_store32((volatile __int32*)(ptr), (__int32)(val)) -#define atomic_store_64(ptr, val) __iso_volatile_store64((volatile __int64*)(ptr), (__int64)(val)) - -#ifdef _M_ARM64 -#define atomic_load_ptr atomic_load_64 -#define atomic_store_ptr atomic_store_64 -#else -#define atomic_load_ptr atomic_load_32 -#define atomic_store_ptr atomic_store_32 -#endif -*/ -#else - -#define atomic_load_8(ptr) (*(char volatile*)(ptr)) -#define atomic_load_16(ptr) (*(short volatile*)(ptr)) -#define atomic_load_32(ptr) (*(long volatile*)(ptr)) -#define atomic_load_64(ptr) (*(__int64 volatile*)(ptr)) -#define atomic_load_ptr(ptr) (*(void* volatile*)(ptr)) - -#define atomic_store_8(ptr, val) ((*(char volatile*)(ptr)) = (char)(val)) -#define atomic_store_16(ptr, val) ((*(short volatile*)(ptr)) = (short)(val)) -#define atomic_store_32(ptr, val) ((*(long volatile*)(ptr)) = (long)(val)) -#define atomic_store_64(ptr, val) ((*(__int64 volatile*)(ptr)) = (__int64)(val)) -#define atomic_store_ptr(ptr, val) ((*(void* volatile*)(ptr)) = (void*)(val)) - -#endif - -#define atomic_exchange_8(ptr, val) _InterlockedExchange8((char volatile*)(ptr), (char)(val)) -#define atomic_exchange_16(ptr, val) _InterlockedExchange16((short volatile*)(ptr), (short)(val)) -#define atomic_exchange_32(ptr, val) _InterlockedExchange((long volatile*)(ptr), (long)(val)) -#define atomic_exchange_64(ptr, val) _InterlockedExchange64((__int64 volatile*)(ptr), (__int64)(val)) -#define atomic_exchange_ptr(ptr, val) _InterlockedExchangePointer((void* volatile*)(ptr), (void*)(val)) - -#ifdef _TD_GO_DLL_ - #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap -#else - #define atomic_val_compare_exchange_8(ptr, oldval, newval) _InterlockedCompareExchange8((char volatile*)(ptr), (char)(newval), (char)(oldval)) -#endif - -#define atomic_val_compare_exchange_16(ptr, oldval, newval) _InterlockedCompareExchange16((short volatile*)(ptr), (short)(newval), (short)(oldval)) -#define atomic_val_compare_exchange_32(ptr, oldval, newval) _InterlockedCompareExchange((long volatile*)(ptr), (long)(newval), (long)(oldval)) -#define atomic_val_compare_exchange_64(ptr, oldval, newval) _InterlockedCompareExchange64((__int64 volatile*)(ptr), (__int64)(newval), (__int64)(oldval)) -#define atomic_val_compare_exchange_ptr(ptr, oldval, newval) _InterlockedCompareExchangePointer((void* volatile*)(ptr), (void*)(newval), (void*)(oldval)) - -char interlocked_add_fetch_8(char volatile *ptr, char val); -short interlocked_add_fetch_16(short volatile *ptr, short val); -long interlocked_add_fetch_32(long volatile *ptr, long val); -__int64 interlocked_add_fetch_64(__int64 volatile *ptr, __int64 val); - -#define atomic_add_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), (char)(val)) -#define atomic_add_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), (short)(val)) -#define atomic_add_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), (long)(val)) -#define atomic_add_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) -#ifdef _WIN64 - #define atomic_add_fetch_ptr atomic_add_fetch_64 -#else - #define atomic_add_fetch_ptr atomic_add_fetch_32 -#endif -#ifdef _TD_GO_DLL_ - #define atomic_fetch_add_8 __sync_fetch_and_ad - #define atomic_fetch_add_16 __sync_fetch_and_add -#else - #define atomic_fetch_add_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_add_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), (short)(val)) -#endif - -#define atomic_fetch_add_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), (long)(val)) -#define atomic_fetch_add_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), (__int64)(val)) -#ifdef _WIN64 - #define atomic_fetch_add_ptr atomic_fetch_add_64 -#else - #define atomic_fetch_add_ptr atomic_fetch_add_32 -#endif - -#define atomic_sub_fetch_8(ptr, val) interlocked_add_fetch_8((char volatile*)(ptr), -(char)(val)) -#define atomic_sub_fetch_16(ptr, val) interlocked_add_fetch_16((short volatile*)(ptr), -(short)(val)) -#define atomic_sub_fetch_32(ptr, val) interlocked_add_fetch_32((long volatile*)(ptr), -(long)(val)) -#define atomic_sub_fetch_64(ptr, val) interlocked_add_fetch_64((__int64 volatile*)(ptr), -(__int64)(val)) -#ifdef _WIN64 - #define atomic_sub_fetch_ptr atomic_sub_fetch_64 -#else - #define atomic_sub_fetch_ptr atomic_sub_fetch_32 -#endif - -#define atomic_fetch_sub_8(ptr, val) _InterlockedExchangeAdd8((char volatile*)(ptr), -(char)(val)) -#define atomic_fetch_sub_16(ptr, val) _InterlockedExchangeAdd16((short volatile*)(ptr), -(short)(val)) -#define atomic_fetch_sub_32(ptr, val) _InterlockedExchangeAdd((long volatile*)(ptr), -(long)(val)) -#define atomic_fetch_sub_64(ptr, val) _InterlockedExchangeAdd64((__int64 volatile*)(ptr), -(__int64)(val)) -#ifdef _WIN64 - #define atomic_fetch_sub_ptr atomic_fetch_sub_64 -#else - #define atomic_fetch_sub_ptr atomic_fetch_sub_32 -#endif -#ifndef _TD_GO_DLL_ - char interlocked_and_fetch_8(char volatile* ptr, char val); - short interlocked_and_fetch_16(short volatile* ptr, short val); -#endif -long interlocked_and_fetch_32(long volatile* ptr, long val); -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val); - -#ifndef _TD_GO_DLL_ - #define atomic_and_fetch_8(ptr, val) interlocked_and_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_and_fetch_16(ptr, val) interlocked_and_fetch_16((short volatile*)(ptr), (short)(val)) -#endif -#define atomic_and_fetch_32(ptr, val) interlocked_and_fetch_32((long volatile*)(ptr), (long)(val)) -#define atomic_and_fetch_64(ptr, val) interlocked_and_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) -#ifdef _WIN64 - #define atomic_and_fetch_ptr atomic_and_fetch_64 -#else - #define atomic_and_fetch_ptr atomic_and_fetch_32 -#endif -#ifndef _TD_GO_DLL_ - #define atomic_fetch_and_8(ptr, val) _InterlockedAnd8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_and_16(ptr, val) _InterlockedAnd16((short volatile*)(ptr), (short)(val)) -#endif -#define atomic_fetch_and_32(ptr, val) _InterlockedAnd((long volatile*)(ptr), (long)(val)) - -#ifdef _M_IX86 - __int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val); - #define atomic_fetch_and_64(ptr, val) interlocked_fetch_and_64((__int64 volatile*)(ptr), (__int64)(val)) -#else - #define atomic_fetch_and_64(ptr, val) _InterlockedAnd64((__int64 volatile*)(ptr), (__int64)(val)) -#endif - -#ifdef _WIN64 - #define atomic_fetch_and_ptr atomic_fetch_and_64 -#else - #define atomic_fetch_and_ptr atomic_fetch_and_32 -#endif -#ifndef _TD_GO_DLL_ - char interlocked_or_fetch_8(char volatile* ptr, char val); - short interlocked_or_fetch_16(short volatile* ptr, short val); -#endif -long interlocked_or_fetch_32(long volatile* ptr, long val); -__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val); - -#ifndef _TD_GO_DLL_ - #define atomic_or_fetch_8(ptr, val) interlocked_or_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_or_fetch_16(ptr, val) interlocked_or_fetch_16((short volatile*)(ptr), (short)(val)) -#endif -#define atomic_or_fetch_32(ptr, val) interlocked_or_fetch_32((long volatile*)(ptr), (long)(val)) -#define atomic_or_fetch_64(ptr, val) interlocked_or_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) -#ifdef _WIN64 - #define atomic_or_fetch_ptr atomic_or_fetch_64 -#else - #define atomic_or_fetch_ptr atomic_or_fetch_32 -#endif -#ifndef _TD_GO_DLL_ - #define atomic_fetch_or_8(ptr, val) _InterlockedOr8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_or_16(ptr, val) _InterlockedOr16((short volatile*)(ptr), (short)(val)) -#endif -#define atomic_fetch_or_32(ptr, val) _InterlockedOr((long volatile*)(ptr), (long)(val)) - -#ifdef _M_IX86 - __int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val); - #define atomic_fetch_or_64(ptr, val) interlocked_fetch_or_64((__int64 volatile*)(ptr), (__int64)(val)) -#else - #define atomic_fetch_or_64(ptr, val) _InterlockedOr64((__int64 volatile*)(ptr), (__int64)(val)) -#endif - -#ifdef _WIN64 - #define atomic_fetch_or_ptr atomic_fetch_or_64 -#else - #define atomic_fetch_or_ptr atomic_fetch_or_32 -#endif - -#ifndef _TD_GO_DLL_ - char interlocked_xor_fetch_8(char volatile* ptr, char val); - short interlocked_xor_fetch_16(short volatile* ptr, short val); -#endif -long interlocked_xor_fetch_32(long volatile* ptr, long val); -__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val); - -#ifndef _TD_GO_DLL_ - #define atomic_xor_fetch_8(ptr, val) interlocked_xor_fetch_8((char volatile*)(ptr), (char)(val)) - #define atomic_xor_fetch_16(ptr, val) interlocked_xor_fetch_16((short volatile*)(ptr), (short)(val)) -#endif -#define atomic_xor_fetch_32(ptr, val) interlocked_xor_fetch_32((long volatile*)(ptr), (long)(val)) -#define atomic_xor_fetch_64(ptr, val) interlocked_xor_fetch_64((__int64 volatile*)(ptr), (__int64)(val)) -#ifdef _WIN64 - #define atomic_xor_fetch_ptr atomic_xor_fetch_64 -#else - #define atomic_xor_fetch_ptr atomic_xor_fetch_32 -#endif - -#ifndef _TD_GO_DLL_ - #define atomic_fetch_xor_8(ptr, val) _InterlockedXor8((char volatile*)(ptr), (char)(val)) - #define atomic_fetch_xor_16(ptr, val) _InterlockedXor16((short volatile*)(ptr), (short)(val)) -#endif -#define atomic_fetch_xor_32(ptr, val) _InterlockedXor((long volatile*)(ptr), (long)(val)) - -#ifdef _M_IX86 - __int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val); - #define atomic_fetch_xor_64(ptr, val) interlocked_fetch_xor_64((__int64 volatile*)(ptr), (__int64)(val)) -#else - #define atomic_fetch_xor_64(ptr, val) _InterlockedXor64((__int64 volatile*)(ptr), (__int64)(val)) -#endif - -#ifdef _WIN64 - #define atomic_fetch_xor_ptr atomic_fetch_xor_64 -#else - #define atomic_fetch_xor_ptr atomic_fetch_xor_32 -#endif - -#define SWAP(a, b, c) \ - do { \ - c __tmp = (c)(a); \ - (a) = (c)(b); \ - (b) = __tmp; \ - } while (0) - -#define MAX(a,b) (((a)>(b))?(a):(b)) -#define MIN(a,b) (((a)<(b))?(a):(b)) - -#ifdef _TD_GO_DLL_ - #define MILLISECOND_PER_SECOND (1000LL) -#else - #define MILLISECOND_PER_SECOND (1000i64) -#endif - -#define tsem_t sem_t -#define tsem_init sem_init -#define tsem_wait sem_wait -#define tsem_post sem_post -#define tsem_destroy sem_destroy - -void osInit(); - -int getline(char **lineptr, size_t *n, FILE *stream); - -int taosWinSetTimer(int ms, void(*callback)(int)); - -int gettimeofday(struct timeval *tv, struct timezone *tz); - -struct tm *localtime_r(const time_t *timep, struct tm *result); - -char *strptime(const char *buf, const char *fmt, struct tm *tm); - -bool taosCheckPthreadValid(pthread_t thread); - -void taosResetPthread(pthread_t *thread); - -int64_t taosGetPthreadId(); - -int taosSetNonblocking(int sock, int on); - -int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen); - - -void taosPrintOsInfo(); - -void taosGetSystemInfo(); - -void taosKillSystem(); - -int32_t BUILDIN_CLZL(uint64_t val); -int32_t BUILDIN_CLZ(uint32_t val); -int32_t BUILDIN_CTZL(uint64_t val); -int32_t BUILDIN_CTZ(uint32_t val); - -//for signal, not dispose -#define SIGALRM 1234 -typedef int sigset_t; - -struct sigaction { - void (*sa_handler)(int); -}; - -typedef struct { - int we_wordc; - char **we_wordv; - int we_offs; - char wordPos[20]; -} wordexp_t; - -int wordexp(const char *words, wordexp_t *pwordexp, int flags); - -void wordfree(wordexp_t *pwordexp); - -int flock(int fd, int option); - -int fsync(int filedes); - -char *getpass(const char *prefix); - -char *strsep(char **stringp, const char *delim); - -typedef int(*__compar_fn_t)(const void *, const void *); - -int sigaction(int, struct sigaction *, void *); - -void sleep(int mseconds); - -bool taosSkipSocketCheck(); - -int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count); - -#define ssize_t int - -#define strdup _strdup - -char *strndup(const char *s, size_t n); - -void taosSetCoreDump(); - -#ifdef __cplusplus -} -#endif -#endif \ No newline at end of file diff --git a/src/os/src/alpine/alpineEnv.c b/src/os/src/alpine/alpineEnv.c index 0a9d81311a..4f84412075 100644 --- a/src/os/src/alpine/alpineEnv.c +++ b/src/os/src/alpine/alpineEnv.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" -#include "tulog.h" void osInit() { strcpy(configDir, "/etc/taos"); diff --git a/src/os/src/darwin/CMakeLists.txt b/src/os/src/darwin/CMakeLists.txt new file mode 100644 index 0000000000..1568d16164 --- /dev/null +++ b/src/os/src/darwin/CMakeLists.txt @@ -0,0 +1,10 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) +AUX_SOURCE_DIRECTORY(. SRC) + +ADD_LIBRARY(os ${SRC}) diff --git a/src/os/src/darwin64/darwinEnv.c b/src/os/src/darwin/darwinEnv.c similarity index 83% rename from src/os/src/darwin64/darwinEnv.c rename to src/os/src/darwin/darwinEnv.c index 27d5a7c99b..7e1031a5af 100644 --- a/src/os/src/darwin64/darwinEnv.c +++ b/src/os/src/darwin/darwinEnv.c @@ -16,15 +16,14 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" -#include "tulog.h" void osInit() { - strcpy(configDir, "/etc/taos"); + strcpy(configDir, "~/TDengine/cfg"); strcpy(tsVnodeDir, ""); strcpy(tsDnodeDir, ""); strcpy(tsMnodeDir, ""); - strcpy(tsDataDir, "/var/lib/taos"); - strcpy(tsLogDir, "~/TDengineLog"); - strcpy(tsScriptDir, "/etc/taos"); + strcpy(tsDataDir, "~/TDengine/data"); + strcpy(tsLogDir, "~/TDengine/log"); + strcpy(tsScriptDir, "~/TDengine/cfg"); strcpy(tsOsName, "Darwin"); } diff --git a/src/os/src/darwin/darwinFile.c b/src/os/src/darwin/darwinFile.c new file mode 100644 index 0000000000..66bdb5b939 --- /dev/null +++ b/src/os/src/darwin/darwinFile.c @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tulog.h" + +#define _SEND_FILE_STEP_ 1000 + +int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { + fseek(in_file, (int32_t)(*offset), 0); + int writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; + + for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); + if (rlen <= 0) { + return writeLen; + } + else if (rlen < _SEND_FILE_STEP_) { + fwrite(buffer, 1, rlen, out_file); + return (int)(writeLen + rlen); + } + else { + fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); + writeLen += _SEND_FILE_STEP_; + } + } + + int remain = count - writeLen; + if (remain > 0) { + size_t rlen = fread(buffer, 1, remain, in_file); + if (rlen <= 0) { + return writeLen; + } + else { + fwrite(buffer, 1, remain, out_file); + writeLen += remain; + } + } + + return writeLen; +} + +ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { + uError("not implemented yet"); + return -1; +} \ No newline at end of file diff --git a/src/os/src/darwin/darwinSemphone.c b/src/os/src/darwin/darwinSemphone.c new file mode 100644 index 0000000000..97ff543789 --- /dev/null +++ b/src/os/src/darwin/darwinSemphone.c @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value) { + *sem = dispatch_semaphore_create(value); + if (*sem == NULL) { + return -1; + } else { + return 0; + } +} + +int tsem_wait(dispatch_semaphore_t *sem) { + dispatch_semaphore_wait(*sem, DISPATCH_TIME_FOREVER); + return 0; +} + +int tsem_post(dispatch_semaphore_t *sem) { + dispatch_semaphore_signal(*sem); + return 0; +} + +int tsem_destroy(dispatch_semaphore_t *sem) { + return 0; +} diff --git a/src/os/src/windows64/w64File.c b/src/os/src/darwin/darwinSocket.c similarity index 54% rename from src/os/src/windows64/w64File.c rename to src/os/src/darwin/darwinSocket.c index 140277f990..e51f2c4fba 100644 --- a/src/os/src/windows64/w64File.c +++ b/src/os/src/darwin/darwinSocket.c @@ -16,23 +16,14 @@ #define _DEFAULT_SOURCE #include "os.h" -void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - char tmpPath[PATH_MAX]; +int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) { + if (level == SOL_SOCKET && optname == SO_SNDBUF) { + return 0; + } - char *tmpDir = getenv("tmp"); - if (tmpDir == NULL) { - tmpDir = ""; + if (level == SOL_SOCKET && optname == SO_RCVBUF) { + return 0; } - - strcpy(tmpPath, tmpDir); - strcat(tmpPath, tdengineTmpFileNamePrefix); - if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { - strcat(tmpPath, fileNamePrefix); - strcat(tmpPath, "-%d-%s"); - } - - char rand[8] = {0}; - taosRandStr(rand, tListLen(rand) - 1); - snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); + + return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); } diff --git a/src/os/src/detail/osUtil.c b/src/os/src/darwin/darwinString.c similarity index 95% rename from src/os/src/detail/osUtil.c rename to src/os/src/darwin/darwinString.c index 10576c9a0a..3042e78666 100644 --- a/src/os/src/detail/osUtil.c +++ b/src/os/src/darwin/darwinString.c @@ -16,11 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#ifndef TAOS_OS_FUNC_UTIL - int64_t tsosStr2int64(char *str) { char *endptr = NULL; return strtoll(str, &endptr, 10); } - -#endif \ No newline at end of file diff --git a/src/os/src/darwin/darwinSysInfo.c b/src/os/src/darwin/darwinSysInfo.c new file mode 100644 index 0000000000..108cc6239f --- /dev/null +++ b/src/os/src/darwin/darwinSysInfo.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tulog.h" + +static void taosGetSystemTimezone() { + // get and set default timezone + SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); + if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { + char *tz = getenv("TZ"); + if (tz == NULL || strlen(tz) == 0) { + strcpy(tsTimezone, "not configured"); + } + else { + strcpy(tsTimezone, tz); + } + cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uInfo("timezone not configured, use default"); + } +} + +static void taosGetSystemLocale() { + // get and set default locale + SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); + if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + strncpy(tsLocale, locale, TSDB_LOCALE_LEN - 1); + cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uInfo("locale not configured, set to default:%s", tsLocale); + } + } + + SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); + if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { + strcpy(tsCharset, "cp936"); + cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uInfo("charset not configured, set to default:%s", tsCharset); + } +} + +void taosPrintOsInfo() {} + +void taosKillSystem() { + uError("function taosKillSystem, exit!"); + exit(0); +} + +void taosGetSystemInfo() { + taosGetSystemTimezone(); + taosGetSystemLocale(); +} + +bool taosGetDisk() { return true; } + +bool taosGetProcIO(float *readKB, float *writeKB) { + *readKB = 0; + *writeKB = 0; + return true; +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; + return true; +} + +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + *memoryUsedMB = 0; + return true; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + *memoryUsedMB = 0; + return true; +} + +int taosSystem(const char *cmd) { + uError("un support funtion"); + return -1; +} + +void taosSetCoreDump() {} \ No newline at end of file diff --git a/src/os/src/darwin/darwinTimer.c b/src/os/src/darwin/darwinTimer.c new file mode 100644 index 0000000000..5fe65fb99e --- /dev/null +++ b/src/os/src/darwin/darwinTimer.c @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +int taosInitTimer(void (*callback)(int), int ms) { + signal(SIGALRM, callback); + + struct itimerval tv; + tv.it_interval.tv_sec = 0; /* my timer resolution */ + tv.it_interval.tv_usec = 1000 * ms; // resolution is in msecond + tv.it_value = tv.it_interval; + + setitimer(ITIMER_REAL, &tv, NULL); + + return 0; +} + +void taosUninitTimer() { + struct itimerval tv = { 0 }; + setitimer(ITIMER_REAL, &tv, NULL); +} + diff --git a/src/os/src/darwin64/CMakeLists.txt b/src/os/src/darwin64/CMakeLists.txt deleted file mode 100644 index 71029d9291..0000000000 --- a/src/os/src/darwin64/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -IF (TD_DARWIN_64) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(os ${SRC}) -ENDIF () - diff --git a/src/os/src/darwin64/darwinPlatform.c b/src/os/src/darwin64/darwinPlatform.c deleted file mode 100644 index 045e4a7099..0000000000 --- a/src/os/src/darwin64/darwinPlatform.c +++ /dev/null @@ -1,370 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "os.h" -#include "taosdef.h" -#include "tglobal.h" -#include "tconfig.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" - -int64_t tsosStr2int64(char *str) { - char *endptr = NULL; - return strtoll(str, &endptr, 10); -} - -/* - to make taosMsleep work, - signal SIGALRM shall be blocked in the calling thread, - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGALRM); - pthread_sigmask(SIG_BLOCK, &set, NULL); -*/ -void taosMsleep(int mseconds) { - struct timeval timeout; - int seconds, useconds; - - seconds = mseconds / 1000; - useconds = (mseconds % 1000) * 1000; - timeout.tv_sec = seconds; - timeout.tv_usec = useconds; - - /* sigset_t set; */ - /* sigemptyset(&set); */ - /* sigaddset(&set, SIGALRM); */ - /* pthread_sigmask(SIG_BLOCK, &set, NULL); */ - - select(0, NULL, NULL, NULL, &timeout); - - /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ -} - -bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } - -void taosResetPthread(pthread_t *thread) { *thread = 0; } - -int64_t taosGetPthreadId() { return (int64_t)pthread_self(); } - -/* -* Function to get the private ip address of current machine. If get IP -* successfully, return 0, else, return -1. The return values is ip. -* -* Use: -* if (taosGetPrivateIp(ip) != 0) { -* perror("Fail to get private IP address\n"); -* exit(EXIT_FAILURE); -* } -*/ -int taosGetPrivateIp(char *const ip) { - bool hasLoCard = false; - - struct ifaddrs *ifaddr, *ifa; - int family, s; - char host[NI_MAXHOST]; - - if (getifaddrs(&ifaddr) == -1) { - return -1; - } - - /* Walk through linked list, maintaining head pointer so we can free list later */ - int flag = 0; - for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) { - if (ifa->ifa_addr == NULL) continue; - - family = ifa->ifa_addr->sa_family; - if (strcmp("lo", ifa->ifa_name) == 0) { - hasLoCard = true; - continue; - } - - if (family == AF_INET) { - /* printf("%-8s", ifa->ifa_name); */ - s = getnameinfo(ifa->ifa_addr, (family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6), - host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST); - if (s != 0) { - freeifaddrs(ifaddr); - return -1; - } - - strcpy(ip, host); - flag = 1; - break; - } - } - - freeifaddrs(ifaddr); - if (flag) { - return 0; - } else { - if (hasLoCard) { - uInfo("no net card was found, use lo:127.0.0.1 as default"); - strcpy(ip, "127.0.0.1"); - return 0; - } - return -1; - } -} - -int taosSetNonblocking(int sock, int on) { - int flags = 0; - if ((flags = fcntl(sock, F_GETFL, 0)) < 0) { - uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - if (on) - flags |= O_NONBLOCK; - else - flags &= ~O_NONBLOCK; - - if ((flags = fcntl(sock, F_SETFL, flags)) < 0) { - uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno)); - return 1; - } - - return 0; -} - -int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) { - if (level == SOL_SOCKET && optname == SO_SNDBUF) { - return 0; - } - - if (level == SOL_SOCKET && optname == SO_RCVBUF) { - return 0; - } - - return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); -} - -int taosInitTimer(void (*callback)(int), int ms) { - signal(SIGALRM, callback); - - struct itimerval tv; - tv.it_interval.tv_sec = 0; /* my timer resolution */ - tv.it_interval.tv_usec = 1000 * ms; // resolution is in msecond - tv.it_value = tv.it_interval; - - setitimer(ITIMER_REAL, &tv, NULL); - - return 0; -} - -void taosUninitTimer() { - struct itimerval tv = { 0 }; - setitimer(ITIMER_REAL, &tv, NULL); -} - -void taosGetSystemTimezone() { - // get and set default timezone - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); - } - else { - strcpy(tsTimezone, tz); - } - cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("timezone not configured, use default"); - } -} - -void taosGetSystemLocale() { - // get and set default locale - SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); - if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, sizeof(tsLocale)); - cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("locale not configured, set to default:%s", tsLocale); - } - } - - SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); - if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - strcpy(tsCharset, "cp936"); - cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("charset not configured, set to default:%s", tsCharset); - } -} - - -void taosPrintOsInfo() {} - -void taosKillSystem() { - uError("function taosKillSystem, exit!"); - exit(0); -} - -bool taosGetDisk() { - return true; -} - -void taosGetSystemInfo() { - taosGetSystemTimezone(); - taosGetSystemLocale(); -} - -void *taosInitTcpClient(char *ip, uint16_t port, char *flabel, int num, void *fp, void *shandle) { - uError("function taosInitTcpClient is not implemented in darwin system, exit!"); - exit(0); -} - -void taosCloseTcpClientConnection(void *chandle) { - uError("function taosCloseTcpClientConnection is not implemented in darwin system, exit!"); - exit(0); -} - -void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { - uError("function taosOpenTcpClientConnection is not implemented in darwin system, exit!"); - exit(0); -} - -int taosSendTcpClientData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) { - uError("function taosSendTcpClientData is not implemented in darwin system, exit!"); - exit(0); -} - -void taosCleanUpTcpClient(void *chandle) { - uError("function taosCleanUpTcpClient is not implemented in darwin system, exit!"); - exit(0); -} - -void taosCloseTcpServerConnection(void *chandle) { - uError("function taosCloseTcpServerConnection is not implemented in darwin system, exit!"); - exit(0); -} - -void taosCleanUpTcpServer(void *handle) { - uError("function taosCleanUpTcpServer is not implemented in darwin system, exit!"); - exit(0); -} - -void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - uError("function taosInitTcpServer is not implemented in darwin system, exit!"); - exit(0); -} - -int taosSendTcpServerData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) { - uError("function taosSendTcpServerData is not implemented in darwin system, exit!"); - exit(0); -} - -void taosFreeMsgHdr(void *hdr) { - uError("function taosFreeMsgHdr is not implemented in darwin system, exit!"); - exit(0); -} - -int taosMsgHdrSize(void *hdr) { - uError("function taosMsgHdrSize is not implemented in darwin system, exit!"); - exit(0); -} - -void taosSendMsgHdr(void *hdr, int fd) { - uError("function taosSendMsgHdr is not implemented in darwin system, exit!"); - exit(0); -} - -void taosInitMsgHdr(void **hdr, void *dest, int maxPkts) { - uError("function taosInitMsgHdr is not implemented in darwin system, exit!"); - exit(0); -} - -void taosSetMsgHdrData(void *hdr, char *data, int dataLen) { - uError("function taosSetMsgHdrData is not implemented in darwin system, exit!"); - exit(0); -} - -bool taosSkipSocketCheck() { - return true; -} - -int tsem_init(dispatch_semaphore_t *sem, int pshared, unsigned int value) { - *sem = dispatch_semaphore_create(value); - if (*sem == NULL) { - return -1; - } else { - return 0; - } -} - -int tsem_wait(dispatch_semaphore_t *sem) { - dispatch_semaphore_wait(*sem, DISPATCH_TIME_FOREVER); - return 0; -} - -int tsem_post(dispatch_semaphore_t *sem) { - dispatch_semaphore_signal(*sem); - return 0; -} - -int tsem_destroy(dispatch_semaphore_t *sem) { - return 0; -} - -int32_t __sync_val_load_32(int32_t *ptr) { - return __atomic_load_n(ptr, __ATOMIC_ACQUIRE); -} - -void __sync_val_restore_32(int32_t *ptr, int32_t newval) { - __atomic_store_n(ptr, newval, __ATOMIC_RELEASE); -} - -#define _SEND_FILE_STEP_ 1000 - -int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { - fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - - for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); - if (rlen <= 0) { - return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); - return (int)(writeLen + rlen); - } - else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); - writeLen += _SEND_FILE_STEP_; - } - } - - int remain = count - writeLen; - if (remain > 0) { - size_t rlen = fread(buffer, 1, remain, in_file); - if (rlen <= 0) { - return writeLen; - } - else { - fwrite(buffer, 1, remain, out_file); - writeLen += remain; - } - } - - return writeLen; -} - -void taosSetCoreDump() {} diff --git a/src/os/src/detail/osCoredump.c b/src/os/src/detail/osCoredump.c deleted file mode 100644 index 1ead88ab9e..0000000000 --- a/src/os/src/detail/osCoredump.c +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tulog.h" -#include "tsystem.h" - -#ifndef TAOS_OS_FUNC_CORE - -int _sysctl(struct __sysctl_args *args ); - -void taosSetCoreDump() { - if (0 == tsEnableCoreFile) { - return; - } - - // 1. set ulimit -c unlimited - struct rlimit rlim; - struct rlimit rlim_new; - if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - rlim_new.rlim_cur = RLIM_INFINITY; - rlim_new.rlim_max = RLIM_INFINITY; - if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { - uInfo("set unlimited fail, error: %s", strerror(errno)); - rlim_new.rlim_cur = rlim.rlim_max; - rlim_new.rlim_max = rlim.rlim_max; - (void)setrlimit(RLIMIT_CORE, &rlim_new); - } - } - - if (getrlimit(RLIMIT_CORE, &rlim) == 0) { - uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); - } - -#ifndef _TD_ARM_ - // 2. set the path for saving core file - struct __sysctl_args args; - int old_usespid = 0; - size_t old_len = 0; - int new_usespid = 1; - size_t new_len = sizeof(new_usespid); - - int name[] = {CTL_KERN, KERN_CORE_USES_PID}; - - memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; - args.oldlenp = &old_len; - args.newval = &new_usespid; - args.newlen = new_len; - - old_len = sizeof(old_usespid); - - if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); - } - - uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); - - - old_usespid = 0; - old_len = 0; - memset(&args, 0, sizeof(struct __sysctl_args)); - args.name = name; - args.nlen = sizeof(name)/sizeof(name[0]); - args.oldval = &old_usespid; - args.oldlenp = &old_len; - - old_len = sizeof(old_usespid); - - if (syscall(SYS__sysctl, &args) == -1) { - uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); - } - - uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); -#endif - -} - -#endif \ No newline at end of file diff --git a/src/os/src/detail/osDir.c b/src/os/src/detail/osDir.c index 27555938c2..9496b74405 100644 --- a/src/os/src/detail/osDir.c +++ b/src/os/src/detail/osDir.c @@ -60,9 +60,8 @@ void taosMvDir(char* destDir, char *srcDir) { //(void)snprintf(shellCmd, 1024, "cp -rf %s %s", srcDir, destDir); (void)snprintf(shellCmd, 1024, "mv %s %s", srcDir, destDir); - tSystem(shellCmd); + taosSystem(shellCmd); uInfo("shell cmd:%s is executed", shellCmd); } - #endif \ No newline at end of file diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 5502dfb4a6..516b7bb19e 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -15,27 +15,26 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "ttime.h" -#ifndef TAOS_OS_FUNC_FILE +#ifndef TAOS_OS_FUNC_FILE_GETTMPFILEPATH +void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { + const char *tdengineTmpFileNamePrefix = "tdengine-"; -void getTmpfilePath(const char *fileNamePrefix, char *dstPath) { - const char* tdengineTmpFileNamePrefix = "tdengine-"; - - char tmpPath[PATH_MAX]; + char tmpPath[PATH_MAX]; char *tmpDir = "/tmp/"; - + strcpy(tmpPath, tmpDir); strcat(tmpPath, tdengineTmpFileNamePrefix); if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { strcat(tmpPath, fileNamePrefix); strcat(tmpPath, "-%d-%s"); } - + char rand[8] = {0}; taosRandStr(rand, tListLen(rand) - 1); snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); } +#endif // rename file name int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstPath) { @@ -65,4 +64,74 @@ int32_t taosFileRename(char *fullPath, char *suffix, char delimiter, char **dstP return rename(fullPath, *dstPath); } +ssize_t taosTReadImp(int fd, void *buf, size_t count) { + size_t leftbytes = count; + ssize_t readbytes; + char * tbuf = (char *)buf; + + while (leftbytes > 0) { + readbytes = read(fd, (void *)tbuf, leftbytes); + if (readbytes < 0) { + if (errno == EINTR) { + continue; + } else { + return -1; + } + } else if (readbytes == 0) { + return (ssize_t)(count - leftbytes); + } + + leftbytes -= readbytes; + tbuf += readbytes; + } + + return (ssize_t)count; +} + +ssize_t taosTWriteImp(int fd, void *buf, size_t n) { + size_t nleft = n; + ssize_t nwritten = 0; + char * tbuf = (char *)buf; + + while (nleft > 0) { + nwritten = write(fd, (void *)tbuf, nleft); + if (nwritten < 0) { + if (errno == EINTR) { + continue; + } + return -1; + } + nleft -= nwritten; + tbuf += nwritten; + } + + return n; +} + +#ifndef TAOS_OS_FUNC_FILE_SENDIFLE +ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { + size_t leftbytes = size; + ssize_t sentbytes; + + while (leftbytes > 0) { + /* + * TODO : Think to check if file is larger than 1GB + */ + // if (leftbytes > 1000000000) leftbytes = 1000000000; + sentbytes = sendfile(dfd, sfd, offset, leftbytes); + if (sentbytes == -1) { + if (errno == EINTR) { + continue; + } else { + return -1; + } + } else if (sentbytes == 0) { + return (ssize_t)(size - leftbytes); + } + + leftbytes -= sentbytes; + } + + return size; +} #endif \ No newline at end of file diff --git a/src/os/src/detail/osFileOp.c b/src/os/src/detail/osFileOp.c deleted file mode 100644 index 897b6c3f03..0000000000 --- a/src/os/src/detail/osFileOp.c +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -ssize_t taosTReadImp(int fd, void *buf, size_t count) { - size_t leftbytes = count; - ssize_t readbytes; - char * tbuf = (char *)buf; - - while (leftbytes > 0) { - readbytes = read(fd, (void *)tbuf, leftbytes); - if (readbytes < 0) { - if (errno == EINTR) { - continue; - } else { - return -1; - } - } else if (readbytes == 0) { - return (ssize_t)(count - leftbytes); - } - - leftbytes -= readbytes; - tbuf += readbytes; - } - - return (ssize_t)count; -} - -ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { - size_t leftbytes = size; - ssize_t sentbytes; - - while (leftbytes > 0) { - /* - * TODO : Think to check if file is larger than 1GB - */ - //if (leftbytes > 1000000000) leftbytes = 1000000000; - sentbytes = sendfile(dfd, sfd, offset, leftbytes); - if (sentbytes == -1) { - if (errno == EINTR) { - continue; - } - else { - return -1; - } - } else if (sentbytes == 0) { - return (ssize_t)(size - leftbytes); - } - - leftbytes -= sentbytes; - } - - return size; -} - -ssize_t taosTWriteImp(int fd, void *buf, size_t n) { - size_t nleft = n; - ssize_t nwritten = 0; - char *tbuf = (char *)buf; - - while (nleft > 0) { - nwritten = write(fd, (void *)tbuf, nleft); - if (nwritten < 0) { - if (errno == EINTR) { - continue; - } - return -1; - } - nleft -= nwritten; - tbuf += nwritten; - } - - return n; -} diff --git a/src/os/src/detail/osMalloc.c b/src/os/src/detail/osMalloc.c deleted file mode 100644 index 66377827da..0000000000 --- a/src/os/src/detail/osMalloc.c +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" - -#ifndef TAOS_OS_FUNC_MALLOC - -void *tmalloc(size_t size) { - if (size <= 0) return NULL; - - void *ret = malloc(size + sizeof(size_t)); - if (ret == NULL) return NULL; - - *(size_t *)ret = size; - - return (void *)((char *)ret + sizeof(size_t)); -} - -void *tcalloc(size_t nmemb, size_t size) { - size_t tsize = nmemb * size; - void * ret = tmalloc(tsize); - if (ret == NULL) return NULL; - - tmemset(ret, 0); - return ret; -} - -size_t tsizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } - -void tmemset(void *ptr, int c) { memset(ptr, c, tsizeof(ptr)); } - -void * trealloc(void *ptr, size_t size) { - if (ptr == NULL) return tmalloc(size); - - if (size <= tsizeof(ptr)) return ptr; - - void * tptr = (void *)((char *)ptr - sizeof(size_t)); - size_t tsize = size + sizeof(size_t); - tptr = realloc(tptr, tsize); - if (tptr == NULL) return NULL; - - *(size_t *)tptr = size; - - return (void *)((char *)tptr + sizeof(size_t)); -} - -void tzfree(void *ptr) { - if (ptr) { - free((void *)((char *)ptr - sizeof(size_t))); - } -} - -#endif \ No newline at end of file diff --git a/src/os/src/detail/osMem.c b/src/os/src/detail/osMemory.c similarity index 87% rename from src/os/src/detail/osMem.c rename to src/os/src/detail/osMemory.c index 93943c6543..3bbe806369 100644 --- a/src/os/src/detail/osMem.c +++ b/src/os/src/detail/osMemory.c @@ -19,7 +19,7 @@ #ifdef TAOS_MEM_CHECK -static int allocMode = TAOS_ALLOC_MODE_DEFAULT; +static ETaosMemoryAllocMode allocMode = TAOS_ALLOC_MODE_DEFAULT; static FILE* fpAllocLog = NULL; //////////////////////////////////////////////////////////////////////////////// @@ -62,7 +62,7 @@ static void* realloc_random(void* ptr, size_t size, const char* file, uint32_t l static char* strdup_random(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - return random_alloc_fail(len + 1, file, line) ? NULL : strdup(str); + return random_alloc_fail(len + 1, file, line) ? NULL : taosStrdupImp(str); } static char* strndup_random(const char* str, size_t size, const char* file, uint32_t line) { @@ -70,11 +70,11 @@ static char* strndup_random(const char* str, size_t size, const char* file, uint if (len > size) { len = size; } - return random_alloc_fail(len + 1, file, line) ? NULL : strndup(str, len); + return random_alloc_fail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); } static ssize_t getline_random(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - return random_alloc_fail(*n, file, line) ? -1 : getline(lineptr, n, stream); + return random_alloc_fail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); } //////////////////////////////////////////////////////////////////////////////// @@ -242,7 +242,7 @@ static char* strndup_detect_leak(const char* str, size_t size, const char* file, static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { char* buf = NULL; size_t bufSize = 0; - ssize_t size = getline(&buf, &bufSize, stream); + ssize_t size = taosGetlineImp(&buf, &bufSize, stream); if (size != -1) { if (*n < size + 1) { void* p = realloc_detect_leak(*lineptr, size + 1, file, line); @@ -372,7 +372,7 @@ void taos_free(void* ptr, const char* file, uint32_t line) { char* taos_strdup(const char* str, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return strdup(str); + return taosStrdupImp(str); case TAOS_ALLOC_MODE_RANDOM_FAIL: return strdup_random(str, file, line); @@ -380,13 +380,13 @@ char* taos_strdup(const char* str, const char* file, uint32_t line) { case TAOS_ALLOC_MODE_DETECT_LEAK: return strdup_detect_leak(str, file, line); } - return strdup(str); + return taosStrdupImp(str); } char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return strndup(str, size); + return taosStrndupImp(str, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: return strndup_random(str, size, file, line); @@ -394,13 +394,13 @@ char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line case TAOS_ALLOC_MODE_DETECT_LEAK: return strndup_detect_leak(str, size, file, line); } - return strndup(str, size); + return taosStrndupImp(str, size); } ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: - return getline(lineptr, n, stream); + return taosGetlineImp(lineptr, n, stream); case TAOS_ALLOC_MODE_RANDOM_FAIL: return getline_random(lineptr, n, stream, file, line); @@ -408,7 +408,7 @@ ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, case TAOS_ALLOC_MODE_DETECT_LEAK: return getline_detect_leak(lineptr, n, stream, file, line); } - return getline(lineptr, n, stream); + return taosGetlineImp(lineptr, n, stream); } static void close_alloc_log() { @@ -472,3 +472,48 @@ void taosDumpMemoryLeak() { } #endif // TAOS_MEM_CHECK + +void *taosTMalloc(size_t size) { + if (size <= 0) return NULL; + + void *ret = malloc(size + sizeof(size_t)); + if (ret == NULL) return NULL; + + *(size_t *)ret = size; + + return (void *)((char *)ret + sizeof(size_t)); +} + +void *taosTCalloc(size_t nmemb, size_t size) { + size_t tsize = nmemb * size; + void * ret = taosTMalloc(tsize); + if (ret == NULL) return NULL; + + taosTMemset(ret, 0); + return ret; +} + +size_t taosTSizeof(void *ptr) { return (ptr) ? (*(size_t *)((char *)ptr - sizeof(size_t))) : 0; } + +void taosTMemset(void *ptr, int c) { memset(ptr, c, taosTSizeof(ptr)); } + +void * taosTRealloc(void *ptr, size_t size) { + if (ptr == NULL) return taosTMalloc(size); + + if (size <= taosTSizeof(ptr)) return ptr; + + void * tptr = (void *)((char *)ptr - sizeof(size_t)); + size_t tsize = size + sizeof(size_t); + tptr = realloc(tptr, tsize); + if (tptr == NULL) return NULL; + + *(size_t *)tptr = size; + + return (void *)((char *)tptr + sizeof(size_t)); +} + +void taosTZfree(void *ptr) { + if (ptr) { + free((void *)((char *)ptr - sizeof(size_t))); + } +} \ No newline at end of file diff --git a/src/os/src/detail/osRand.c b/src/os/src/detail/osRand.c index 7e5f585634..edb8642bd6 100644 --- a/src/os/src/detail/osRand.c +++ b/src/os/src/detail/osRand.c @@ -20,7 +20,7 @@ uint32_t taosRand(void) { return rand(); } -uint32_t trand(void) { +uint32_t taosSafeRand(void) { int fd; int seed; diff --git a/src/os/src/detail/osPThread.c b/src/os/src/detail/osSemphone.c similarity index 95% rename from src/os/src/detail/osPThread.c rename to src/os/src/detail/osSemphone.c index 18207dc12e..82b916b4d7 100644 --- a/src/os/src/detail/osPThread.c +++ b/src/os/src/detail/osSemphone.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "os.h" -#ifndef TAOS_OS_FUNC_PTHREAD +#ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } int64_t taosGetPthreadId() { return (int64_t)pthread_self(); } diff --git a/src/os/src/detail/osSocket.c b/src/os/src/detail/osSocket.c index ea0b92de5f..7e4031eff9 100644 --- a/src/os/src/detail/osSocket.c +++ b/src/os/src/detail/osSocket.c @@ -39,10 +39,6 @@ int taosSetNonblocking(int sock, int on) { return 0; } -int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) { - return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); -} - void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); @@ -53,4 +49,12 @@ void taosBlockSIGPIPE() { } } +#endif + +#ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT + +int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) { + return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen); +} + #endif \ No newline at end of file diff --git a/src/os/src/detail/osWchar.c b/src/os/src/detail/osString.c similarity index 92% rename from src/os/src/detail/osWchar.c rename to src/os/src/detail/osString.c index 4e02973e50..31ffd6b87c 100644 --- a/src/os/src/detail/osWchar.c +++ b/src/os/src/detail/osString.c @@ -16,14 +16,18 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" -#include "taosdef.h" -#ifndef TAOS_OS_FUNC_WCHAR +#ifndef TAOS_OS_FUNC_STRING_STR2INT64 +int64_t tsosStr2int64(char *str) { + char *endptr = NULL; + return strtoll(str, &endptr, 10); +} +#endif -int tasoUcs4Compare(void* f1_ucs4, void *f2_ucs4, int bytes) { +#ifndef TAOS_OS_FUNC_STRING_WCHAR +int tasoUcs4Compare(void *f1_ucs4, void *f2_ucs4, int bytes) { return wcsncmp((wchar_t *)f1_ucs4, (wchar_t *)f2_ucs4, bytes / TSDB_NCHAR_SIZE); } - #endif #ifdef USE_LIBICONV @@ -120,4 +124,4 @@ char *taosCharsetReplace(char *charsetstr) { } return strdup(charsetstr); -} \ No newline at end of file +} diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 7a395285af..a03199e8d1 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -18,8 +18,6 @@ #include "tconfig.h" #include "tglobal.h" #include "tulog.h" -#include "tsystem.h" -#include "taosdef.h" #ifndef TAOS_OS_FUNC_SYSINFO @@ -63,7 +61,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { size_t len; char * line = NULL; while (!feof(fp)) { - tfree(line); + taosTFree(line); len = 0; getline(&line, &len, fp); if (line == NULL) { @@ -85,7 +83,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - tfree(line); + taosTFree(line); fclose(fp); return true; } @@ -109,7 +107,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { char cpu[10] = {0}; sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - tfree(line); + taosTFree(line); fclose(fp); return true; } @@ -138,7 +136,7 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - tfree(line); + taosTFree(line); fclose(fp); return true; } @@ -378,7 +376,7 @@ static bool taosGetCardInfo(int64_t *bytes) { *bytes += (rbytes + tbytes); } - tfree(line); + taosTFree(line); fclose(fp); return true; @@ -433,7 +431,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { int readIndex = 0; while (!feof(fp)) { - tfree(line); + taosTFree(line); len = 0; getline(&line, &len, fp); if (line == NULL) { @@ -451,7 +449,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { if (readIndex >= 2) break; } - tfree(line); + taosTFree(line); fclose(fp); if (readIndex < 2) { @@ -543,12 +541,12 @@ void taosKillSystem() { kill(tsProcId, 2); } -int tSystem(const char *cmd) { +int taosSystem(const char *cmd) { FILE *fp; int res; char buf[1024]; if (cmd == NULL) { - uError("tSystem cmd is NULL!\n"); + uError("taosSystem cmd is NULL!\n"); return -1; } @@ -570,4 +568,75 @@ int tSystem(const char *cmd) { } } +int _sysctl(struct __sysctl_args *args ); +void taosSetCoreDump() { + if (0 == tsEnableCoreFile) { + return; + } + + // 1. set ulimit -c unlimited + struct rlimit rlim; + struct rlimit rlim_new; + if (getrlimit(RLIMIT_CORE, &rlim) == 0) { + uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + rlim_new.rlim_cur = RLIM_INFINITY; + rlim_new.rlim_max = RLIM_INFINITY; + if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) { + uInfo("set unlimited fail, error: %s", strerror(errno)); + rlim_new.rlim_cur = rlim.rlim_max; + rlim_new.rlim_max = rlim.rlim_max; + (void)setrlimit(RLIMIT_CORE, &rlim_new); + } + } + + if (getrlimit(RLIMIT_CORE, &rlim) == 0) { + uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max); + } + +#ifndef _TD_ARM_ + // 2. set the path for saving core file + struct __sysctl_args args; + int old_usespid = 0; + size_t old_len = 0; + int new_usespid = 1; + size_t new_len = sizeof(new_usespid); + + int name[] = {CTL_KERN, KERN_CORE_USES_PID}; + + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name)/sizeof(name[0]); + args.oldval = &old_usespid; + args.oldlenp = &old_len; + args.newval = &new_usespid; + args.newlen = new_len; + + old_len = sizeof(old_usespid); + + if (syscall(SYS__sysctl, &args) == -1) { + uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno)); + } + + uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); + + + old_usespid = 0; + old_len = 0; + memset(&args, 0, sizeof(struct __sysctl_args)); + args.name = name; + args.nlen = sizeof(name)/sizeof(name[0]); + args.oldval = &old_usespid; + args.oldlenp = &old_len; + + old_len = sizeof(old_usespid); + + if (syscall(SYS__sysctl, &args) == -1) { + uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno)); + } + + uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid); +#endif + +} + #endif \ No newline at end of file diff --git a/src/util/src/ttime.c b/src/os/src/detail/osTime.c similarity index 99% rename from src/util/src/ttime.c rename to src/os/src/detail/osTime.c index 018c0d640b..6d41692d80 100644 --- a/src/util/src/ttime.c +++ b/src/os/src/detail/osTime.c @@ -17,13 +17,10 @@ #define _XOPEN_SOURCE #define _DEFAULT_SOURCE -#include -#include -#include - +#include "os.h" #include "taosdef.h" -#include "ttime.h" #include "tutil.h" + /* * mktime64 - Converts date to seconds. * Converts Gregorian date to seconds since 1970-01-01 00:00:00. diff --git a/src/os/src/detail/osTimer.c b/src/os/src/detail/osTimer.c index e0a2e90314..22f7b94c3a 100644 --- a/src/os/src/detail/osTimer.c +++ b/src/os/src/detail/osTimer.c @@ -15,42 +15,11 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "taosdef.h" -#include "tglobal.h" #include "ttimer.h" #include "tulog.h" #ifndef TAOS_OS_FUNC_TIMER -/* - to make taosMsleep work, - signal SIGALRM shall be blocked in the calling thread, - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGALRM); - pthread_sigmask(SIG_BLOCK, &set, NULL); -*/ -void taosMsleep(int mseconds) { - struct timeval timeout; - int seconds, useconds; - - seconds = mseconds / 1000; - useconds = (mseconds % 1000) * 1000; - timeout.tv_sec = seconds; - timeout.tv_usec = useconds; - - /* sigset_t set; */ - /* sigemptyset(&set); */ - /* sigaddset(&set, SIGALRM); */ - /* pthread_sigmask(SIG_BLOCK, &set, NULL); */ - - select(0, NULL, NULL, NULL, &timeout); - - /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ -} - - static void taosDeleteTimer(void *tharg) { timer_t *pTimer = tharg; timer_delete(*pTimer); @@ -129,4 +98,35 @@ void taosUninitTimer() { pthread_join(timerThread, NULL); } +#endif + +#ifndef TAOS_OS_FUNC_TIMER_SLEEP +/* + to make taosMsleep work, + signal SIGALRM shall be blocked in the calling thread, + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGALRM); + pthread_sigmask(SIG_BLOCK, &set, NULL); +*/ +void taosMsleep(int mseconds) { + struct timeval timeout; + int seconds, useconds; + + seconds = mseconds / 1000; + useconds = (mseconds % 1000) * 1000; + timeout.tv_sec = seconds; + timeout.tv_usec = useconds; + + /* sigset_t set; */ + /* sigemptyset(&set); */ + /* sigaddset(&set, SIGALRM); */ + /* pthread_sigmask(SIG_BLOCK, &set, NULL); */ + + select(0, NULL, NULL, NULL, &timeout); + + /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */ +} + #endif \ No newline at end of file diff --git a/src/os/src/linux64/CMakeLists.txt b/src/os/src/linux64/CMakeLists.txt index 79cbf917e2..0c577374ed 100644 --- a/src/os/src/linux64/CMakeLists.txt +++ b/src/os/src/linux64/CMakeLists.txt @@ -9,4 +9,3 @@ AUX_SOURCE_DIRECTORY(. SRC) ADD_LIBRARY(os ${SRC}) TARGET_LINK_LIBRARIES(os m rt) - diff --git a/src/os/src/linux64/linuxEnv.c b/src/os/src/linux64/linuxEnv.c index 0a9d81311a..4f84412075 100644 --- a/src/os/src/linux64/linuxEnv.c +++ b/src/os/src/linux64/linuxEnv.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" -#include "tulog.h" void osInit() { strcpy(configDir, "/etc/taos"); diff --git a/src/os/src/windows/CMakeLists.txt b/src/os/src/windows/CMakeLists.txt new file mode 100644 index 0000000000..60fab63cd1 --- /dev/null +++ b/src/os/src/windows/CMakeLists.txt @@ -0,0 +1,15 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.8) +PROJECT(TDengine) + +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc) +INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc) +AUX_SOURCE_DIRECTORY(. SRC) + +ADD_LIBRARY(os ${SRC}) + +TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32) diff --git a/src/os/src/windows/w64Atomic.c b/src/os/src/windows/w64Atomic.c new file mode 100644 index 0000000000..0425f4ed3f --- /dev/null +++ b/src/os/src/windows/w64Atomic.c @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "tglobal.h" +#include "ttimer.h" +#include "tulog.h" +#include "tutil.h" + +// add +char interlocked_add_fetch_8(char volatile* ptr, char val) { + #ifdef _TD_GO_DLL_ + return __sync_fetch_and_add(ptr, val) + val; + #else + return _InterlockedExchangeAdd8(ptr, val) + val; + #endif +} + +short interlocked_add_fetch_16(short volatile* ptr, short val) { + #ifdef _TD_GO_DLL_ + return __sync_fetch_and_add(ptr, val) + val; + #else + return _InterlockedExchangeAdd16(ptr, val) + val; + #endif +} + +long interlocked_add_fetch_32(long volatile* ptr, long val) { + return _InterlockedExchangeAdd(ptr, val) + val; +} + +__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { + return _InterlockedExchangeAdd64(ptr, val) + val; +} + +// and +#ifndef _TD_GO_DLL_ +char interlocked_and_fetch_8(char volatile* ptr, char val) { + return _InterlockedAnd8(ptr, val) & val; +} + +short interlocked_and_fetch_16(short volatile* ptr, short val) { + return _InterlockedAnd16(ptr, val) & val; +} +#endif + +long interlocked_and_fetch_32(long volatile* ptr, long val) { + return _InterlockedAnd(ptr, val) & val; +} + +#ifndef _M_IX86 + +__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { + return _InterlockedAnd64(ptr, val) & val; +} + +#else + +__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { + __int64 old, res; + do { + old = *ptr; + res = old & val; + } while(_InterlockedCompareExchange64(ptr, res, old) != old); + return res; +} + +__int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val) { + __int64 old; + do { + old = *ptr; + } while(_InterlockedCompareExchange64(ptr, old & val, old) != old); + return old; +} + +#endif + +// or +#ifndef _TD_GO_DLL_ +char interlocked_or_fetch_8(char volatile* ptr, char val) { + return _InterlockedOr8(ptr, val) | val; +} + +short interlocked_or_fetch_16(short volatile* ptr, short val) { + return _InterlockedOr16(ptr, val) | val; +} +#endif +long interlocked_or_fetch_32(long volatile* ptr, long val) { + return _InterlockedOr(ptr, val) | val; +} + +#ifndef _M_IX86 + +__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { + return _InterlockedOr64(ptr, val) & val; +} + +#else + +__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { + __int64 old, res; + do { + old = *ptr; + res = old | val; + } while(_InterlockedCompareExchange64(ptr, res, old) != old); + return res; +} + +__int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val) { + __int64 old; + do { + old = *ptr; + } while(_InterlockedCompareExchange64(ptr, old | val, old) != old); + return old; +} + +#endif + +// xor +#ifndef _TD_GO_DLL_ +char interlocked_xor_fetch_8(char volatile* ptr, char val) { + return _InterlockedXor8(ptr, val) ^ val; +} + +short interlocked_xor_fetch_16(short volatile* ptr, short val) { + return _InterlockedXor16(ptr, val) ^ val; +} +#endif +long interlocked_xor_fetch_32(long volatile* ptr, long val) { + return _InterlockedXor(ptr, val) ^ val; +} + +#ifndef _M_IX86 + +__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { + return _InterlockedXor64(ptr, val) ^ val; +} + +#else + +__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { + __int64 old, res; + do { + old = *ptr; + res = old ^ val; + } while(_InterlockedCompareExchange64(ptr, res, old) != old); + return res; +} + +__int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { + __int64 old; + do { + old = *ptr; + } while(_InterlockedCompareExchange64(ptr, old ^ val, old) != old); + return old; +} + +#endif + diff --git a/src/os/src/windows64/twinenv.c b/src/os/src/windows/w64Env.c similarity index 100% rename from src/os/src/windows64/twinenv.c rename to src/os/src/windows/w64Env.c diff --git a/src/os/src/windows/w64File.c b/src/os/src/windows/w64File.c new file mode 100644 index 0000000000..f2c59c3639 --- /dev/null +++ b/src/os/src/windows/w64File.c @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" + +void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) { + const char* tdengineTmpFileNamePrefix = "tdengine-"; + char tmpPath[PATH_MAX]; + + char *tmpDir = getenv("tmp"); + if (tmpDir == NULL) { + tmpDir = ""; + } + + strcpy(tmpPath, tmpDir); + strcat(tmpPath, tdengineTmpFileNamePrefix); + if (strlen(tmpPath) + strlen(fileNamePrefix) + strlen("-%d-%s") < PATH_MAX) { + strcat(tmpPath, fileNamePrefix); + strcat(tmpPath, "-%d-%s"); + } + + char rand[8] = {0}; + taosRandStr(rand, tListLen(rand) - 1); + snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand); +} + +#define _SEND_FILE_STEP_ 1000 + +int taosFSendFileImp(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { + fseek(in_file, (int32_t)(*offset), 0); + int writeLen = 0; + uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; + + for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { + size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); + if (rlen <= 0) { + return writeLen; + } + else if (rlen < _SEND_FILE_STEP_) { + fwrite(buffer, 1, rlen, out_file); + return (int)(writeLen + rlen); + } + else { + fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); + writeLen += _SEND_FILE_STEP_; + } + } + + int remain = count - writeLen; + if (remain > 0) { + size_t rlen = fread(buffer, 1, remain, in_file); + if (rlen <= 0) { + return writeLen; + } + else { + fwrite(buffer, 1, remain, out_file); + writeLen += remain; + } + } + + return writeLen; +} + +ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) { + uError("taosTSendFileImp no implemented yet"); + return 0; +} \ No newline at end of file diff --git a/src/os/src/windows/w64Getline.c b/src/os/src/windows/w64Getline.c new file mode 100644 index 0000000000..3e8701e19b --- /dev/null +++ b/src/os/src/windows/w64Getline.c @@ -0,0 +1,124 @@ +/* getline.c -- Replacement for GNU C library function getline + +Copyright (C) 1993 Free Software Foundation, Inc. + +This program is free software; you can redistribute it and/or +modify it under the terms of the GNU General Public License as +published by the Free Software Foundation; either version 2 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. */ + +/* Written by Jan Brittenson, bson@gnu.ai.mit.edu. */ + +#ifdef HAVE_CONFIG_H +#include +#endif + +#include +#include +#include +#include + +#if STDC_HEADERS +#include +#else +char *malloc(), *realloc(); +#endif + +/* Always add at least this many bytes when extending the buffer. */ +#define MIN_CHUNK 64 + +/* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR ++ OFFSET (and null-terminate it). *LINEPTR is a pointer returned from +malloc (or NULL), pointing to *N characters of space. It is realloc'd +as necessary. Return the number of characters read (not including the +null terminator), or -1 on error or EOF. On a -1 return, the caller +should check feof(), if not then errno has been set to indicate +the error. */ + +int getstr(char **lineptr, size_t *n, FILE *stream, char terminator, int offset) { + int nchars_avail; /* Allocated but unused chars in *LINEPTR. */ + char *read_pos; /* Where we're reading into *LINEPTR. */ + int ret; + + if (!lineptr || !n || !stream) { + errno = EINVAL; + return -1; + } + + if (!*lineptr) { + *n = MIN_CHUNK; + *lineptr = malloc(*n); + if (!*lineptr) { + errno = ENOMEM; + return -1; + } + } + + nchars_avail = (int)(*n - offset); + read_pos = *lineptr + offset; + + for (;;) { + int save_errno; + register int c = getc(stream); + + save_errno = errno; + + /* We always want at least one char left in the buffer, since we + always (unless we get an error while reading the first char) + NUL-terminate the line buffer. */ + + assert((*lineptr + *n) == (read_pos + nchars_avail)); + if (nchars_avail < 2) { + if (*n > MIN_CHUNK) + *n *= 2; + else + *n += MIN_CHUNK; + + nchars_avail = (int)(*n + *lineptr - read_pos); + *lineptr = realloc(*lineptr, *n); + if (!*lineptr) { + errno = ENOMEM; + return -1; + } + read_pos = *n - nchars_avail + *lineptr; + assert((*lineptr + *n) == (read_pos + nchars_avail)); + } + + if (ferror(stream)) { + /* Might like to return partial line, but there is no + place for us to store errno. And we don't want to just + lose errno. */ + errno = save_errno; + return -1; + } + + if (c == EOF) { + /* Return partial line, if any. */ + if (read_pos == *lineptr) + return -1; + else + break; + } + + *read_pos++ = c; + nchars_avail--; + + if (c == terminator) /* Return the line. */ + break; + } + + /* Done - NUL terminate and return the number of chars read. */ + *read_pos = '\0'; + + ret = (int)(read_pos - (*lineptr + offset)); + return ret; +} + +int taosGetlineImp(char **lineptr, size_t *n, FILE *stream) { + return getstr(lineptr, n, stream, '\n', 0); +} \ No newline at end of file diff --git a/src/os/src/windows/w64Godll.c b/src/os/src/windows/w64Godll.c new file mode 100644 index 0000000000..b270dab2f4 --- /dev/null +++ b/src/os/src/windows/w64Godll.c @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "tglobal.h" +#include "ttimer.h" +#include "tulog.h" +#include "tutil.h" + +#ifdef _TD_GO_DLL_ +int64_t tsosStr2int64(char *str) { + char *endptr = NULL; + return strtoll(str, &endptr, 10); +} + +uint64_t htonll(uint64_t val) { + return (((uint64_t) htonl(val)) << 32) + htonl(val >> 32); +} +#endif diff --git a/src/os/src/windows/w64Lz4.c b/src/os/src/windows/w64Lz4.c new file mode 100644 index 0000000000..96556c1f1c --- /dev/null +++ b/src/os/src/windows/w64Lz4.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "tglobal.h" +#include "ttimer.h" +#include "tulog.h" +#include "tutil.h" + +int32_t BUILDIN_CLZL(uint64_t val) { + unsigned long r = 0; + _BitScanReverse64(&r, val); + return (int)(r >> 3); +} + +int32_t BUILDIN_CLZ(uint32_t val) { + unsigned long r = 0; + _BitScanReverse(&r, val); + return (int)(r >> 3); +} + +int32_t BUILDIN_CTZL(uint64_t val) { + unsigned long r = 0; + _BitScanForward64(&r, val); + return (int)(r >> 3); +} + +int32_t BUILDIN_CTZ(uint32_t val) { + unsigned long r = 0; + _BitScanForward(&r, val); + return (int)(r >> 3); +} diff --git a/src/os/src/windows/w64Semphone.c b/src/os/src/windows/w64Semphone.c new file mode 100644 index 0000000000..7ed6228228 --- /dev/null +++ b/src/os/src/windows/w64Semphone.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "tglobal.h" +#include "ttimer.h" +#include "tulog.h" +#include "tutil.h" + +bool taosCheckPthreadValid(pthread_t thread) { return thread.p != NULL; } + +void taosResetPthread(pthread_t *thread) { thread->p = 0; } + +int64_t taosGetPthreadId() { +#ifdef PTW32_VERSION + return pthread_getw32threadid_np(pthread_self()); +#else + return (int64_t)pthread_self(); +#endif +} diff --git a/src/os/src/windows/w64Socket.c b/src/os/src/windows/w64Socket.c new file mode 100644 index 0000000000..cdf46825e8 --- /dev/null +++ b/src/os/src/windows/w64Socket.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include + +void taosWinSocketInit() { + static char flag = 0; + if (flag == 0) { + WORD wVersionRequested; + WSADATA wsaData; + wVersionRequested = MAKEWORD(1, 1); + if (WSAStartup(wVersionRequested, &wsaData) == 0) { + flag = 1; + } + } +} + +int taosSetNonblocking(SOCKET sock, int on) { + u_long mode; + if (on) { + mode = 1; + ioctlsocket(sock, FIONBIO, &mode); + } else { + mode = 0; + ioctlsocket(sock, FIONBIO, &mode); + } + return 0; +} + +void taosBlockSIGPIPE() {} + +int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) { + if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { + return 0; + } + + if (level == SOL_TCP && optname == TCP_KEEPIDLE) { + return 0; + } + + if (level == SOL_TCP && optname == TCP_KEEPINTVL) { + return 0; + } + + return setsockopt(socketfd, level, optname, optval, optlen); +} \ No newline at end of file diff --git a/src/os/src/windows64/w64Wchar.c b/src/os/src/windows/w64String.c similarity index 57% rename from src/os/src/windows64/w64Wchar.c rename to src/os/src/windows/w64String.c index d5930d1de9..8057c89a55 100644 --- a/src/os/src/windows64/w64Wchar.c +++ b/src/os/src/windows/w64String.c @@ -15,8 +15,68 @@ #define _DEFAULT_SOURCE #include "os.h" +#include "taosdef.h" #include "tglobal.h" +#include "ttimer.h" #include "tulog.h" +#include "tutil.h" + +/* + * Get next token from string *stringp, where tokens are possibly-empty + * strings separated by characters from delim. + * + * Writes NULs into the string at *stringp to end tokens. + * delim need not remain constant from call to call. + * On return, *stringp points past the last NUL written (if there might + * be further tokens), or is NULL (if there are definitely no moretokens). + * + * If *stringp is NULL, strsep returns NULL. + */ +char *strsep(char **stringp, const char *delim) { + char *s; + const char *spanp; + int c, sc; + char *tok; + if ((s = *stringp) == NULL) + return (NULL); + for (tok = s;;) { + c = *s++; + spanp = delim; + do { + if ((sc = *spanp++) == c) { + if (c == 0) + s = NULL; + else + s[-1] = 0; + *stringp = s; + return (tok); + } + } while (sc != 0); + } + /* NOTREACHED */ +} + +char *getpass(const char *prefix) { + static char passwd[TSDB_KEY_LEN] = {0}; + + printf("%s", prefix); + scanf("%s", passwd); + + char n = getchar(); + return passwd; +} + +char *strndup(const char *s, size_t n) { + int len = strlen(s); + if (len >= n) { + len = n; + } + + char *r = calloc(len + 1, 1); + memcpy(r, s, len); + r[len] = 0; + return r; +} size_t twcslen(const wchar_t *wcs) { int *wstr = (int *)wcs; diff --git a/src/os/src/windows64/twinstrptime.c b/src/os/src/windows/w64Strptime.c similarity index 100% rename from src/os/src/windows64/twinstrptime.c rename to src/os/src/windows/w64Strptime.c diff --git a/src/os/src/windows/w64Sysinfo.c b/src/os/src/windows/w64Sysinfo.c new file mode 100644 index 0000000000..0a680ac0b7 --- /dev/null +++ b/src/os/src/windows/w64Sysinfo.c @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "tglobal.h" +#include "tconfig.h" +#include "ttimer.h" +#include "tulog.h" +#include "tutil.h" + +static void taosGetSystemTimezone() { + // get and set default timezone + SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); + if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { + char *tz = getenv("TZ"); + if (tz == NULL || strlen(tz) == 0) { + strcpy(tsTimezone, "not configured"); + } else { + strcpy(tsTimezone, tz); + } + cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uInfo("timezone not configured, use default"); + } +} + +static void taosGetSystemLocale() { + // get and set default locale + SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); + if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { + char *locale = setlocale(LC_CTYPE, "chs"); + if (locale != NULL) { + tstrncpy(tsLocale, locale, sizeof(tsLocale)); + cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uInfo("locale not configured, set to default:%s", tsLocale); + } + } + + SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); + if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { + strcpy(tsCharset, "cp936"); + cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; + uInfo("charset not configured, set to default:%s", tsCharset); + } +} + +void taosPrintOsInfo() {} + +void taosKillSystem() { + uError("function taosKillSystem, exit!"); + exit(0); +} + +void taosGetSystemInfo() { + taosGetSystemTimezone(); + taosGetSystemLocale(); +} + +bool taosGetDisk() { return true; } + +bool taosGetProcIO(float *readKB, float *writeKB) { + *readKB = 0; + *writeKB = 0; + return true; +} + +bool taosGetBandSpeed(float *bandSpeedKb) { + *bandSpeedKb = 0; + return true; +} + +bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) { + *sysCpuUsage = 0; + *procCpuUsage = 0; + return true; +} + +bool taosGetProcMemory(float *memoryUsedMB) { + *memoryUsedMB = 0; + return true; +} + +bool taosGetSysMemory(float *memoryUsedMB) { + *memoryUsedMB = 0; + return true; +} + +int taosSystem(const char *cmd) { + uError("taosSystem not support"); + return -1; +} + +int flock(int fd, int option) { + return 0; +} + +int fsync(int filedes) { + return 0; +} + +int sigaction(int sig, struct sigaction *d, void *p) { + return 0; +} + +void taosSetCoreDump() {} \ No newline at end of file diff --git a/src/os/src/windows64/twintime.c b/src/os/src/windows/w64Time.c similarity index 100% rename from src/os/src/windows64/twintime.c rename to src/os/src/windows/w64Time.c diff --git a/src/os/src/windows64/twintimer.c b/src/os/src/windows/w64Timer.c similarity index 93% rename from src/os/src/windows64/twintimer.c rename to src/os/src/windows/w64Timer.c index 2bb8478f09..32e04746f1 100644 --- a/src/os/src/windows64/twintimer.c +++ b/src/os/src/windows/w64Timer.c @@ -22,8 +22,7 @@ typedef void (*win_timer_f)(int signo); -void WINAPI taosWinOnTimer(UINT wTimerID, UINT msg, DWORD_PTR dwUser, DWORD_PTR dwl, DWORD_PTR dw2) -{ +void WINAPI taosWinOnTimer(UINT wTimerID, UINT msg, DWORD_PTR dwUser, DWORD_PTR dwl, DWORD_PTR dw2) { win_timer_f callback = *((win_timer_f *)&dwUser); if (callback != NULL) { callback(0); @@ -48,7 +47,3 @@ void taosUninitTimer() { void taosMsleep(int mseconds) { Sleep(mseconds); } - -void sleep(int mseconds) { - taosMsleep(mseconds); -} \ No newline at end of file diff --git a/src/os/src/windows64/twintcpserver.c b/src/os/src/windows/w64Wordexp.c similarity index 55% rename from src/os/src/windows64/twintcpserver.c rename to src/os/src/windows/w64Wordexp.c index 10c0348bef..bb9acde25a 100644 --- a/src/os/src/windows64/twintcpserver.c +++ b/src/os/src/windows/w64Wordexp.c @@ -13,22 +13,21 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE +#include "os.h" +#include "taosdef.h" +#include "tglobal.h" +#include "ttimer.h" #include "tulog.h" +#include "tutil.h" -void taosCloseTcpServerConnection(void *chandle) { - tError("CloseTcpServerConnection not support in windows"); -} - -void taosCleanUpTcpServer(void *handle) { - tError("CleanUpTcpServer not support in windows"); -} - -void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { - tError("InitTcpServer not support in windows"); +int wordexp(const char *words, wordexp_t *pwordexp, int flags) { + pwordexp->we_offs = 0; + pwordexp->we_wordc = 1; + pwordexp->we_wordv = (char **)(pwordexp->wordPos); + pwordexp->we_wordv[0] = (char *)words; return 0; } -int taosSendTcpServerData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) { - tError("SendTcpServerData not support in windows"); - return 0; -} +void wordfree(wordexp_t *pwordexp) {} + diff --git a/src/os/src/windows64/CMakeLists.txt b/src/os/src/windows64/CMakeLists.txt deleted file mode 100644 index dc60b736ea..0000000000 --- a/src/os/src/windows64/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -CMAKE_MINIMUM_REQUIRED(VERSION 2.8) -PROJECT(TDengine) - -IF (TD_WINDOWS_64) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) - INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) - INCLUDE_DIRECTORIES(inc) - AUX_SOURCE_DIRECTORY(src SRC) - ADD_LIBRARY(os ${SRC}) - TARGET_LINK_LIBRARIES(os winmm IPHLPAPI ws2_32) -ENDIF () diff --git a/src/os/src/windows64/twindows.c b/src/os/src/windows64/twindows.c deleted file mode 100644 index 4284dcb14e..0000000000 --- a/src/os/src/windows64/twindows.c +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taosdef.h" -#include "tglobal.h" -#include "ttimer.h" -#include "tulog.h" -#include "tutil.h" - -bool taosCheckPthreadValid(pthread_t thread) { - return thread.p != NULL; -} - -void taosResetPthread(pthread_t *thread) { - thread->p = 0; -} - -int64_t taosGetPthreadId() { -#ifdef PTW32_VERSION - return pthread_getw32threadid_np(pthread_self()); -#else - return (int64_t)pthread_self(); -#endif -} - -int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) { - if (level == SOL_SOCKET && optname == TCP_KEEPCNT) { - return 0; - } - - if (level == SOL_TCP && optname == TCP_KEEPIDLE) { - return 0; - } - - if (level == SOL_TCP && optname == TCP_KEEPINTVL) { - return 0; - } - - return setsockopt(socketfd, level, optname, optval, optlen); -} - -// add -char interlocked_add_fetch_8(char volatile* ptr, char val) { - #ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; - #else - return _InterlockedExchangeAdd8(ptr, val) + val; - #endif -} - -short interlocked_add_fetch_16(short volatile* ptr, short val) { - #ifdef _TD_GO_DLL_ - return __sync_fetch_and_add(ptr, val) + val; - #else - return _InterlockedExchangeAdd16(ptr, val) + val; - #endif -} - -long interlocked_add_fetch_32(long volatile* ptr, long val) { - return _InterlockedExchangeAdd(ptr, val) + val; -} - -__int64 interlocked_add_fetch_64(__int64 volatile* ptr, __int64 val) { - return _InterlockedExchangeAdd64(ptr, val) + val; -} - -// and -#ifndef _TD_GO_DLL_ -char interlocked_and_fetch_8(char volatile* ptr, char val) { - return _InterlockedAnd8(ptr, val) & val; -} - -short interlocked_and_fetch_16(short volatile* ptr, short val) { - return _InterlockedAnd16(ptr, val) & val; -} -#endif - -long interlocked_and_fetch_32(long volatile* ptr, long val) { - return _InterlockedAnd(ptr, val) & val; -} - -#ifndef _M_IX86 - -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { - return _InterlockedAnd64(ptr, val) & val; -} - -#else - -__int64 interlocked_and_fetch_64(__int64 volatile* ptr, __int64 val) { - __int64 old, res; - do { - old = *ptr; - res = old & val; - } while(_InterlockedCompareExchange64(ptr, res, old) != old); - return res; -} - -__int64 interlocked_fetch_and_64(__int64 volatile* ptr, __int64 val) { - __int64 old; - do { - old = *ptr; - } while(_InterlockedCompareExchange64(ptr, old & val, old) != old); - return old; -} - -#endif - -// or -#ifndef _TD_GO_DLL_ -char interlocked_or_fetch_8(char volatile* ptr, char val) { - return _InterlockedOr8(ptr, val) | val; -} - -short interlocked_or_fetch_16(short volatile* ptr, short val) { - return _InterlockedOr16(ptr, val) | val; -} -#endif -long interlocked_or_fetch_32(long volatile* ptr, long val) { - return _InterlockedOr(ptr, val) | val; -} - -#ifndef _M_IX86 - -__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { - return _InterlockedOr64(ptr, val) & val; -} - -#else - -__int64 interlocked_or_fetch_64(__int64 volatile* ptr, __int64 val) { - __int64 old, res; - do { - old = *ptr; - res = old | val; - } while(_InterlockedCompareExchange64(ptr, res, old) != old); - return res; -} - -__int64 interlocked_fetch_or_64(__int64 volatile* ptr, __int64 val) { - __int64 old; - do { - old = *ptr; - } while(_InterlockedCompareExchange64(ptr, old | val, old) != old); - return old; -} - -#endif - -// xor -#ifndef _TD_GO_DLL_ -char interlocked_xor_fetch_8(char volatile* ptr, char val) { - return _InterlockedXor8(ptr, val) ^ val; -} - -short interlocked_xor_fetch_16(short volatile* ptr, short val) { - return _InterlockedXor16(ptr, val) ^ val; -} -#endif -long interlocked_xor_fetch_32(long volatile* ptr, long val) { - return _InterlockedXor(ptr, val) ^ val; -} - -#ifndef _M_IX86 - -__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { - return _InterlockedXor64(ptr, val) ^ val; -} - -#else - -__int64 interlocked_xor_fetch_64(__int64 volatile* ptr, __int64 val) { - __int64 old, res; - do { - old = *ptr; - res = old ^ val; - } while(_InterlockedCompareExchange64(ptr, res, old) != old); - return res; -} - -__int64 interlocked_fetch_xor_64(__int64 volatile* ptr, __int64 val) { - __int64 old; - do { - old = *ptr; - } while(_InterlockedCompareExchange64(ptr, old ^ val, old) != old); - return old; -} - -#endif - -void taosPrintOsInfo() {} - -void taosGetSystemTimezone() { - // get and set default timezone - SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone"); - if (cfg_timezone && cfg_timezone->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *tz = getenv("TZ"); - if (tz == NULL || strlen(tz) == 0) { - strcpy(tsTimezone, "not configured"); - } - else { - strcpy(tsTimezone, tz); - } - cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("timezone not configured, use default"); - } -} - -void taosGetSystemLocale() { - // get and set default locale - SGlobalCfg *cfg_locale = taosGetConfigOption("locale"); - if (cfg_locale && cfg_locale->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - char *locale = setlocale(LC_CTYPE, "chs"); - if (locale != NULL) { - tstrncpy(tsLocale, locale, sizeof(tsLocale)); - cfg_locale->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("locale not configured, set to default:%s", tsLocale); - } - } - - SGlobalCfg *cfg_charset = taosGetConfigOption("charset"); - if (cfg_charset && cfg_charset->cfgStatus < TAOS_CFG_CSTATUS_DEFAULT) { - strcpy(tsCharset, "cp936"); - cfg_charset->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT; - uInfo("charset not configured, set to default:%s", tsCharset); - } -} - -void taosGetSystemInfo() { - taosGetSystemTimezone(); - taosGetSystemLocale(); -} - -void taosKillSystem() { - exit(0); -} - -/* - * Get next token from string *stringp, where tokens are possibly-empty - * strings separated by characters from delim. - * - * Writes NULs into the string at *stringp to end tokens. - * delim need not remain constant from call to call. - * On return, *stringp points past the last NUL written (if there might - * be further tokens), or is NULL (if there are definitely no moretokens). - * - * If *stringp is NULL, strsep returns NULL. - */ -char *strsep(char **stringp, const char *delim) { - char *s; - const char *spanp; - int c, sc; - char *tok; - if ((s = *stringp) == NULL) - return (NULL); - for (tok = s;;) { - c = *s++; - spanp = delim; - do { - if ((sc = *spanp++) == c) { - if (c == 0) - s = NULL; - else - s[-1] = 0; - *stringp = s; - return (tok); - } - } while (sc != 0); - } - /* NOTREACHED */ -} - -char *getpass(const char *prefix) { - static char passwd[TSDB_KEY_LEN] = {0}; - - printf("%s", prefix); - scanf("%s", passwd); - - char n = getchar(); - return passwd; -} - -int flock(int fd, int option) { - return 0; -} - -int fsync(int filedes) { - return 0; -} - -int sigaction(int sig, struct sigaction *d, void *p) { - return 0; -} - -int wordexp(const char *words, wordexp_t *pwordexp, int flags) { - pwordexp->we_offs = 0; - pwordexp->we_wordc = 1; - pwordexp->we_wordv = (char **)(pwordexp->wordPos); - pwordexp->we_wordv[0] = (char *)words; - return 0; -} - -void wordfree(wordexp_t *pwordexp) {} - -void taosGetDisk() {} - -bool taosSkipSocketCheck() { - return false; -} - -#define _SEND_FILE_STEP_ 1000 - -int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count) { - fseek(in_file, (int32_t)(*offset), 0); - int writeLen = 0; - uint8_t buffer[_SEND_FILE_STEP_] = { 0 }; - - for (int len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) { - size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file); - if (rlen <= 0) { - return writeLen; - } - else if (rlen < _SEND_FILE_STEP_) { - fwrite(buffer, 1, rlen, out_file); - return (int)(writeLen + rlen); - } - else { - fwrite(buffer, 1, _SEND_FILE_STEP_, in_file); - writeLen += _SEND_FILE_STEP_; - } - } - - int remain = count - writeLen; - if (remain > 0) { - size_t rlen = fread(buffer, 1, remain, in_file); - if (rlen <= 0) { - return writeLen; - } - else { - fwrite(buffer, 1, remain, out_file); - writeLen += remain; - } - } - - return writeLen; -} - -int32_t BUILDIN_CLZL(uint64_t val) { - unsigned long r = 0; - _BitScanReverse64(&r, val); - return (int)(r >> 3); -} - -int32_t BUILDIN_CLZ(uint32_t val) { - unsigned long r = 0; - _BitScanReverse(&r, val); - return (int)(r >> 3); -} - -int32_t BUILDIN_CTZL(uint64_t val) { - unsigned long r = 0; - _BitScanForward64(&r, val); - return (int)(r >> 3); -} - -int32_t BUILDIN_CTZ(uint32_t val) { - unsigned long r = 0; - _BitScanForward(&r, val); - return (int)(r >> 3); -} - -char *strndup(const char *s, size_t n) { - int len = strlen(s); - if (len >= n) { - len = n; - } - - char *r = calloc(len + 1, 1); - memcpy(r, s, len); - r[len] = 0; - return r; -} - -void taosSetCoreDump() {} - -#ifdef _TD_GO_DLL_ -int64_t tsosStr2int64(char *str) { - char *endptr = NULL; - return strtoll(str, &endptr, 10); -} - -uint64_t htonll(uint64_t val) -{ - return (((uint64_t) htonl(val)) << 32) + htonl(val >> 32); -} -#endif \ No newline at end of file diff --git a/src/os/src/windows64/twingetline.c b/src/os/src/windows64/twingetline.c deleted file mode 100644 index 1dd56fd547..0000000000 --- a/src/os/src/windows64/twingetline.c +++ /dev/null @@ -1,145 +0,0 @@ -/* getline.c -- Replacement for GNU C library function getline - -Copyright (C) 1993 Free Software Foundation, Inc. - -This program is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License as -published by the Free Software Foundation; either version 2 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -General Public License for more details. */ - -/* Written by Jan Brittenson, bson@gnu.ai.mit.edu. */ - -#ifdef HAVE_CONFIG_H -#include -#endif - -#include -#include -#include -#include - -#if STDC_HEADERS -#include -#else -char *malloc(), *realloc(); -#endif - -/* Always add at least this many bytes when extending the buffer. */ -#define MIN_CHUNK 64 - -/* Read up to (and including) a TERMINATOR from STREAM into *LINEPTR -+ OFFSET (and null-terminate it). *LINEPTR is a pointer returned from -malloc (or NULL), pointing to *N characters of space. It is realloc'd -as necessary. Return the number of characters read (not including the -null terminator), or -1 on error or EOF. On a -1 return, the caller -should check feof(), if not then errno has been set to indicate -the error. */ - -int -getstr(lineptr, n, stream, terminator, offset) -char **lineptr; -size_t *n; -FILE *stream; -char terminator; -int offset; -{ - int nchars_avail; /* Allocated but unused chars in *LINEPTR. */ - char *read_pos; /* Where we're reading into *LINEPTR. */ - int ret; - - if (!lineptr || !n || !stream) - { - errno = EINVAL; - return -1; - } - - if (!*lineptr) - { - *n = MIN_CHUNK; - *lineptr = malloc(*n); - if (!*lineptr) - { - errno = ENOMEM; - return -1; - } - } - - nchars_avail = (int)(*n - offset); - read_pos = *lineptr + offset; - - for (;;) - { - int save_errno; - register int c = getc(stream); - - save_errno = errno; - - /* We always want at least one char left in the buffer, since we - always (unless we get an error while reading the first char) - NUL-terminate the line buffer. */ - - assert((*lineptr + *n) == (read_pos + nchars_avail)); - if (nchars_avail < 2) - { - if (*n > MIN_CHUNK) - *n *= 2; - else - *n += MIN_CHUNK; - - nchars_avail = (int)(*n + *lineptr - read_pos); - *lineptr = realloc(*lineptr, *n); - if (!*lineptr) - { - errno = ENOMEM; - return -1; - } - read_pos = *n - nchars_avail + *lineptr; - assert((*lineptr + *n) == (read_pos + nchars_avail)); - } - - if (ferror(stream)) - { - /* Might like to return partial line, but there is no - place for us to store errno. And we don't want to just - lose errno. */ - errno = save_errno; - return -1; - } - - if (c == EOF) - { - /* Return partial line, if any. */ - if (read_pos == *lineptr) - return -1; - else - break; - } - - *read_pos++ = c; - nchars_avail--; - - if (c == terminator) - /* Return the line. */ - break; - } - - /* Done - NUL terminate and return the number of chars read. */ - *read_pos = '\0'; - - ret = (int)(read_pos - (*lineptr + offset)); - return ret; -} - -int -getline(lineptr, n, stream) -char **lineptr; -size_t *n; -FILE *stream; -{ - return getstr(lineptr, n, stream, '\n', 0); -} \ No newline at end of file diff --git a/src/os/src/windows64/twinmsghdr.c b/src/os/src/windows64/twinmsghdr.c deleted file mode 100644 index 9f35d3ec62..0000000000 --- a/src/os/src/windows64/twinmsghdr.c +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include - -void taosFreeMsgHdr(void *hdr) { - WSAMSG *msgHdr = (WSAMSG *)hdr; - free(msgHdr->lpBuffers); -} - -int taosMsgHdrSize(void *hdr) { - WSAMSG *msgHdr = (WSAMSG *)hdr; - return msgHdr->dwBufferCount; -} - -void taosSendMsgHdr(void *hdr, int fd) { - WSAMSG *msgHdr = (WSAMSG *)hdr; - DWORD len; - - WSASendMsg(fd, msgHdr, 0, &len, 0, 0); - msgHdr->dwBufferCount = 0; -} - -void taosInitMsgHdr(void **hdr, void *dest, int maxPkts) { - WSAMSG *msgHdr = (WSAMSG *)malloc(sizeof(WSAMSG)); - memset(msgHdr, 0, sizeof(WSAMSG)); - *hdr = msgHdr; - - // see ws2def.h - // the size of LPSOCKADDR and sockaddr_in * is same, so it's safe - msgHdr->name = (LPSOCKADDR)dest; - msgHdr->namelen = sizeof(struct sockaddr_in); - int size = sizeof(WSABUF) * maxPkts; - msgHdr->lpBuffers = (LPWSABUF)malloc(size); - memset(msgHdr->lpBuffers, 0, size); - msgHdr->dwBufferCount = 0; -} - -void taosSetMsgHdrData(void *hdr, char *data, int dataLen) { - WSAMSG *msgHdr = (WSAMSG *)hdr; - msgHdr->lpBuffers[msgHdr->dwBufferCount].buf = data; - msgHdr->lpBuffers[msgHdr->dwBufferCount].len = dataLen; - msgHdr->dwBufferCount++; -} - diff --git a/src/os/src/windows64/twinsocket.c b/src/os/src/windows64/twinsocket.c deleted file mode 100644 index 0690a4ceb1..0000000000 --- a/src/os/src/windows64/twinsocket.c +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include -#include -#include -#include -#include -#include -#include - -void taosWinSocketInit() { - static char flag = 0; - if (flag == 0) { - WORD wVersionRequested; - WSADATA wsaData; - wVersionRequested = MAKEWORD(1, 1); - if (WSAStartup(wVersionRequested, &wsaData) == 0) { - flag = 1; - } - } -} - -int taosSetNonblocking(SOCKET sock, int on) { - u_long mode; - if (on) { - mode = 1; - ioctlsocket(sock, FIONBIO, &mode); - } - else { - mode = 0; - ioctlsocket(sock, FIONBIO, &mode); - } - return 0; -} - -int taosGetPrivateIp(char *const ip) { - PIP_ADAPTER_ADDRESSES pAddresses = 0; - IP_ADAPTER_DNS_SERVER_ADDRESS *pDnServer = 0; - ULONG outBufLen = 0; - DWORD dwRetVal = 0; - char buff[100]; - DWORD bufflen = 100; - int i; - int flag = -1; - - taosWinSocketInit(); - GetAdaptersAddresses(AF_UNSPEC, 0, NULL, pAddresses, &outBufLen); - pAddresses = (IP_ADAPTER_ADDRESSES *)malloc(outBufLen); - if ((dwRetVal = GetAdaptersAddresses(AF_INET, GAA_FLAG_SKIP_ANYCAST, NULL, pAddresses, &outBufLen)) == NO_ERROR) { - while (pAddresses) { - if (wcsstr(pAddresses->FriendlyName, L"Loopback") != 0) { - pAddresses = pAddresses->Next; - continue; - } - if (pAddresses->OperStatus == IfOperStatusUp) { - //printf("%s, Status: active\n", pAddresses->FriendlyName); - } - else { - //printf("%s, Status: deactive\n", pAddresses->FriendlyName); - pAddresses = pAddresses->Next; - continue; - } - - PIP_ADAPTER_UNICAST_ADDRESS pUnicast = pAddresses->FirstUnicastAddress; - for (i = 0; pUnicast != NULL; i++) { - if (pUnicast->Address.lpSockaddr->sa_family == AF_INET) { - struct sockaddr_in *sa_in = (struct sockaddr_in *)pUnicast->Address.lpSockaddr; - strcpy(ip, inet_ntop(AF_INET, &(sa_in->sin_addr), buff, bufflen)); - flag = 0; - //printf("%s\n", ip); - } - else if (pUnicast->Address.lpSockaddr->sa_family == AF_INET6) { - struct sockaddr_in6 *sa_in6 = (struct sockaddr_in6 *)pUnicast->Address.lpSockaddr; - strcpy(ip, inet_ntop(AF_INET6, &(sa_in6->sin6_addr), buff, bufflen)); - flag = 0; - //printf("%s\n", ip); - } - else { - } - pUnicast = pUnicast->Next; - } - pAddresses = pAddresses->Next; - } - } - else { - LPVOID lpMsgBuf; - printf("Call to GetAdaptersAddresses failed.\n"); - if (FormatMessage( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - dwRetVal, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPTSTR) & lpMsgBuf, - 0, - NULL)) { - printf("\tError: %s", lpMsgBuf); - } - LocalFree(lpMsgBuf); - } - free(pAddresses); - return flag; -} diff --git a/src/os/src/windows64/twintcpclient.c b/src/os/src/windows64/twintcpclient.c deleted file mode 100644 index 4736042aba..0000000000 --- a/src/os/src/windows64/twintcpclient.c +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "tulog.h" - -void *taosInitTcpClient(char *ip, uint16_t port, char *label, int num, void *fp, void *shandle) { - tError("InitTcpClient not support in windows"); - return 0; -} - -void taosCloseTcpClientConnection(void *chandle) { - tError("CloseTcpClientConnection not support in windows"); -} - -void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) { - tError("OpenTcpClientConnection not support in windows"); - return 0; -} - -int taosSendTcpClientData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) { - tError("SendTcpClientData not support in windows"); - return 0; -} - -void taosCleanUpTcpClient(void *chandle) { - tError("SendTcpClientData not support in windows"); -} diff --git a/src/plugins/http/CMakeLists.txt b/src/plugins/http/CMakeLists.txt index 3280a37c94..4dbfde8ce6 100644 --- a/src/plugins/http/CMakeLists.txt +++ b/src/plugins/http/CMakeLists.txt @@ -14,7 +14,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(http ${SRC}) - TARGET_LINK_LIBRARIES(http taos_static z) +# TARGET_LINK_LIBRARIES(http taos_static z) + TARGET_LINK_LIBRARIES(http taos z) IF (TD_ADMIN) TARGET_LINK_LIBRARIES(http admin) diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index b229673df2..98028fdbb4 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -18,7 +18,6 @@ #include "taosmsg.h" #include "tsocket.h" #include "tutil.h" -#include "ttime.h" #include "ttimer.h" #include "tglobal.h" #include "tcache.h" @@ -53,8 +52,7 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { HttpThread *pThread = pContext->pThread; if (pContext->fd >= 0) { epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL); - int32_t fd = pContext->fd; - pContext->fd = -1; + int32_t fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1); taosCloseSocket(fd); if (!tsHttpServer.fallback) { ehttpDecContextRef(&pContext); @@ -65,7 +63,7 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) { static void httpDestroyContext(void *data) { HttpContext *pContext = *(HttpContext **)data; D("==context[%p] destroyed==", pContext); - if (pContext->fd > 0) tclose(pContext->fd); + if (pContext->fd > 0) taosClose(pContext->fd); HttpThread *pThread = pContext->pThread; httpRemoveContextFromEpoll(pContext); @@ -88,7 +86,7 @@ static void httpDestroyContext(void *data) { } } - tfree(pContext); + taosTFree(pContext); } bool httpInitContexts() { diff --git a/src/plugins/http/src/httpServer.c b/src/plugins/http/src/httpServer.c index 5a785d2e55..d2aee3f15f 100644 --- a/src/plugins/http/src/httpServer.c +++ b/src/plugins/http/src/httpServer.c @@ -18,7 +18,6 @@ #include "taosmsg.h" #include "tsocket.h" #include "tutil.h" -#include "ttime.h" #include "ttimer.h" #include "tglobal.h" #include "httpInt.h" @@ -354,7 +353,7 @@ static void *httpAcceptHttpConnection(void *arg) { if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { httpError("context:%p, fd:%d, ip:%s, thread:%s, failed to add http fd for epoll, error:%s", pContext, connFd, pContext->ipstr, pThread->label, strerror(errno)); - tclose(pContext->fd); + taosClose(pContext->fd); if (tsHttpServer.fallback) httpReleaseContext(pContext); ehttpDecContextRef(&pContext); continue; diff --git a/src/plugins/http/src/httpSession.c b/src/plugins/http/src/httpSession.c index 256b0c9549..fce85df45e 100644 --- a/src/plugins/http/src/httpSession.c +++ b/src/plugins/http/src/httpSession.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "taos.h" -#include "ttime.h" #include "tglobal.h" #include "tcache.h" #include "httpInt.h" diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 43466ee57e..1ad994e5c2 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -105,7 +105,7 @@ void httpCleanUpSystem() { httpCleanupContexts(); httpCleanUpSessions(); pthread_mutex_destroy(&tsHttpServer.serverMutex); - tfree(tsHttpServer.pThreads); + taosTFree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; tsHttpServer.status = HTTP_SERVER_CLOSED; diff --git a/src/plugins/monitor/CMakeLists.txt b/src/plugins/monitor/CMakeLists.txt index 9a20286797..9b6cbfeef3 100644 --- a/src/plugins/monitor/CMakeLists.txt +++ b/src/plugins/monitor/CMakeLists.txt @@ -11,5 +11,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(./src SRC) ADD_LIBRARY(monitor ${SRC}) - TARGET_LINK_LIBRARIES(monitor taos_static) +# TARGET_LINK_LIBRARIES(monitor taos_static) + TARGET_LINK_LIBRARIES(monitor taos) ENDIF () diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 0cc28bb82c..b31fc368af 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -18,7 +18,6 @@ #include "taosdef.h" #include "taoserror.h" #include "tlog.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "tsystem.h" diff --git a/src/plugins/mqtt/CMakeLists.txt b/src/plugins/mqtt/CMakeLists.txt index 5fc8f9039f..929232cfe1 100644 --- a/src/plugins/mqtt/CMakeLists.txt +++ b/src/plugins/mqtt/CMakeLists.txt @@ -14,7 +14,8 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) INCLUDE_DIRECTORIES(inc) AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(mqtt ${SRC}) - TARGET_LINK_LIBRARIES(mqtt taos_static cJson mqttc) +# TARGET_LINK_LIBRARIES(mqtt taos_static cJson mqttc) + TARGET_LINK_LIBRARIES(mqtt taos cJson mqttc) IF (TD_ADMIN) TARGET_LINK_LIBRARIES(mqtt admin cJson) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6efc8a827e..1277e7bfbb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -28,7 +28,6 @@ #include "queryLog.h" #include "tlosertree.h" #include "tscompression.h" -#include "ttime.h" /** * check if the primary column is load by default, otherwise, the program will @@ -930,7 +929,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { - tfree(sasArray); + taosTFree(sasArray); return; } @@ -993,10 +992,10 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * continue; } - tfree(sasArray[i].data); + taosTFree(sasArray[i].data); } - tfree(sasArray); + taosTFree(sasArray); } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { @@ -1301,7 +1300,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - tfree(sasArray[i].data); + taosTFree(sasArray[i].data); } free(sasArray); @@ -1468,7 +1467,7 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - tfree(pTagCtx); + taosTFree(pTagCtx); } } } @@ -1576,8 +1575,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order return TSDB_CODE_SUCCESS; _clean: - tfree(pRuntimeEnv->resultInfo); - tfree(pRuntimeEnv->pCtx); + taosTFree(pRuntimeEnv->resultInfo); + taosTFree(pRuntimeEnv->pCtx); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -1602,11 +1601,11 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { } tVariantDestroy(&pCtx->tag); - tfree(pCtx->tagInfo.pTagCtxList); + taosTFree(pCtx->tagInfo.pTagCtxList); } - tfree(pRuntimeEnv->resultInfo); - tfree(pRuntimeEnv->pCtx); + taosTFree(pRuntimeEnv->resultInfo); + taosTFree(pRuntimeEnv->pCtx); } pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo); @@ -2739,8 +2738,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { - tfree(posList); - tfree(pTableList); + taosTFree(posList); + taosTFree(pTableList); qError("QInfo:%p failed alloc memory", pQInfo); longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -2759,8 +2758,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } if (numOfTables == 0) { - tfree(posList); - tfree(pTableList); + taosTFree(posList); + taosTFree(pTableList); assert(pQInfo->numOfGroupResultPages == 0); return 0; @@ -2843,10 +2842,10 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - tfree(pTree); - tfree(pTableList); - tfree(posList); - tfree(pResultInfo); + taosTFree(pTree); + taosTFree(pTableList); + taosTFree(posList); + taosTFree(pResultInfo); return -1; } @@ -2860,14 +2859,14 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - tfree(pTableList); - tfree(posList); - tfree(pTree); + taosTFree(pTableList); + taosTFree(posList); + taosTFree(pTree); pQInfo->offset = 0; - tfree(pResultInfo); - tfree(buf); + taosTFree(pResultInfo); + taosTFree(buf); return pQInfo->numOfGroupResultPages; } @@ -5454,13 +5453,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, return TSDB_CODE_SUCCESS; _cleanup: - tfree(*pExpr); + taosTFree(*pExpr); taosArrayDestroy(*pTableIdList); *pTableIdList = NULL; - tfree(*tbnameCond); - tfree(*groupbyCols); - tfree(*tagCols); - tfree(*tagCond); + taosTFree(*tbnameCond); + taosTFree(*groupbyCols); + taosTFree(*tagCols); + taosTFree(*tagCond); return code; } @@ -5511,7 +5510,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); if (code != TSDB_CODE_SUCCESS) { - tfree(pExprs); + taosTFree(pExprs); return code; } @@ -5540,7 +5539,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * int32_t param = pExprs[i].base.arg[0].argValue.i64; if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { - tfree(pExprs); + taosTFree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; } @@ -5905,14 +5904,14 @@ _cleanup_query: taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); } - tfree(pTagCols); + taosTFree(pTagCols); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; if (pExprInfo->pExpr != NULL) { tExprTreeDestroy(&pExprInfo->pExpr, NULL); } } - tfree(pExprs); + taosTFree(pExprs); _cleanup: freeQInfo(pQInfo); @@ -5997,7 +5996,7 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p start to free QInfo", pQInfo); for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - tfree(pQuery->sdata[col]); + taosTFree(pQuery->sdata[col]); } teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); @@ -6005,7 +6004,7 @@ static void freeQInfo(SQInfo *pQInfo) { for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { - tfree(pColFilter->pFilters); + taosTFree(pColFilter->pFilters); } } @@ -6018,11 +6017,11 @@ static void freeQInfo(SQInfo *pQInfo) { } } - tfree(pQuery->pSelectExpr); + taosTFree(pQuery->pSelectExpr); } if (pQuery->fillVal != NULL) { - tfree(pQuery->fillVal); + taosTFree(pQuery->fillVal); } // todo refactor, extract method to destroytableDataInfo @@ -6041,7 +6040,7 @@ static void freeQInfo(SQInfo *pQInfo) { } } - tfree(pQInfo->pBuf); + taosTFree(pQInfo->pBuf); taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList); taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); @@ -6049,27 +6048,27 @@ static void freeQInfo(SQInfo *pQInfo) { if (pQuery->pGroupbyExpr != NULL) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - tfree(pQuery->pGroupbyExpr); + taosTFree(pQuery->pGroupbyExpr); } - tfree(pQuery->tagColList); - tfree(pQuery->pFilterInfo); + taosTFree(pQuery->tagColList); + taosTFree(pQuery->pFilterInfo); if (pQuery->colList != NULL) { for (int32_t i = 0; i < pQuery->numOfCols; i++) { SColumnInfo* column = pQuery->colList + i; freeColumnFilterInfo(column->filters, column->numOfFilters); } - tfree(pQuery->colList); + taosTFree(pQuery->colList); } - tfree(pQuery->sdata); - tfree(pQuery); + taosTFree(pQuery->sdata); + taosTFree(pQuery); pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); - tfree(pQInfo); + taosTFree(pQInfo); } static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { @@ -6692,7 +6691,7 @@ void qCleanupQueryMgmt(void* pQMgmt) { taosCacheCleanup(pqinfoPool); pthread_mutex_destroy(&pQueryMgmt->lock); - tfree(pQueryMgmt); + taosTFree(pQueryMgmt); qDebug("vgId:%d queryMgmt cleanup completed", vgId); } diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index fb57f71199..21b5361acb 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -38,7 +38,7 @@ tExtMemBuffer* createExtMemBuffer(int32_t inMemSize, int32_t elemSize, int32_t p pMemBuffer->numOfElemsPerPage = (pMemBuffer->pageSize - sizeof(tFilePage)) / pMemBuffer->nElemSize; char name[MAX_TMPFILE_PATH_LENGTH] = {0}; - getTmpfilePath("extbuf", name); + taosGetTmpfilePath("extbuf", name); pMemBuffer->path = strdup(name); uDebug("create tmp file:%s", pMemBuffer->path); @@ -65,7 +65,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { // release flush out info link SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) { - tfree(pFileMeta->flushoutData.pFlushoutInfo); + taosTFree(pFileMeta->flushoutData.pFlushoutInfo); } // release all in-memory buffer pages @@ -73,7 +73,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { while (pFilePages != NULL) { tFilePagesItem *pTmp = pFilePages; pFilePages = pFilePages->pNext; - tfree(pTmp); + taosTFree(pTmp); } // close temp file @@ -88,8 +88,8 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { destroyColumnModel(pMemBuffer->pColumnModel); - tfree(pMemBuffer->path); - tfree(pMemBuffer); + taosTFree(pMemBuffer->path); + taosTFree(pMemBuffer); return NULL; } @@ -276,7 +276,7 @@ int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) { tFilePagesItem *ptmp = first; first = first->pNext; - tfree(ptmp); // release all data in memory buffer + taosTFree(ptmp); // release all data in memory buffer } fflush(pMemBuffer->file); // flush to disk @@ -301,7 +301,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) { while (first != NULL) { tFilePagesItem *ptmp = first; first = first->pNext; - tfree(ptmp); + taosTFree(ptmp); } pMemBuffer->fileMeta.numOfElemsInFile = 0; @@ -788,7 +788,7 @@ void destroyColumnModel(SColumnModel *pModel) { return; } - tfree(pModel); + taosTFree(pModel); } static void printBinaryData(char *data, int32_t len) { @@ -1073,5 +1073,5 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) { } destroyColumnModel(pDesc->pColumnModel); - tfree(pDesc); + taosTFree(pDesc); } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index 9dec2598bc..e65d793dc9 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -74,18 +74,18 @@ void* taosDestoryFillInfo(SFillInfo* pFillInfo) { return NULL; } - tfree(pFillInfo->prevValues); - tfree(pFillInfo->nextValues); - tfree(pFillInfo->pTags); + taosTFree(pFillInfo->prevValues); + taosTFree(pFillInfo->nextValues); + taosTFree(pFillInfo->pTags); for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - tfree(pFillInfo->pData[i]); + taosTFree(pFillInfo->pData[i]); } - tfree(pFillInfo->pData); - tfree(pFillInfo->pFillCol); + taosTFree(pFillInfo->pData); + taosTFree(pFillInfo->pFillCol); - tfree(pFillInfo); + taosTFree(pFillInfo); return NULL; } @@ -454,7 +454,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu pFillInfo->numOfRows = 0; /* the raw data block is exhausted, next value does not exists */ - tfree(*nextValues); + taosTFree(*nextValues); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 7cf995193b..f556e2dd53 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -22,7 +22,6 @@ #include "tglobal.h" #include "tstoken.h" #include "tstrbuild.h" -#include "ttime.h" #include "ttokendef.h" #include "tutil.h" @@ -609,7 +608,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { doDestroyQuerySql(pQuerySql); } - tfree(pClause->pClause); + taosTFree(pClause->pClause); } SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pStableName, @@ -679,12 +678,12 @@ void SQLInfoDestroy(SSqlInfo *pInfo) { tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns); tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals); - tfree(pInfo->pCreateTableInfo); + taosTFree(pInfo->pCreateTableInfo); } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { tVariantListDestroy(pInfo->pAlterInfo->varList); tFieldListDestroy(pInfo->pAlterInfo->pAddColumns); - tfree(pInfo->pAlterInfo); + taosTFree(pInfo->pAlterInfo); } else { if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) { free(pInfo->pDCLInfo->a); @@ -694,7 +693,7 @@ void SQLInfoDestroy(SSqlInfo *pInfo) { tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep); } - tfree(pInfo->pDCLInfo); + taosTFree(pInfo->pDCLInfo); } } diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index 85e45e46b3..7f5fa08cf1 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -70,7 +70,7 @@ static tFilePage *loadIntoBucketFromDisk(tMemBucket *pMemBucket, int32_t segIdx, printf("id: %d count: %" PRIu64 "\n", j, buffer->num); } } - tfree(pPage); + taosTFree(pPage); assert(buffer->num == pMemBuffer->fileMeta.numOfElemsInFile); } @@ -284,7 +284,7 @@ tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE }; default: { uError("MemBucket:%p,not support data type %d,failed", pBucket, pBucket->dataType); - tfree(pBucket); + taosTFree(pBucket); return NULL; } } @@ -292,14 +292,14 @@ tMemBucket *tMemBucketCreate(int32_t totalSlots, int32_t nBufferSize, int16_t nE int32_t numOfCols = pDesc->pColumnModel->numOfCols; if (numOfCols != 1) { uError("MemBucket:%p,only consecutive data is allowed,invalid numOfCols:%d", pBucket, numOfCols); - tfree(pBucket); + taosTFree(pBucket); return NULL; } SSchema* pSchema = getColumnModelSchema(pDesc->pColumnModel, 0); if (pSchema->type != dataType) { uError("MemBucket:%p,data type is not consistent,%d in schema, %d in param", pBucket, pSchema->type, dataType); - tfree(pBucket); + taosTFree(pBucket); return NULL; } @@ -329,7 +329,7 @@ void tMemBucketDestroy(tMemBucket *pBucket) { if (pBucket->pSegs) { for (int32_t i = 0; i < pBucket->numOfSegs; ++i) { tMemBucketSegment *pSeg = &(pBucket->pSegs[i]); - tfree(pSeg->pBoundingEntries); + taosTFree(pSeg->pBoundingEntries); if (pSeg->pBuffer == NULL || pSeg->numOfSlots == 0) { continue; @@ -340,12 +340,12 @@ void tMemBucketDestroy(tMemBucket *pBucket) { pSeg->pBuffer[j] = destoryExtMemBuffer(pSeg->pBuffer[j]); } } - tfree(pSeg->pBuffer); + taosTFree(pSeg->pBuffer); } } - tfree(pBucket->pSegs); - tfree(pBucket); + taosTFree(pBucket->pSegs); + taosTFree(pBucket); } /* @@ -789,7 +789,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) } } double val = (1 - fraction) * td + fraction * nd; - tfree(buffer); + taosTFree(buffer); return val; } else { // incur a second round bucket split @@ -891,8 +891,8 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) if (unlink(pMemBuffer->path) != 0) { uError("MemBucket:%p, remove tmp file %s failed", pMemBucket, pMemBuffer->path); } - tfree(pMemBuffer); - tfree(pPage); + taosTFree(pMemBuffer); + taosTFree(pPage); return getPercentileImpl(pMemBucket, count - num, fraction); } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 93ccad8e27..b73a7cc887 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -39,7 +39,7 @@ int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t ro pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false); char path[PATH_MAX] = {0}; - getTmpfilePath("qbuf", path); + taosGetTmpfilePath("qbuf", path); pResBuf->path = strdup(path); pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); @@ -258,7 +258,7 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { assert(d->pn == pn); d->pn = NULL; - tfree(pn); + taosTFree(pn); bufPage = flushPageToDisk(pResultBuf, d); } @@ -406,7 +406,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } unlink(pResultBuf->path); - tfree(pResultBuf->path); + taosTFree(pResultBuf->path); SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet); while(taosHashIterNext(iter)) { @@ -414,8 +414,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { size_t n = taosArrayGetSize(*p); for(int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); - tfree(pi->pData); - tfree(pi); + taosTFree(pi->pData); + taosTFree(pi); } taosArrayDestroy(*p); @@ -428,8 +428,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { taosHashCleanup(pResultBuf->groupSet); taosHashCleanup(pResultBuf->all); - tfree(pResultBuf->assistBuf); - tfree(pResultBuf); + taosTFree(pResultBuf->assistBuf); + taosTFree(pResultBuf); } SPageInfo* getLastPageInfo(SIDList pList) { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 90cf394035..fe39fe4e4a 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -21,7 +21,7 @@ STSBuf* tsBufCreate(bool autoDelete, int32_t order) { return NULL; } - getTmpfilePath("join", pTSBuf->path); + taosGetTmpfilePath("join", pTSBuf->path); pTSBuf->f = fopen(pTSBuf->path, "w+"); if (pTSBuf->f == NULL) { free(pTSBuf); @@ -142,11 +142,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) { return NULL; } - tfree(pTSBuf->assistBuf); - tfree(pTSBuf->tsData.rawBuf); + taosTFree(pTSBuf->assistBuf); + taosTFree(pTSBuf->tsData.rawBuf); - tfree(pTSBuf->pData); - tfree(pTSBuf->block.payload); + taosTFree(pTSBuf->pData); + taosTFree(pTSBuf->block.payload); fclose(pTSBuf->f); @@ -739,11 +739,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { int64_t offset = getDataStartOffset(); int32_t size = pSrcBuf->fileSize - offset; -#ifdef LINUX - ssize_t rc = taosTSendFile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size); -#else - ssize_t rc = fsendfile(pDestBuf->f, pSrcBuf->f, &offset, size); -#endif + ssize_t rc = taosFSendFile(pDestBuf->f, pSrcBuf->f, &offset, size); if (rc == -1) { // tscError("failed to merge tsBuf from:%s to %s, reason:%s\n", pSrcBuf->path, pDestBuf->path, strerror(errno)); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index c9143b3a53..7175262270 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -14,11 +14,8 @@ */ #include "os.h" - #include "hash.h" #include "taosmsg.h" -#include "ttime.h" - #include "qExecutor.h" #include "qUtil.h" @@ -89,7 +86,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { } taosHashCleanup(pWindowResInfo->hashList); - tfree(pWindowResInfo->pResult); + taosTFree(pWindowResInfo->pResult); } void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) { diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index e9827518e1..28b1d9cefe 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -1,13 +1,12 @@ +#include "os.h" #include #include #include #include "taos.h" #include "tsdb.h" - #include "qTsbuf.h" #include "tstoken.h" -#include "ttime.h" #include "tutil.h" namespace { diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index 59fd326ef4..b59e0783a2 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -1,3 +1,4 @@ +#include "os.h" #include #include #include @@ -6,7 +7,6 @@ #include "tsdb.h" #include "../../client/inc/tscUtil.h" -#include "ttime.h" #include "tutil.h" #include "tvariant.h" #include "ttokendef.h" @@ -766,7 +766,7 @@ TEST(testCase, getTempFilePath_test) { char path[4096] = {0}; memset(path, 1, 4096); - getTmpfilePath("new_tmp", path); + taosGetTmpfilePath("new_tmp", path); printf("%s\n", path); } diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt index 902c8b66e4..15a8923d21 100644 --- a/src/rpc/CMakeLists.txt +++ b/src/rpc/CMakeLists.txt @@ -10,19 +10,12 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc) INCLUDE_DIRECTORIES(inc) IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM)) - AUX_SOURCE_DIRECTORY(./src SRC) + AUX_SOURCE_DIRECTORY(src SRC) ELSEIF (TD_DARWIN_64) - #LIST(APPEND SRC ./src/thaship.c) - #LIST(APPEND SRC ./src/trpc.c) - #LIST(APPEND SRC ./src/tstring.c) - #LIST(APPEND SRC ./src/tudp.c) AUX_SOURCE_DIRECTORY(src SRC) ELSEIF (TD_WINDOWS_64) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread) - LIST(APPEND SRC ./src/thaship.c) - LIST(APPEND SRC ./src/trpc.c) - LIST(APPEND SRC ./src/tstring.c) - LIST(APPEND SRC ./src/tudp.c) + AUX_SOURCE_DIRECTORY(src SRC) ENDIF () ADD_LIBRARY(trpc ${SRC}) diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 6f89969305..3f76b202c9 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -16,7 +16,6 @@ #include "os.h" #include "tglobal.h" #include "tmempool.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "rpcLog.h" @@ -102,9 +101,9 @@ void rpcCloseConnCache(void *handle) { if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); - tfree(pCache->connHashList); - tfree(pCache->count); - tfree(pCache->lockedBy); + taosTFree(pCache->connHashList); + taosTFree(pCache->count); + taosTFree(pCache->lockedBy); pthread_mutex_unlock(&pCache->mutex); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 7425558535..be609c4ee8 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -17,7 +17,6 @@ #include "tidpool.h" #include "tmd5.h" #include "tmempool.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "lz4.h" @@ -1563,10 +1562,10 @@ static void rpcDecRef(SRpcInfo *pRpc) taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); - tfree(pRpc->connList); + taosTFree(pRpc->connList); pthread_mutex_destroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); - tfree(pRpc); + taosTFree(pRpc); } } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 5cb45a6d26..7fa05af9b0 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -213,8 +213,8 @@ void taosCleanUpTcpServer(void *handle) { tDebug("%s TCP server is cleaned up", pServerObj->label); - tfree(pServerObj->pThreadObj); - tfree(pServerObj); + taosTFree(pServerObj->pThreadObj); + taosTFree(pServerObj); } static void *taosAcceptTcpConnection(void *arg) { @@ -324,7 +324,7 @@ void taosCleanUpTcpClient(void *chandle) { taosStopTcpThread(pThreadObj); tDebug ("%s TCP client is cleaned up", pThreadObj->label); - tfree(pThreadObj); + taosTFree(pThreadObj); } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { @@ -518,7 +518,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, int fd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - tfree(pFdObj); + taosTFree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -571,5 +571,5 @@ static void taosFreeFdObj(SFdObj *pFdObj) { tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); - tfree(pFdObj); + taosTFree(pFdObj); } diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index bdaae4c597..f75375e2fe 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -145,7 +145,7 @@ void taosStopUdpConnection(void *handle) { for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; if (pConn->thread) pthread_join(pConn->thread, NULL); - tfree(pConn->buffer); + taosTFree(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); } @@ -164,7 +164,7 @@ void taosCleanUpUdpConnection(void *handle) { } tDebug("%s UDP is cleaned up", pSet->label); - tfree(pSet); + taosTFree(pSet); } void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 5195e70364..2cc026484b 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -20,7 +20,6 @@ #include "tlog.h" #include "tutil.h" #include "ttimer.h" -#include "ttime.h" #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" @@ -438,9 +437,9 @@ static void syncDecNodeRef(SSyncNode *pNode) { if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) { pthread_mutex_destroy(&pNode->mutex); - tfree(pNode->pRecv); - tfree(pNode->pSyncFwds); - tfree(pNode); + taosTFree(pNode->pRecv); + taosTFree(pNode->pSyncFwds); + taosTFree(pNode); if (atomic_sub_fetch_32(&tsNodeNum, 1) == 0) { if (tsTcpPool) taosCloseTcpThreadPool(tsTcpPool); @@ -466,8 +465,8 @@ int syncDecPeerRef(SSyncPeer *pPeer) syncDecNodeRef(pPeer->pSyncNode); sDebug("%s, resource is freed", pPeer->id); - tfree(pPeer->watchFd); - tfree(pPeer); + taosTFree(pPeer->watchFd); + taosTFree(pPeer); return 0; } @@ -477,7 +476,7 @@ int syncDecPeerRef(SSyncPeer *pPeer) static void syncClosePeerConn(SSyncPeer *pPeer) { taosTmrStopA(&pPeer->timer); - tclose(pPeer->syncFd); + taosClose(pPeer->syncFd); if (pPeer->peerFd >=0) { pPeer->peerFd = -1; taosFreeTcpConn(pPeer->pConn); @@ -756,7 +755,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) if (nodeRole != TAOS_SYNC_ROLE_MASTER) { sError("%s, I am not master anymore", pPeer->id); - tclose(pPeer->syncFd); + taosClose(pPeer->syncFd); return; } @@ -1053,7 +1052,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) if (ret < 0) { sError("%s, failed to create sync thread", pPeer->id); - tclose(pPeer->syncFd); + taosClose(pPeer->syncFd); syncDecPeerRef(pPeer); } else { sInfo("%s, sync connection is up", pPeer->id); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 07d99b916e..505882b209 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -225,10 +225,10 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { - tfree(pNode->pRecv->buffer); + taosTFree(pNode->pRecv->buffer); } - tfree(pNode->pRecv); + taosTFree(pNode->pRecv); } static int syncOpenRecvBuffer(SSyncNode *pNode) @@ -319,7 +319,7 @@ void *syncRestoreData(void *param) (*pNode->notifyRole)(pNode->ahandle, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; - tclose(pPeer->syncFd) + taosClose(pPeer->syncFd) syncCloseRecvBuffer(pNode); __sync_fetch_and_sub(&tsSyncNum, 1); syncDecPeerRef(pPeer); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 8aa7d101e7..f881b680f5 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -190,7 +190,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { pPeer->watchNum = 0; - tclose(pPeer->notifyFd); + taosClose(pPeer->notifyFd); pPeer->notifyFd = inotify_init1(IN_NONBLOCK); if (pPeer->notifyFd < 0) { sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno)); @@ -271,7 +271,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, } free(pHead); - tclose(sfd); + taosClose(sfd); if (code == 0) return bytes; return -1; @@ -352,7 +352,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) sDebug("%s, last wal is closed, try new one", pPeer->id); } - tclose(pPeer->notifyFd); + taosClose(pPeer->notifyFd); return code; } @@ -486,8 +486,8 @@ void *syncRetrieveData(void *param) } pPeer->fileChanged = 0; - tclose(pPeer->notifyFd); - tclose(pPeer->syncFd); + taosClose(pPeer->notifyFd); + taosClose(pPeer->syncFd); syncDecPeerRef(pPeer); return NULL; diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index fa94caeed7..88c2c53be4 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -102,7 +102,7 @@ void taosCloseTcpThreadPool(void *param) if (pThread) taosStopPoolThread(pThread); } - tfree(pPool->pThread); + taosTFree(pPool->pThread); free(pPool); uDebug("%p TCP pool is closed", pPool); } @@ -166,7 +166,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) { pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); - tclose(pConn->fd); + taosClose(pConn->fd); free(pConn); } @@ -255,7 +255,7 @@ static void *taosAcceptPeerTcpConnection(void *argv) { (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } - tclose(pPool->acceptFd); + taosClose(pPool->acceptFd); return NULL; } @@ -320,6 +320,6 @@ static void taosStopPoolThread(SThreadObj* pThread) { } pthread_join(thread, NULL); - tclose(fd); + taosClose(fd); } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index c308c2a454..3c6db88a9c 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -19,7 +19,6 @@ #include "tlog.h" #include "tutil.h" #include "ttimer.h" -#include "ttime.h" #include "tsocket.h" #include "tglobal.h" #include "taoserror.h" @@ -133,7 +132,7 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); if (firstPkt.syncHead.vgId) { sDebug("%s, vgId in head is not zero, close the connection", pNode->id); - tfree(pNode); + taosTFree(pNode); taosCloseSocket(connFd); return; } @@ -149,7 +148,7 @@ static void arbProcessBrokenLink(void *param) { SNodeConn *pNode = param; sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno)); - tfree(pNode); + taosTFree(pNode); } static int arbProcessPeerMsg(void *param, void *buffer) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 6a54cb2399..d6f73ee1be 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -183,10 +183,10 @@ typedef struct { } STsdbFileH; typedef struct { - int numOfFGroups; - SFileGroup *base; - SFileGroup *pFileGroup; int direction; + STsdbFileH* pFileH; + int fileId; + int index; } SFileGroupIter; // ------------------ tsdbMain.c diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index dcc9d4ca1b..2e097c6ff7 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -157,4 +157,4 @@ _err: return NULL; } -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } \ No newline at end of file +static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { taosTFree(pBufBlock); } \ No newline at end of file diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 6e7b39830e..b5702eb1e7 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -14,12 +14,11 @@ */ #define _DEFAULT_SOURCE #include "os.h" - #include "talgo.h" #include "tchecksum.h" #include "tsdbMain.h" #include "tutil.h" -#include "ttime.h" +#define TAOS_RANDOM_FILE_FAIL_TEST #ifdef TSDB_IDX const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"}; @@ -65,7 +64,7 @@ _err: void tsdbFreeFileH(STsdbFileH *pFileH) { if (pFileH) { pthread_rwlock_destroy(&pFileH->fhlock); - tfree(pFileH->pFGroup); + taosTFree(pFileH->pFGroup); free(pFileH); } } @@ -116,14 +115,14 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); } - tfree(tDataDir); + taosTFree(tDataDir); closedir(dir); return 0; _err: for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); - tfree(tDataDir); + taosTFree(tDataDir); if (dir != NULL) closedir(dir); tsdbCloseFileH(pRepo); return -1; @@ -156,8 +155,10 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { goto _err; } + pthread_rwlock_wrlock(&pFileH->fhlock); pFileH->pFGroup[pFileH->nFGroups++] = fGroup; qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); + pthread_rwlock_unlock(&pFileH->fhlock); return tsdbSearchFGroup(pFileH, fid, TD_EQ); } @@ -168,54 +169,72 @@ _err: return NULL; } -void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { // TODO +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { + pIter->pFileH = pFileH; pIter->direction = direction; - pIter->base = pFileH->pFGroup; - pIter->numOfFGroups = pFileH->nFGroups; + if (pFileH->nFGroups == 0) { - pIter->pFileGroup = NULL; + pIter->index = -1; + pIter->fileId = -1; } else { if (direction == TSDB_FGROUP_ITER_FORWARD) { - pIter->pFileGroup = pFileH->pFGroup; + pIter->index = 0; } else { - pIter->pFileGroup = pFileH->pFGroup + pFileH->nFGroups - 1; + pIter->index = pFileH->nFGroups - 1; } + pIter->fileId = pFileH->pFGroup[pIter->index].fileId; } } -void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { // TODO - if (pIter->numOfFGroups == 0) { - assert(pIter->pFileGroup == NULL); +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { + STsdbFileH *pFileH = pIter->pFileH; + + if (pFileH->nFGroups == 0) { + pIter->index = -1; + pIter->fileId = -1; return; } int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; - void *ptr = taosbsearch(&fid, pIter->base, pIter->numOfFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); + void *ptr = taosbsearch(&fid, (void *)pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); if (ptr == NULL) { - pIter->pFileGroup = NULL; + pIter->index = -1; + pIter->fileId = -1; } else { - pIter->pFileGroup = (SFileGroup *)ptr; + pIter->index = POINTER_DISTANCE(ptr, pFileH->pFGroup) / sizeof(SFileGroup); + pIter->fileId = ((SFileGroup *)ptr)->fileId; } } -SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {//TODO - SFileGroup *ret = pIter->pFileGroup; - if (ret == NULL) return NULL; +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { + STsdbFileH *pFileH = pIter->pFileH; + SFileGroup *pFGroup = NULL; + + if (pIter->index < 0 || pIter->index >= pFileH->nFGroups || pIter->fileId < 0) return NULL; + + pFGroup = &pFileH->pFGroup[pIter->index]; + if (pFGroup->fileId != pIter->fileId) { + tsdbSeekFileGroupIter(pIter, pIter->fileId); + } + + if (pIter->index < 0) return NULL; + + pFGroup = &pFileH->pFGroup[pIter->index]; + ASSERT(pFGroup->fileId == pIter->fileId); if (pIter->direction == TSDB_FGROUP_ITER_FORWARD) { - if ((pIter->pFileGroup + 1) == (pIter->base + pIter->numOfFGroups)) { - pIter->pFileGroup = NULL; - } else { - pIter->pFileGroup += 1; - } + pIter->index++; } else { - if (pIter->pFileGroup == pIter->base) { - pIter->pFileGroup = NULL; - } else { - pIter->pFileGroup -= 1; - } + pIter->index--; } - return ret; + + if (pIter->index >= 0 && pIter->index < pFileH->nFGroups) { + pIter->fileId = pFileH->pFGroup[pIter->index].fileId; + } else { + pIter->fileId = -1; + } + + return pFGroup; } int tsdbOpenFile(SFile *pFile, int oflag) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b04b0be9ba..1486adb65b 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -14,8 +14,6 @@ */ // no test file errors here -#undef TAOS_RANDOM_FILE_FAIL - #include "tsdbMain.h" #include "os.h" #include "talgo.h" @@ -23,10 +21,8 @@ #include "tchecksum.h" #include "tscompression.h" #include "tsdb.h" -#include "ttime.h" #include "tulog.h" - #define TSDB_CFG_FILE_NAME "config" #define TSDB_DATA_DIR_NAME "data" #define TSDB_META_FILE_NAME "meta" @@ -214,7 +210,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ char *sdup = strdup(pRepo->rootDir); char *prefix = dirname(sdup); int prefixLen = strlen(prefix); - tfree(sdup); + taosTFree(sdup); if (name[0] == 0) { // get the file from index or after, but not larger than eindex int fid = (*index) / TSDB_FILE_TYPE_MAX; @@ -262,14 +258,14 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } if (stat(fname, &fState) < 0) { - tfree(fname); + taosTFree(fname); return 0; } *size = fState.st_size; // magic = *size; - tfree(fname); + taosTFree(fname); return magic; } @@ -565,7 +561,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { return 0; _err: - tfree(fname); + taosTFree(fname); if (fd >= 0) close(fd); return -1; } @@ -602,13 +598,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { tsdbDecodeCfg(buf, pCfg); - tfree(fname); + taosTFree(fname); close(fd); return 0; _err: - tfree(fname); + taosTFree(fname); if (fd >= 0) close(fd); return -1; } @@ -681,7 +677,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { tsdbFreeMeta(pRepo->tsdbMeta); // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); - tfree(pRepo->rootDir); + taosTFree(pRepo->rootDir); pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } @@ -787,6 +783,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC); while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; + if (tsdbLoadCompIdx(&rhelper, NULL) < 0) goto _err; for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 506f3c11c1..44f787a8c0 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -398,7 +398,7 @@ static void tsdbFreeMemTable(SMemTable* pMemTable) { tdListFree(pMemTable->bufBlockList); tdListFree(pMemTable->actList); - tfree(pMemTable->tData); + taosTFree(pMemTable->tData); free(pMemTable); } } @@ -574,6 +574,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pGroup = NULL; SMemTable * pMem = pRepo->imem; + bool newLast = false; TSKEY minKey = 0, maxKey = 0; tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); @@ -603,6 +604,13 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe goto _err; } + newLast = TSDB_NLAST_FILE_OPENED(pHelper); + + if (tsdbLoadCompIdx(pHelper, NULL) < 0) { + tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + // Loop to commit data in each table for (int tid = 1; tid < pMem->maxTables; tid++) { SCommitIter *pIter = iters + tid; @@ -647,22 +655,34 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe goto _err; } - tfree(dataDir); + taosTFree(dataDir); tsdbCloseHelperFile(pHelper, 0); pthread_rwlock_wrlock(&(pFileH->fhlock)); + #ifdef TSDB_IDX - pGroup->files[TSDB_FILE_TYPE_IDX] = *(helperIdxF(pHelper)); + rename(helperNewIdxF(pHelper)->fname, helperIdxF(pHelper)->fname); + pGroup->files[TSDB_FILE_TYPE_IDX].info = helperNewIdxF(pHelper)->info; #endif - pGroup->files[TSDB_FILE_TYPE_HEAD] = *(helperHeadF(pHelper)); - pGroup->files[TSDB_FILE_TYPE_DATA] = *(helperDataF(pHelper)); - pGroup->files[TSDB_FILE_TYPE_LAST] = *(helperLastF(pHelper)); + + rename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname); + pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info; + + if (newLast) { + rename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname); + pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info; + } else { + pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info; + } + + pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info; + pthread_rwlock_unlock(&(pFileH->fhlock)); return 0; _err: - tfree(dataDir); + taosTFree(dataDir); tsdbCloseHelperFile(pHelper, 1); return -1; } @@ -737,7 +757,7 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { pMemTable->tData = pTableData; taosWUnLockLatch(&(pMemTable->latch)); - tfree(tData); + taosTFree(tData); return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index d389a96fee..8db6a4a32a 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -185,7 +185,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { return 0; _err: - tfree(tbname); + taosTFree(tbname); return -1; } @@ -453,7 +453,7 @@ void tsdbFreeMeta(STsdbMeta *pMeta) { if (pMeta) { taosHashCleanup(pMeta->uidMap); tdListFree(pMeta->superList); - tfree(pMeta->tables); + taosTFree(pMeta->tables); pthread_rwlock_destroy(&pMeta->rwLock); free(pMeta); } @@ -477,11 +477,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) { } tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); - tfree(fname); + taosTFree(fname); return 0; _err: - tfree(fname); + taosTFree(fname); return -1; } @@ -745,7 +745,7 @@ static void tsdbFreeTable(STable *pTable) { if (pTable->name != NULL) tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - tfree(TABLE_NAME(pTable)); + taosTFree(TABLE_NAME(pTable)); if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) { tdFreeSchema(pTable->schema[i]); @@ -759,7 +759,7 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); - tfree(pTable->sql); + taosTFree(pTable->sql); free(pTable); } } @@ -1065,9 +1065,9 @@ void tsdbClearTableCfg(STableCfg *config) { if (config->schema) tdFreeSchema(config->schema); if (config->tagSchema) tdFreeSchema(config->tagSchema); if (config->tagValues) kvRowFree(config->tagValues); - tfree(config->name); - tfree(config->sname); - tfree(config->sql); + taosTFree(config->name); + taosTFree(config->sname); + taosTFree(config->sql); free(config); } } @@ -1291,7 +1291,7 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { STable **tTables = pMeta->tables; pMeta->tables = tables; - tfree(tTables); + taosTFree(tTables); tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 8d3908666e..a5ef10cd6b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -20,6 +20,7 @@ #include "tcoding.h" #include "tscompression.h" #include "tsdbMain.h" +#define TAOS_RANDOM_FILE_FAIL_TEST #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) #define TSDB_KEY_COL_OFFSET 0 @@ -72,8 +73,8 @@ int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { void tsdbDestroyHelper(SRWHelper *pHelper) { if (pHelper) { - tzfree(pHelper->pBuffer); - tzfree(pHelper->compBuffer); + taosTZfree(pHelper->pBuffer); + taosTZfree(pHelper->compBuffer); tsdbDestroyHelperFile(pHelper); tsdbDestroyHelperTable(pHelper); tsdbDestroyHelperBlock(pHelper); @@ -118,12 +119,12 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Open the files #ifdef TSDB_IDX - if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperIdxF(pHelper), O_RDONLY) < 0) return -1; #endif - if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) return -1; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) goto _err; - if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1; + if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1; #ifdef TSDB_IDX // Create and open .i file @@ -144,23 +145,20 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { pFile = helperNewLastF(pHelper); - if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) goto _err; + if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1; pFile->info.size = TSDB_FILE_HEAD_SIZE; pFile->info.magic = TSDB_FILE_INIT_MAGIC; pFile->info.len = 0; if (tsdbUpdateFileHeader(pFile, 0) < 0) return -1; } } else { - if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) goto _err; - if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) goto _err; + if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) return -1; + if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) return -1; } helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN); - return tsdbLoadCompIdx(pHelper, NULL); - -_err: - return -1; + return 0; } int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { @@ -183,8 +181,12 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { pFile = helperDataF(pHelper); if (pFile->fd > 0) { if (helperType(pHelper) == TSDB_WRITE_HELPER) { - tsdbUpdateFileHeader(pFile, 0); - fsync(pFile->fd); + if (!hasError) { + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + } else { + // TODO: shrink back to origin + } } close(pFile->fd); pFile->fd = -1; @@ -193,7 +195,12 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { pFile = helperLastF(pHelper); if (pFile->fd > 0) { if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) { - fsync(pFile->fd); + if (!hasError) { + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + } else { + // TODO: shrink back to origin + } } close(pFile->fd); pFile->fd = -1; @@ -203,60 +210,36 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { #ifdef TSDB_IDX pFile = helperNewIdxF(pHelper); if (pFile->fd > 0) { - if (!hasError) tsdbUpdateFileHeader(pFile, 0); - fsync(pFile->fd); + if (!hasError) { + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + } close(pFile->fd); pFile->fd = -1; - if (hasError) { - (void)remove(pFile->fname); - } else { - if (rename(pFile->fname, helperIdxF(pHelper)->fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperIdxF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - helperIdxF(pHelper)->info = pFile->info; - } + if (hasError) (void)remove(pFile->fname); } #endif pFile = helperNewHeadF(pHelper); if (pFile->fd > 0) { - if (!hasError) tsdbUpdateFileHeader(pFile, 0); - fsync(pFile->fd); + if (!hasError) { + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + } close(pFile->fd); pFile->fd = -1; - if (hasError) { - (void)remove(pFile->fname); - } else { - if (rename(pFile->fname, helperHeadF(pHelper)->fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperHeadF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - helperHeadF(pHelper)->info = pFile->info; - } + if (hasError) (void)remove(pFile->fname); } pFile = helperNewLastF(pHelper); if (pFile->fd > 0) { - if (!hasError) tsdbUpdateFileHeader(pFile, 0); - fsync(pFile->fd); + if (!hasError) { + tsdbUpdateFileHeader(pFile, 0); + fsync(pFile->fd); + } close(pFile->fd); pFile->fd = -1; - if (hasError) { - (void)remove(pFile->fname); - } else { - if (rename(pFile->fname, helperLastF(pHelper)->fname) < 0) { - tsdbError("failed to rename file from %s to %s since %s", pFile->fname, helperLastF(pHelper)->fname, - strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - helperLastF(pHelper)->info = helperNewLastF(pHelper)->info; - } + if (hasError) (void)remove(pFile->fname); } } return 0; @@ -444,8 +427,8 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { pFile = helperNewIdxF(pHelper); #endif - if (tsizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { - pHelper->pWIdx = trealloc(pHelper->pWIdx, tsizeof(pHelper->pWIdx) == 0 ? 1024 : tsizeof(pHelper->pWIdx) * 2); + if (taosTSizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) { + pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, taosTSizeof(pHelper->pWIdx) == 0 ? 1024 : taosTSizeof(pHelper->pWIdx) * 2); if (pHelper->pWIdx == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -470,8 +453,8 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { #endif pFile->info.len += sizeof(TSCKSUM); - if (tsizeof(pHelper->pWIdx) < pFile->info.len) { - pHelper->pWIdx = trealloc(pHelper->pWIdx, pFile->info.len); + if (taosTSizeof(pHelper->pWIdx) < pFile->info.len) { + pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, pFile->info.len); if (pHelper->pWIdx == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -512,7 +495,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object if (pFile->info.len > 0) { - if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { + if ((pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } @@ -541,11 +524,11 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { pHelper->idxH.numOfIdx = 0; void *ptr = pHelper->pBuffer; while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { - size_t tlen = tsizeof(pHelper->idxH.pIdxArray); + size_t tlen = taosTSizeof(pHelper->idxH.pIdxArray); pHelper->idxH.numOfIdx++; if (tlen < pHelper->idxH.numOfIdx * sizeof(SCompIdx)) { - pHelper->idxH.pIdxArray = (SCompIdx *)trealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); + pHelper->idxH.pIdxArray = (SCompIdx *)taosTRealloc(pHelper->idxH.pIdxArray, (tlen == 0) ? 1024 : tlen * 2); if (pHelper->idxH.pIdxArray == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -593,7 +576,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { return -1; } - pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len); + pHelper->pCompInfo = taosTRealloc((void *)pHelper->pCompInfo, pIdx->len); if (taosTRead(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) { tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len, helperHeadF(pHelper)->fname, strerror(errno)); @@ -629,7 +612,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { } size_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols); - pHelper->pCompData = trealloc((void *)pHelper->pCompData, tsize); + pHelper->pCompData = taosTRealloc((void *)pHelper->pCompData, tsize); if (pHelper->pCompData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -807,7 +790,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa if (pCfg->compression) { if (pCfg->compression == TWO_STAGE_COMP) { - pHelper->compBuffer = trealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES); + pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES); if (pHelper->compBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -815,8 +798,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } flen = (*(tDataTypeDesc[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr, - tsizeof(pHelper->pBuffer) - lsize, pCfg->compression, - pHelper->compBuffer, tsizeof(pHelper->compBuffer)); + taosTSizeof(pHelper->pBuffer) - lsize, pCfg->compression, + pHelper->compBuffer, taosTSizeof(pHelper->compBuffer)); } else { flen = tlen; memcpy(tptr, pDataCol->pData, flen); @@ -895,9 +878,9 @@ static int compareKeyBlock(const void *arg1, const void *arg2) { } static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { - if (tsizeof((void *)pHelper->pCompInfo) <= esize) { + if (taosTSizeof((void *)pHelper->pCompInfo) <= esize) { size_t tsize = esize + sizeof(SCompBlock) * 16; - pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize); + pHelper->pCompInfo = (SCompInfo *)taosTRealloc(pHelper->pCompInfo, tsize); if (pHelper->pCompInfo == NULL) return -1; } @@ -923,8 +906,8 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int // Memmove if needed int tsize = pIdx->len - (sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx); if (tsize > 0) { - ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) < tsizeof(pHelper->pCompInfo)); - ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) + tsize <= tsizeof(pHelper->pCompInfo)); + ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) < taosTSizeof(pHelper->pCompInfo)); + ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) + tsize <= taosTSizeof(pHelper->pCompInfo)); memmove(POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1)), POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx), tsize); } @@ -932,7 +915,7 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int pIdx->numOfBlocks++; pIdx->len += sizeof(SCompBlock); - ASSERT(pIdx->len <= tsizeof(pHelper->pCompInfo)); + ASSERT(pIdx->len <= taosTSizeof(pHelper->pCompInfo)); pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; pIdx->hasLast = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; @@ -1094,8 +1077,8 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { static void tsdbDestroyHelperFile(SRWHelper *pHelper) { tsdbCloseHelperFile(pHelper, false); tsdbResetHelperFileImpl(pHelper); - tzfree(pHelper->idxH.pIdxArray); - tzfree(pHelper->pWIdx); + taosTZfree(pHelper->idxH.pIdxArray); + taosTZfree(pHelper->pWIdx); } // ---------- Operations on Helper Table part @@ -1112,7 +1095,7 @@ static void tsdbResetHelperTable(SRWHelper *pHelper) { static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); } -static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); } +static void tsdbDestroyHelperTable(SRWHelper *pHelper) { taosTZfree((void *)pHelper->pCompInfo); } // ---------- Operations on Helper Block part static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { @@ -1142,7 +1125,7 @@ static int tsdbInitHelperBlock(SRWHelper *pHelper) { } static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { - tzfree(pHelper->pCompData); + taosTZfree(pHelper->pCompData); tdFreeDataCols(pHelper->pDataCols[0]); tdFreeDataCols(pHelper->pDataCols[1]); } @@ -1167,7 +1150,7 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t // TODO: pMeta->maxRowBytes and pMeta->maxCols may change here causing invalid write pHelper->pBuffer = - tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols + + taosTMalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols + pMeta->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM)); if (pHelper->pBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1212,13 +1195,13 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl SDataCol *pDataCol) { ASSERT(pDataCol->colId == pCompCol->colId); int tsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; - pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompCol->len); + pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompCol->len); if (pHelper->pBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; } - pHelper->compBuffer = trealloc(pHelper->compBuffer, tsize); + pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tsize); if (pHelper->compBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -1240,7 +1223,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm, pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock, - pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) { + pHelper->compBuffer, taosTSizeof(pHelper->compBuffer)) < 0) { tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname, pCompCol->colId, offset); return -1; @@ -1331,7 +1314,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper); - pHelper->pBuffer = trealloc(pHelper->pBuffer, pCompBlock->len); + pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompBlock->len); if (pHelper->pBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -1395,7 +1378,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows); } - pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize); + pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, zsize); if (pHelper->compBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -1403,7 +1386,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa } if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + toffset, tlen, pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints, pHelper->compBuffer, - tsizeof(pHelper->compBuffer)) < 0) { + taosTSizeof(pHelper->compBuffer)) < 0) { tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d", REPO_ID(pHelper->pRepo), pFile->fname, tcolId, (int64_t)pCompBlock->offset, toffset); goto _err; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 0e3a657fde..ccc631fb58 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -17,7 +17,6 @@ #include "tulog.h" #include "talgo.h" #include "tutil.h" -#include "ttime.h" #include "tcompare.h" #include "exception.h" @@ -555,16 +554,6 @@ static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSK } static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { - SFileGroup* fileGroup = pQueryHandle->pFileGroup; - assert(fileGroup->files[TSDB_FILE_TYPE_HEAD].fname > 0); - - int32_t code = tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, fileGroup); - - //open file failed, return error code to client - if (code != TSDB_CODE_SUCCESS) { - return code; - } - // load all the comp offset value for all tables in this file *numOfBlocks = 0; size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); @@ -1291,15 +1280,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { } static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { - tfree(pSupporter->numOfBlocksPerTable); - tfree(pSupporter->blockIndexArray); + taosTFree(pSupporter->numOfBlocksPerTable); + taosTFree(pSupporter->blockIndexArray); for (int32_t i = 0; i < numOfTables; ++i) { STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i]; - tfree(pBlockInfo); + taosTFree(pBlockInfo); } - tfree(pSupporter->pDataBlockInfo); + taosTFree(pSupporter->pDataBlockInfo); } static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { @@ -1461,12 +1450,20 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; STimeWindow win = TSWINDOW_INITIALIZER; - while ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) != NULL) { + while (true) { + pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + + if ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) == NULL) { + pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + break; + } + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { + pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle, pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo); pQueryHandle->pFileGroup = NULL; @@ -1474,6 +1471,19 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex break; } + if (tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { + pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + code = terrno; + break; + } + + pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); + + if (tsdbLoadCompIdx(&pQueryHandle->rhelper, NULL) < 0) { + code = terrno; + break; + } + if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } @@ -1528,8 +1538,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order); tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid); + pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock); return getDataBlocksInFilesImpl(pQueryHandle, exists); } else { @@ -1770,11 +1782,11 @@ void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { tSkipListDestroyIter(pTableCheckInfo->iter); if (pTableCheckInfo->pDataCols != NULL) { - tfree(pTableCheckInfo->pDataCols->buf); + taosTFree(pTableCheckInfo->pDataCols->buf); } - tfree(pTableCheckInfo->pDataCols); - tfree(pTableCheckInfo->pCompInfo); + taosTFree(pTableCheckInfo->pDataCols); + taosTFree(pTableCheckInfo->pCompInfo); } STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); @@ -2033,7 +2045,7 @@ static void destroyHelper(void* param) { tQueryInfo* pInfo = (tQueryInfo*)param; if (pInfo->optr != TSDB_RELATION_IN) { - tfree(pInfo->q); + taosTFree(pInfo->q); } // tVariantDestroy(&(pInfo->q)); @@ -2185,7 +2197,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosqsort(pTableList->pData, size, POINTER_BYTES, pSupp, tableGroupComparFn); createTableGroupImpl(pTableGroup, pTableList, size, pSupp, tableGroupComparFn); - tfree(pSupp); + taosTFree(pSupp); } return pTableGroup; @@ -2431,11 +2443,11 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { destroyTableMemIterator(pTableCheckInfo); if (pTableCheckInfo->pDataCols != NULL) { - tfree(pTableCheckInfo->pDataCols->buf); + taosTFree(pTableCheckInfo->pDataCols->buf); } - tfree(pTableCheckInfo->pDataCols); - tfree(pTableCheckInfo->pCompInfo); + taosTFree(pTableCheckInfo->pDataCols); + taosTFree(pTableCheckInfo->pCompInfo); } taosArrayDestroy(pQueryHandle->pTableCheckInfo); } @@ -2444,14 +2456,14 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { size_t cols = taosArrayGetSize(pQueryHandle->pColumns); for (int32_t i = 0; i < cols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - tfree(pColInfo->pData); + taosTFree(pColInfo->pData); } taosArrayDestroy(pQueryHandle->pColumns); } taosArrayDestroy(pQueryHandle->defaultLoadColumn); - tfree(pQueryHandle->pDataBlockInfo); - tfree(pQueryHandle->statis); + taosTFree(pQueryHandle->pDataBlockInfo); + taosTFree(pQueryHandle->statis); // todo check error tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, pQueryHandle->mem, pQueryHandle->imem); @@ -2462,7 +2474,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); - tfree(pQueryHandle); + taosTFree(pQueryHandle); } void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index 5e678a8ee5..e504109cec 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -80,7 +80,7 @@ static int insertData(SInsertInfo *pInfo) { pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) { - tfree(pMsg); + taosTFree(pMsg); return -1; } } @@ -88,7 +88,7 @@ static int insertData(SInsertInfo *pInfo) { double etime = getCurTime(); printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows); - tfree(pMsg); + taosTFree(pMsg); return 0; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 2521c582d1..d91ef075ef 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -34,38 +34,7 @@ ELSEIF (TD_WINDOWS_64) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/iconv) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/regex) INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc) - LIST(APPEND SRC ./src/hash.c) - LIST(APPEND SRC ./src/ihash.c) - LIST(APPEND SRC ./src/lz4.c) - LIST(APPEND SRC ./src/shash.c) - LIST(APPEND SRC ./src/tbase64.c) - LIST(APPEND SRC ./src/tcache.c) - LIST(APPEND SRC ./src/tcompression.c) - LIST(APPEND SRC ./src/textbuffer.c) - LIST(APPEND SRC ./src/tglobalcfg.c) - LIST(APPEND SRC ./src/thash.c) - LIST(APPEND SRC ./src/thashutil.c) - LIST(APPEND SRC ./src/thistogram.c) - LIST(APPEND SRC ./src/tidpool.c) - LIST(APPEND SRC ./src/tinterpolation.c) - LIST(APPEND SRC ./src/tlog.c) - LIST(APPEND SRC ./src/tlosertree.c) - LIST(APPEND SRC ./src/tmd5.c) - LIST(APPEND SRC ./src/tmem.c) - LIST(APPEND SRC ./src/tmempool.c) - LIST(APPEND SRC ./src/tmodule.c) - LIST(APPEND SRC ./src/tnote.c) - LIST(APPEND SRC ./src/tpercentile.c) - LIST(APPEND SRC ./src/tsched.c) - LIST(APPEND SRC ./src/tskiplist.c) - LIST(APPEND SRC ./src/tsocket.c) - LIST(APPEND SRC ./src/tstrbuild.c) - LIST(APPEND SRC ./src/ttime.c) - LIST(APPEND SRC ./src/ttimer.c) - LIST(APPEND SRC ./src/ttokenizer.c) - LIST(APPEND SRC ./src/ttypes.c) - LIST(APPEND SRC ./src/tutil.c) - LIST(APPEND SRC ./src/version.c) + AUX_SOURCE_DIRECTORY(src SRC) ADD_LIBRARY(tutil ${SRC}) TARGET_LINK_LIBRARIES(tutil iconv regex pthread osdetail winmm IPHLPAPI ws2_32 lz4) ELSEIF(TD_DARWIN_64) diff --git a/src/util/inc/talgo.h b/src/util/inc/talgo.h index e71e340a21..9e3692225b 100644 --- a/src/util/inc/talgo.h +++ b/src/util/inc/talgo.h @@ -20,6 +20,11 @@ extern "C" { #endif +#ifndef __COMPAR_FN_T +# define __COMPAR_FN_T +typedef int (*__compar_fn_t) (const void *, const void *); +#endif + #define TD_EQ 0x1 #define TD_GT 0x2 #define TD_LT 0x4 diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 695529d02f..88f02aff7e 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -303,7 +303,7 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { pNode->next = NULL; pNode->prev = NULL; - tfree(pNode); + taosTFree(pNode); __unlock(pHashObj->lock); } @@ -335,7 +335,7 @@ void taosHashCleanup(SHashObj *pHashObj) { __unlock(pHashObj->lock); __lock_destroy(pHashObj->lock); - tfree(pHashObj->lock); + taosTFree(pHashObj->lock); memset(pHashObj, 0, sizeof(SHashObj)); free(pHashObj); } diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 3a594faeb9..6879461953 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -153,7 +153,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) { char *buf = calloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, numOfElem - 1, size, param, comparFn, buf); - tfree(buf); + taosTFree(buf); } void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index dc9128d4a9..4a30a66871 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -16,7 +16,6 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tulog.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "tcache.h" @@ -634,7 +633,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { taosTrashCanEmpty(pCacheObj, true); __cache_lock_destroy(pCacheObj); - tfree(pCacheObj->name); + taosTFree(pCacheObj->name); memset(pCacheObj, 0, sizeof(SCacheObj)); free(pCacheObj); } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index 59bc09e877..9fc9386b42 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -297,7 +297,7 @@ void taosReadGlobalLogCfg() { taosReadLogOption(option, value); } - tfree(line); + taosTFree(line); fclose(fp); } @@ -349,7 +349,7 @@ bool taosReadGlobalCfg() { fclose(fp); - tfree(line); + taosTFree(line); return true; } diff --git a/src/util/src/tdes.c b/src/util/src/tdes.c index c76938d3aa..871ae15a8a 100644 --- a/src/util/src/tdes.c +++ b/src/util/src/tdes.c @@ -1,8 +1,5 @@ -#include -#include -#include -#include -#include + +#include "os.h" #include "tkey.h" #define ENCRYPTION_MODE 1 diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index d7bf9d7857..edcf9d45f2 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -21,6 +21,7 @@ #include "tcoding.h" #include "tkvstore.h" #include "tulog.h" +#define TAOS_RANDOM_FILE_FAIL_TEST #define TD_KVSTORE_HEADER_SIZE 512 #define TD_KVSTORE_MAJOR_VERSION 1 @@ -434,9 +435,9 @@ _err: static void tdFreeKVStore(SKVStore *pStore) { if (pStore) { - tfree(pStore->fname); - tfree(pStore->fsnap); - tfree(pStore->fnew); + taosTFree(pStore->fname); + taosTFree(pStore->fsnap); + taosTFree(pStore->fnew); taosHashCleanup(pStore->map); free(pStore); } @@ -575,11 +576,11 @@ static int tdRestoreKVStore(SKVStore *pStore) { if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); taosHashDestroyIter(pIter); - tfree(buf); + taosTFree(buf); return 0; _err: taosHashDestroyIter(pIter); - tfree(buf); + taosTFree(buf); return -1; } diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 96ae417297..913989bf52 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -14,9 +14,6 @@ */ #define _DEFAULT_SOURCE -// no test file errors here -#undef TAOS_RANDOM_FILE_FAIL - #include "os.h" #include "tulog.h" #include "tlog.h" @@ -210,10 +207,10 @@ static bool taosCheckFileIsOpen(char *logFileName) { if (taosLockFile(fd)) { taosUnLockFile(fd); - tclose(fd); + taosClose(fd); return false; } else { - tclose(fd); + taosClose(fd); return true; } } @@ -457,7 +454,7 @@ void taosCloseLog() { static void taosCloseLogByFd(int32_t fd) { if (fd >= 0) { taosUnLockFile(fd); - tclose(fd); + taosClose(fd); } } @@ -482,8 +479,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { return tLogBuff; _err: - tfree(LOG_BUF_BUFFER(tLogBuff)); - tfree(tLogBuff); + taosTFree(LOG_BUF_BUFFER(tLogBuff)); + taosTFree(tLogBuff); return NULL; } @@ -492,7 +489,7 @@ static void taosLogBuffDestroy(SLogBuff *tLogBuff) { tsem_destroy(&(tLogBuff->buffNotEmpty)); pthread_mutex_destroy(&(tLogBuff->buffMutex)); free(tLogBuff->buffer); - tfree(tLogBuff); + taosTFree(tLogBuff); } #endif diff --git a/src/util/src/tmempool.c b/src/util/src/tmempool.c index f5329800f3..4a899ebb48 100644 --- a/src/util/src/tmempool.c +++ b/src/util/src/tmempool.c @@ -56,9 +56,9 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); - tfree(pool_p->freeList); - tfree(pool_p->pool); - tfree(pool_p); + taosTFree(pool_p->freeList); + taosTFree(pool_p->pool); + taosTFree(pool_p); return NULL; } diff --git a/src/util/src/tnote.c b/src/util/src/tnote.c index 91916c92d7..ea09709449 100644 --- a/src/util/src/tnote.c +++ b/src/util/src/tnote.c @@ -13,9 +13,7 @@ * along with this program. If not, see . */ -// no test file errors here -#undef TAOS_RANDOM_FILE_FAIL - +#include "os.h" #include "tnote.h" taosNoteInfo m_HttpNote; diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index aacc4a5487..14029fbd81 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -162,7 +162,7 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui pSkipList->level = 1; if (!initForwardBackwardPtr(pSkipList)) { - tfree(pSkipList); + taosTFree(pSkipList); return NULL; } @@ -170,8 +170,8 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, ui pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t)); if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { - tfree(pSkipList->pHead); - tfree(pSkipList); + taosTFree(pSkipList->pHead); + taosTFree(pSkipList); return NULL; } @@ -201,7 +201,7 @@ void *tSkipListDestroy(SSkipList *pSkipList) { while (pNode != pSkipList->pTail) { SSkipListNode *pTemp = pNode; pNode = SL_GET_FORWARD_POINTER(pNode, 0); - tfree(pTemp); + taosTFree(pTemp); } } @@ -209,11 +209,11 @@ void *tSkipListDestroy(SSkipList *pSkipList) { pthread_rwlock_unlock(pSkipList->lock); pthread_rwlock_destroy(pSkipList->lock); - tfree(pSkipList->lock); + taosTFree(pSkipList->lock); } - tfree(pSkipList->pHead); - tfree(pSkipList); + taosTFree(pSkipList->pHead); + taosTFree(pSkipList); return NULL; } @@ -470,7 +470,7 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { } if (pSkipList->keyInfo.freeNode) { - tfree(p); + taosTFree(p); } ++count; @@ -509,7 +509,7 @@ void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { } if (pSkipList->keyInfo.freeNode) { - tfree(pNode); + taosTFree(pNode); } atomic_sub_fetch_32(&pSkipList->size, 1); @@ -592,7 +592,7 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { return NULL; } - tfree(iter); + taosTFree(iter); return NULL; } diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 5cdc6ff918..b2982d353f 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -16,7 +16,6 @@ #include "os.h" #include "tlog.h" #include "tsched.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" diff --git a/src/util/src/tutil.c b/src/util/src/tutil.c index eaa60d739b..f4b6ae629d 100644 --- a/src/util/src/tutil.c +++ b/src/util/src/tutil.c @@ -16,7 +16,6 @@ #include "os.h" #include "tcrc32c.h" #include "tglobal.h" -#include "ttime.h" #include "taosdef.h" #include "tutil.h" #include "tulog.h" diff --git a/src/util/tests/cacheTest.cpp b/src/util/tests/cacheTest.cpp index 9100b7e7f6..e0debd53f4 100644 --- a/src/util/tests/cacheTest.cpp +++ b/src/util/tests/cacheTest.cpp @@ -1,3 +1,4 @@ +#include "os.h" #include #include #include @@ -10,7 +11,6 @@ #include "tutil.h" #include "tcache.h" #include "ttimer.h" -#include "ttime.h" namespace { int32_t tsMaxMgmtConnections = 10000; diff --git a/src/util/tests/hashTest.cpp b/src/util/tests/hashTest.cpp index 93a1989741..f9f636db90 100644 --- a/src/util/tests/hashTest.cpp +++ b/src/util/tests/hashTest.cpp @@ -1,3 +1,4 @@ +#include "os.h" #include #include #include @@ -5,7 +6,6 @@ #include "hash.h" #include "taos.h" -#include "ttime.h" namespace { // the simple test code for basic operations @@ -153,4 +153,4 @@ TEST(testCase, hashTest) { stringKeyTest(); noLockPerformanceTest(); multithreadsTest(); -} \ No newline at end of file +} diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 70445a3d65..77174f69fd 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -3,9 +3,9 @@ #include #include +#include "os.h" #include "taosmsg.h" #include "tskiplist.h" -#include "ttime.h" #include "tutil.h" namespace { @@ -59,7 +59,7 @@ void doubleSkipListTest() { } if (size > 0) { - tfree(pNodes); + taosTFree(pNodes); } } @@ -196,7 +196,7 @@ void stringKeySkiplistTest() { tSkipListRemoveNode(pSkipList, pres[0]); if (num > 0) { - tfree(pres); + taosTFree(pres); } } @@ -276,7 +276,7 @@ void skiplistPerformanceTest() { assert(tSkipListGetSize(pSkipList) == size); tSkipListDestroy(pSkipList); - tfree(total); + taosTFree(total); } // todo not support duplicated key yet @@ -357,7 +357,7 @@ TEST(testCase, skiplist_test) { printf("-----%lf\n", pNodes[i]->key.dKey); } printf("the range query result size is: %d\n", size); - tfree(pNodes); + taosTFree(pNodes); SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); for (int32_t i = 0; i < 8; i += 2) { @@ -371,7 +371,7 @@ TEST(testCase, skiplist_test) { for (int32_t i = 0; i < r; ++i) { // printf("%lf ", pNodes[i]->key.dKey); } - tfree(pNodes); + taosTFree(pNodes); free(pKeys);*/ } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 8a56a2c500..6abd255dc4 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -25,7 +25,6 @@ #include "tglobal.h" #include "trpc.h" #include "tsdb.h" -#include "ttime.h" #include "ttimer.h" #include "tutil.h" #include "vnode.h" @@ -348,7 +347,7 @@ void vnodeRelease(void *pVnodeRaw) { dnodeFreeVnodeRqueue(pVnode->rqueue); pVnode->rqueue = NULL; - tfree(pVnode->rootDir); + taosTFree(pVnode->rootDir); if (pVnode->dropped) { char rootDir[TSDB_FILENAME_LEN] = {0}; @@ -870,7 +869,7 @@ static int32_t vnodeReadCfg(SVnodeObj *pVnode) { } PARSE_OVER: - tfree(content); + taosTFree(content); cJSON_Delete(root); if (fp) fclose(fp); return terrno; @@ -945,7 +944,7 @@ static int32_t vnodeReadVersion(SVnodeObj *pVnode) { vInfo("vgId:%d, read vnode version successfully, version:%" PRId64, pVnode->vgId, pVnode->version); PARSE_OVER: - tfree(content); + taosTFree(content); cJSON_Delete(root); if(fp) fclose(fp); return terrno; diff --git a/src/wal/src/walMain.c b/src/wal/src/walMain.c index bf3b0f63ea..4ac8a096c6 100644 --- a/src/wal/src/walMain.c +++ b/src/wal/src/walMain.c @@ -22,6 +22,7 @@ #include "taoserror.h" #include "twal.h" #include "tqueue.h" +#define TAOS_RANDOM_FILE_FAIL_TEST #define walPrefix "wal" @@ -128,7 +129,7 @@ void walClose(void *handle) { if (handle == NULL) return; SWal *pWal = handle; - tclose(pWal->fd); + taosClose(pWal->fd); if (pWal->timer) taosTmrStopA(&pWal->timer); if (pWal->keep == 0) { diff --git a/tests/test/c/createTablePerformance.c b/tests/test/c/createTablePerformance.c index 69022bdecf..eae104291a 100644 --- a/tests/test/c/createTablePerformance.c +++ b/tests/test/c/createTablePerformance.c @@ -18,7 +18,6 @@ #include "taoserror.h" #include "taos.h" #include "tulog.h" -#include "ttime.h" #include "tutil.h" #include "tglobal.h" #include "hash.h" diff --git a/tests/test/c/hashPerformance.c b/tests/test/c/hashPerformance.c index fa3612d6e8..db3be0e950 100644 --- a/tests/test/c/hashPerformance.c +++ b/tests/test/c/hashPerformance.c @@ -17,7 +17,6 @@ #include "os.h" #include "taos.h" #include "tulog.h" -#include "ttime.h" #include "tutil.h" #include "hash.h" diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 3469180607..fdec38b4c7 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -20,6 +20,7 @@ #include "tglobal.h" #include "tutil.h" #include "cJSON.h" +#undef TAOS_MEM_CHECK void simLogSql(char *sql, bool useSharp) { static FILE *fp = NULL; diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index 5b2fc87307..ef1a488f60 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -16,6 +16,7 @@ #include "os.h" #include "tglobal.h" #include "sim.h" +#undef TAOS_MEM_CHECK bool simAsyncQuery = false; diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 9fea1f115b..2528fde9f5 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -57,12 +57,11 @@ * */ +#include "os.h" #include "sim.h" -#include -#include -#include #include "simParse.h" #include "tutil.h" +#undef TAOS_MEM_CHECK static SCommand *cmdHashList[MAX_NUM_CMD]; static SCmdLine cmdLine[MAX_CMD_LINES]; diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index b50c853ea8..17df7f306a 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -20,6 +20,7 @@ #include "ttimer.h" #include "tutil.h" #include "tsocket.h" +#undef TAOS_MEM_CHECK SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; @@ -94,9 +95,9 @@ void simFreeScript(SScript *script) { } taos_close(script->taos); - tfree(script->lines); - tfree(script->optionBuffer); - tfree(script); + taosTFree(script->lines); + taosTFree(script->optionBuffer); + taosTFree(script); } SScript *simProcessCallOver(SScript *script) {